From 5ce5042ac40eccd23f16f882edaf15ea2122ee9e Mon Sep 17 00:00:00 2001 From: susmonteiro Date: Wed, 7 May 2025 11:56:27 +0100 Subject: [PATCH 01/54] [Swift C++ Interop] Propagate hardening build setting to Swift --- .../SWBUniversalPlatform/Specs/Swift.xcspec | 15 +++ .../SwiftTaskConstructionTests.swift | 96 +++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/Sources/SWBUniversalPlatform/Specs/Swift.xcspec b/Sources/SWBUniversalPlatform/Specs/Swift.xcspec index 67972fa7..f7de6969 100644 --- a/Sources/SWBUniversalPlatform/Specs/Swift.xcspec +++ b/Sources/SWBUniversalPlatform/Specs/Swift.xcspec @@ -1303,6 +1303,21 @@ }; }, + // Hidden clang importer options to control C++ behavior + // in the clang importer, not visible in build settings. + { + Name = "SWIFT_CLANG_CXX_STANDARD_LIBRARY_HARDENING"; + Type = String; + DefaultValue = "$(CLANG_CXX_STANDARD_LIBRARY_HARDENING)"; + CommandLineArgs = { + "none" = ("-Xcc", "-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_NONE"); + "fast" = ("-Xcc", "-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_FAST"); + "extensive" = ("-Xcc", "-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_EXTENSIVE"); + "debug" = ("-Xcc", "-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG"); + "<>" = (); + }; + }, + { Name = "SWIFT_OVERLOAD_PREBUILT_MODULE_CACHE_PATH"; Type = Path; diff --git a/Tests/SWBTaskConstructionTests/SwiftTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/SwiftTaskConstructionTests.swift index 89277df7..ccff18ef 100644 --- a/Tests/SWBTaskConstructionTests/SwiftTaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/SwiftTaskConstructionTests.swift @@ -3984,6 +3984,102 @@ fileprivate struct SwiftTaskConstructionTests: CoreBasedTests { } } + @Test(.requireSDKs(.macOS)) + func enableHardeningInSwift() async throws { + + func setupHardeningTest(_ tmpDir: Path, + hardeningMode: String) async throws -> TaskConstructionTester { + let testProject = try await TestProject( + "TestProject", + sourceRoot: tmpDir, + groupTree: TestGroup( + "SomeFiles", + children: [ + TestFile("source.swift"), + TestFile("source.cpp") + ]), + targets: [ + TestStandardTarget( + "testFramework", type: .framework, + buildConfigurations: [ + TestBuildConfiguration("Debug", buildSettings: [ + "GENERATE_INFOPLIST_FILE": "YES", + "PRODUCT_NAME": "$(TARGET_NAME)", + "SWIFT_EXEC": swiftCompilerPath.str, + "SWIFT_VERSION": swiftVersion, + "CLANG_CXX_STANDARD_LIBRARY_HARDENING": hardeningMode + ]), + ], + buildPhases: [ + TestSourcesBuildPhase(["source.swift", "source.cpp"]) + ] + ) + ]) + let tester = try await TaskConstructionTester(getCore(), testProject) + return tester + } + + // Verify that we don't enable hardening in Swift compilations when C++ + // hardening is none. + try await withTemporaryDirectory { tmpDir in + let tester = try await setupHardeningTest(tmpDir, hardeningMode: "none") + await tester.checkBuild(runDestination: .macOS) { results in + results.checkTask(.matchRuleType("SwiftDriver Compilation")) { task in + task.checkCommandLineContainsUninterrupted(["-Xcc", "-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_NONE"]) + } + } + } + + try await withTemporaryDirectory { tmpDir in + let tester = try await setupHardeningTest(tmpDir, hardeningMode: "fast") + await tester.checkBuild(runDestination: .macOS) { results in + results.checkTask(.matchRuleType("SwiftDriver Compilation")) { task in + task.checkCommandLineContainsUninterrupted(["-Xcc", "-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_FAST"]) + } + } + } + + try await withTemporaryDirectory { tmpDir in + let tester = try await setupHardeningTest(tmpDir, hardeningMode: "extensive") + await tester.checkBuild(runDestination: .macOS) { results in + results.checkTask(.matchRuleType("SwiftDriver Compilation")) { task in + task.checkCommandLineContainsUninterrupted(["-Xcc", "-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_EXTENSIVE"]) + } + } + } + + try await withTemporaryDirectory { tmpDir in + let tester = try await setupHardeningTest(tmpDir, hardeningMode: "debug") + await tester.checkBuild(runDestination: .macOS) { results in + results.checkTask(.matchRuleType("SwiftDriver Compilation")) { task in + task.checkCommandLineContainsUninterrupted(["-Xcc", "-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG"]) + } + } + } + + // Verify that we don't enable hardening in Swift compilations when C++ + // hardening mode is garbage. + try await withTemporaryDirectory { tmpDir in + let tester = try await setupHardeningTest(tmpDir, hardeningMode: "unexpected") + await tester.checkBuild(runDestination: .macOS) { results in + results.checkTask(.matchRuleType("SwiftDriver Compilation")) { task in + task.checkCommandLineNoMatch([.prefix("-D_LIBCPP_HARDENING_MODE=")]) + } + } + } + + // Verify that we don't enable hardening in Swift compilations when C++ + // hardening mode is empty. + try await withTemporaryDirectory { tmpDir in + let tester = try await setupHardeningTest(tmpDir, hardeningMode: "") + await tester.checkBuild(runDestination: .macOS) { results in + results.checkTask(.matchRuleType("SwiftDriver Compilation")) { task in + task.checkCommandLineNoMatch([.prefix("-D_LIBCPP_HARDENING_MODE=")]) + } + } + } + } + @Test(.requireSDKs(.macOS)) func cxxInteropLinkerArgGeneration() async throws { // When Swift is generating additional linker args, we should not try to inject the response file when a target is a dependent of a cxx-interop target but has no Swift source of its own. From f564f04fb749252f84f6a486604924494b74417d Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Wed, 28 May 2025 11:02:14 -0700 Subject: [PATCH 02/54] Update automerger PR workflow --- .github/workflows/automerge.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml index 7d8af0d3..bcb856ae 100644 --- a/.github/workflows/automerge.yml +++ b/.github/workflows/automerge.yml @@ -10,7 +10,8 @@ jobs: name: Create PR to merge main into release branch uses: swiftlang/github-workflows/.github/workflows/create_automerge_pr.yml@main with: - base_branch: release/6.2 + head_branch: release/6.2 + base_branch: main permissions: contents: write pull-requests: write From a76b0aa795b6043e37f1f1ef2b6cac10ef631513 Mon Sep 17 00:00:00 2001 From: Jake Petroules Date: Wed, 28 May 2025 13:53:23 -0700 Subject: [PATCH 03/54] Update swift-system to 1.5.0 This is required for System.FilePath to work correctly on Windows in release mode. --- Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index ea2accbc..05599230 100644 --- a/Package.swift +++ b/Package.swift @@ -456,7 +456,7 @@ if useLocalDependencies { } else { package.dependencies += [ .package(url: "https://github.com/swiftlang/swift-driver.git", branch: "main"), - .package(url: "https://github.com/apple/swift-system.git", .upToNextMajor(from: "1.4.1")), + .package(url: "https://github.com/apple/swift-system.git", .upToNextMajor(from: "1.5.0")), .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.0.3"), ] if !useLLBuildFramework { From be93573d5f903e2540c5e740e2160789c5b57d6e Mon Sep 17 00:00:00 2001 From: Jake Petroules Date: Wed, 28 May 2025 13:53:23 -0700 Subject: [PATCH 04/54] Simplify fallback developer directory calculation Have the fallback logic specify the type of developer directory as well as just the location, and eliminate the initialization distinction on whether a Core was initialized with a "fallback" path vs one explicitly given, since it shouldn't matter. --- Sources/SWBApplePlatform/Plugin.swift | 4 +- Sources/SWBCore/Core.swift | 28 ++---- .../DeveloperDirectoryExtension.swift | 2 +- Sources/SWBCore/MacroConfigFileLoader.swift | 2 +- Sources/SWBCore/Settings/Settings.swift | 2 +- Sources/SWBGenericUnixPlatform/Plugin.swift | 4 +- Sources/SWBTestSupport/CoreTestSupport.swift | 2 +- .../SWBTestSupport/DummyCommandProducer.swift | 2 +- Sources/SWBWindowsPlatform/Plugin.swift | 14 +-- Tests/SWBCoreTests/CoreTests.swift | 94 ++++++++++--------- .../SWBCoreTests/PlatformRegistryTests.swift | 2 +- .../SWBCoreTests/ToolchainRegistryTests.swift | 14 ++- 12 files changed, 83 insertions(+), 87 deletions(-) diff --git a/Sources/SWBApplePlatform/Plugin.swift b/Sources/SWBApplePlatform/Plugin.swift index 6be46f8e..2288a849 100644 --- a/Sources/SWBApplePlatform/Plugin.swift +++ b/Sources/SWBApplePlatform/Plugin.swift @@ -31,8 +31,8 @@ import SWBTaskConstruction } struct AppleDeveloperDirectoryExtension: DeveloperDirectoryExtension { - func fallbackDeveloperDirectory(hostOperatingSystem: OperatingSystem) async throws -> Path? { - try await hostOperatingSystem == .macOS ? Xcode.getActiveDeveloperDirectoryPath() : nil + func fallbackDeveloperDirectory(hostOperatingSystem: OperatingSystem) async throws -> Core.DeveloperPath? { + try await hostOperatingSystem == .macOS ? .xcode(Xcode.getActiveDeveloperDirectoryPath()) : nil } } diff --git a/Sources/SWBCore/Core.swift b/Sources/SWBCore/Core.swift index 4864e81c..238a2e92 100644 --- a/Sources/SWBCore/Core.swift +++ b/Sources/SWBCore/Core.swift @@ -70,14 +70,9 @@ public final class Core: Sendable { delegate.error("Could not determine path to developer directory because no extensions provided a fallback value") return nil case 1: - let path = values[0] - if path.str.hasSuffix(".app/Contents/Developer") { - resolvedDeveloperPath = .xcode(path) - } else { - resolvedDeveloperPath = .fallback(values[0]) - } + resolvedDeveloperPath = values[0] default: - delegate.error("Could not determine path to developer directory because multiple extensions provided conflicting fallback values: \(values.sorted().map { $0.str }.joined(separator: ", "))") + delegate.error("Could not determine path to developer directory because multiple extensions provided conflicting fallback values: \(values.map { $0.path.str }.sorted().joined(separator: ", "))") return nil } } @@ -181,12 +176,9 @@ public final class Core: Sendable { // A path to the root of a Swift toolchain, optionally paired with the developer path of an installed Xcode case swiftToolchain(Path, xcodeDeveloperPath: Path?) - // A fallback resolved path. - case fallback(Path) - public var path: Path { switch self { - case .xcode(let path), .swiftToolchain(let path, xcodeDeveloperPath: _), .fallback(let path): + case .xcode(let path), .swiftToolchain(let path, xcodeDeveloperPath: _): return path } } @@ -259,7 +251,7 @@ public final class Core: Sendable { self.xcodeProductBuildVersion = ProductBuildVersion(major: 99, train: "T", build: 999) self.xcodeProductBuildVersionString = xcodeProductBuildVersion.description } - case .swiftToolchain, .fallback: + case .swiftToolchain: // FIXME: Eliminate this requirment for Swift toolchains self.xcodeVersion = Version(99, 99, 99) self.xcodeProductBuildVersion = ProductBuildVersion(major: 99, train: "T", build: 999) @@ -277,12 +269,14 @@ public final class Core: Sendable { case .xcode(let path): toolchainPaths.append((path.join("Toolchains"), strict: path.str.hasSuffix(".app/Contents/Developer"))) case .swiftToolchain(let path, xcodeDeveloperPath: let xcodeDeveloperPath): - toolchainPaths.append((path, strict: true)) + if hostOperatingSystem == .windows { + toolchainPaths.append((path.join("Toolchains"), strict: true)) + } else { + toolchainPaths.append((path, strict: true)) + } if let xcodeDeveloperPath { toolchainPaths.append((xcodeDeveloperPath.join("Toolchains"), strict: xcodeDeveloperPath.str.hasSuffix(".app/Contents/Developer"))) } - case .fallback(let path): - toolchainPaths.append((path.join("Toolchains"), strict: false)) } // FIXME: We should support building the toolchain locally (for `inferiorProductsPath`). @@ -418,7 +412,7 @@ public final class Core: Sendable { let pluginPath = path.join("usr/lib/libToolchainCASPlugin.dylib") let plugin = try? ToolchainCASPlugin(dylib: pluginPath) casPlugin = plugin - case .swiftToolchain, .fallback: + case .swiftToolchain: // Unimplemented break } @@ -454,8 +448,6 @@ public final class Core: Sendable { } else { searchPaths = [] } - case .fallback: - searchPaths = [] } } if let additionalPlatformSearchPaths = getEnvironmentVariable("XCODE_EXTRA_PLATFORM_FOLDERS") { diff --git a/Sources/SWBCore/Extensions/DeveloperDirectoryExtension.swift b/Sources/SWBCore/Extensions/DeveloperDirectoryExtension.swift index 9dc0db37..3e69177c 100644 --- a/Sources/SWBCore/Extensions/DeveloperDirectoryExtension.swift +++ b/Sources/SWBCore/Extensions/DeveloperDirectoryExtension.swift @@ -21,5 +21,5 @@ public struct DeveloperDirectoryExtensionPoint: ExtensionPoint { } public protocol DeveloperDirectoryExtension: Sendable { - func fallbackDeveloperDirectory(hostOperatingSystem: OperatingSystem) async throws -> Path? + func fallbackDeveloperDirectory(hostOperatingSystem: OperatingSystem) async throws -> Core.DeveloperPath? } diff --git a/Sources/SWBCore/MacroConfigFileLoader.swift b/Sources/SWBCore/MacroConfigFileLoader.swift index 224e5e9f..2d5e4248 100644 --- a/Sources/SWBCore/MacroConfigFileLoader.swift +++ b/Sources/SWBCore/MacroConfigFileLoader.swift @@ -133,7 +133,7 @@ final class MacroConfigFileLoader: Sendable { // FIXME: Move this to its proper home, and support the other special cases Xcode has (PLATFORM_DIR and SDK_DIR). This should move to using a generic facility, e.g., source trees: Add search paths for .xcconfig macros to match what Xcode has if path.str.hasPrefix("") { switch developerPath { - case .xcode(let developerPath), .swiftToolchain(let developerPath, _), .fallback(let developerPath): + case .xcode(let developerPath), .swiftToolchain(let developerPath, _): path = Path(path.str.replacingOccurrences(of: "", with: developerPath.str)) } } diff --git a/Sources/SWBCore/Settings/Settings.swift b/Sources/SWBCore/Settings/Settings.swift index eb3b72cc..bc9b7e62 100644 --- a/Sources/SWBCore/Settings/Settings.swift +++ b/Sources/SWBCore/Settings/Settings.swift @@ -1026,7 +1026,7 @@ extension WorkspaceContext { // Add the standard search paths. switch core.developerPath { - case .xcode(let path), .fallback(let path): + case .xcode(let path): paths.append(path.join("usr").join("bin")) paths.append(path.join("usr").join("local").join("bin")) case .swiftToolchain(let path, let xcodeDeveloperPath): diff --git a/Sources/SWBGenericUnixPlatform/Plugin.swift b/Sources/SWBGenericUnixPlatform/Plugin.swift index e4679961..128fde75 100644 --- a/Sources/SWBGenericUnixPlatform/Plugin.swift +++ b/Sources/SWBGenericUnixPlatform/Plugin.swift @@ -23,13 +23,13 @@ import Foundation } struct GenericUnixDeveloperDirectoryExtension: DeveloperDirectoryExtension { - func fallbackDeveloperDirectory(hostOperatingSystem: OperatingSystem) async throws -> Path? { + func fallbackDeveloperDirectory(hostOperatingSystem: OperatingSystem) async throws -> Core.DeveloperPath? { if hostOperatingSystem == .windows || hostOperatingSystem == .macOS { // Handled by the Windows and Apple plugins return nil } - return .root + return .swiftToolchain(.root, xcodeDeveloperPath: nil) } } diff --git a/Sources/SWBTestSupport/CoreTestSupport.swift b/Sources/SWBTestSupport/CoreTestSupport.swift index 356345fa..e95da70f 100644 --- a/Sources/SWBTestSupport/CoreTestSupport.swift +++ b/Sources/SWBTestSupport/CoreTestSupport.swift @@ -38,7 +38,7 @@ extension Core { if hostOperatingSystem == .macOS { developerPath = .xcode(try await Xcode.getActiveDeveloperDirectoryPath()) } else { - developerPath = .fallback(Path.root) + developerPath = .swiftToolchain(.root, xcodeDeveloperPath: nil) } let delegate = TestingCoreDelegate() return await (try Core(delegate: delegate, hostOperatingSystem: hostOperatingSystem, pluginManager: PluginManager(skipLoadingPluginIdentifiers: []), developerPath: developerPath, resourceSearchPaths: [], inferiorProductsPath: nil, additionalContentPaths: [], environment: [:], buildServiceModTime: Date(), connectionMode: .inProcess), delegate.diagnostics) diff --git a/Sources/SWBTestSupport/DummyCommandProducer.swift b/Sources/SWBTestSupport/DummyCommandProducer.swift index de335df2..021ee8fd 100644 --- a/Sources/SWBTestSupport/DummyCommandProducer.swift +++ b/Sources/SWBTestSupport/DummyCommandProducer.swift @@ -60,7 +60,7 @@ package struct MockCommandProducer: CommandProducer, Sendable { paths.append(path) } switch core.developerPath { - case .xcode(let path), .fallback(let path): + case .xcode(let path): paths.append(path.join("usr").join("bin")) paths.append(path.join("usr").join("local").join("bin")) case .swiftToolchain(let path, xcodeDeveloperPath: let xcodeDeveloperPath): diff --git a/Sources/SWBWindowsPlatform/Plugin.swift b/Sources/SWBWindowsPlatform/Plugin.swift index 34037862..9a2c7e40 100644 --- a/Sources/SWBWindowsPlatform/Plugin.swift +++ b/Sources/SWBWindowsPlatform/Plugin.swift @@ -53,14 +53,14 @@ public final class WindowsPlugin: Sendable { } struct WindowsDeveloperDirectoryExtension: DeveloperDirectoryExtension { - func fallbackDeveloperDirectory(hostOperatingSystem: OperatingSystem) async throws -> Path? { + func fallbackDeveloperDirectory(hostOperatingSystem: OperatingSystem) async throws -> Core.DeveloperPath? { guard hostOperatingSystem == .windows else { return nil } guard let userProgramFiles = URL.userProgramFiles, let swiftPath = try? userProgramFiles.appending(component: "Swift").filePath else { throw StubError.error("Could not determine path to user program files") } - return swiftPath + return .swiftToolchain(swiftPath, xcodeDeveloperPath: nil) } } @@ -94,15 +94,7 @@ struct WindowsPlatformExtension: PlatformInfoExtension { return [] } - let platformsPath: Path - switch context.developerPath { - case .xcode(let path): - platformsPath = path.join("Platforms") - case .swiftToolchain(let path, _): - platformsPath = path.join("Platforms") - case .fallback(let path): - platformsPath = path.join("Platforms") - } + let platformsPath = context.developerPath.path.join("Platforms") return try context.fs.listdir(platformsPath).compactMap { version in let versionedPlatformsPath = platformsPath.join(version) guard context.fs.isDirectory(versionedPlatformsPath) else { diff --git a/Tests/SWBCoreTests/CoreTests.swift b/Tests/SWBCoreTests/CoreTests.swift index b19b7a33..36fbd821 100644 --- a/Tests/SWBCoreTests/CoreTests.swift +++ b/Tests/SWBCoreTests/CoreTests.swift @@ -320,6 +320,7 @@ import SWBServiceCore // Validate that the core fails if there are loading errors. try await withTemporaryDirectory { tmpDirPath in let fakePlatformPath = tmpDirPath.join("Platforms/Fake.platform") + try localFS.createDirectory(tmpDirPath.join("Toolchains"), recursive: true) try localFS.createDirectory(fakePlatformPath, recursive: true) try await localFS.writePlist(fakePlatformPath.join("Info.plist"), .plDict([ "Description": .plString("Fake"), @@ -334,7 +335,7 @@ import SWBServiceCore let pluginManager = await PluginManager(skipLoadingPluginIdentifiers: []) await pluginManager.registerExtensionPoint(SpecificationsExtensionPoint()) await pluginManager.register(BuiltinSpecsExtension(), type: SpecificationsExtensionPoint.self) - let core = await Core.getInitializedCore(delegate, pluginManager: pluginManager, developerPath: .fallback(tmpDirPath), buildServiceModTime: Date(), connectionMode: .inProcess) + let core = await Core.getInitializedCore(delegate, pluginManager: pluginManager, developerPath: .swiftToolchain(tmpDirPath, xcodeDeveloperPath: nil), buildServiceModTime: Date(), connectionMode: .inProcess) #expect(core == nil) let results = CoreDelegateResults(delegate.diagnostics) @@ -346,69 +347,75 @@ import SWBServiceCore @Test(.skipIfEnvironmentVariableSet(key: .externalToolchainsDir)) func externalToolchainsDir() async throws { try await withTemporaryDirectory { tmpDir in + try localFS.createDirectory(tmpDir.join("Toolchains")) + let originalToolchain = try await toolchainPathsCount() - try await testExternalToolchainPath(withSetEnv: nil, expecting: [], originalToolchain) - try await testExternalToolchainPath(withSetEnv: tmpDir.join("tmp/Foobar/MyDir").str, expecting: [tmpDir.join("tmp/Foobar/MyDir").str], originalToolchain) - try await testExternalToolchainPath(withSetEnv: nil, expecting: [], originalToolchain) - try await testExternalToolchainPath(withSetEnv: [tmpDir.join("tmp/MetalToolchain1.0").str, tmpDir.join("tmp/MetalToolchain2.0").str, tmpDir.join("tmp/MetalToolchain3.0").str].joined(separator: String(Path.pathEnvironmentSeparator)), expecting: [ + try await testExternalToolchainPath(toolchainPath: tmpDir, withSetEnv: nil, expecting: [], originalToolchain) + try await testExternalToolchainPath(toolchainPath: tmpDir, withSetEnv: tmpDir.join("tmp/Foobar/MyDir").str, expecting: [tmpDir.join("tmp/Foobar/MyDir").str], originalToolchain) + try await testExternalToolchainPath(toolchainPath: tmpDir, withSetEnv: nil, expecting: [], originalToolchain) + try await testExternalToolchainPath(toolchainPath: tmpDir, withSetEnv: [tmpDir.join("tmp/MetalToolchain1.0").str, tmpDir.join("tmp/MetalToolchain2.0").str, tmpDir.join("tmp/MetalToolchain3.0").str].joined(separator: String(Path.pathEnvironmentSeparator)), expecting: [ tmpDir.join("tmp/MetalToolchain1.0").str, tmpDir.join("tmp/MetalToolchain2.0").str, tmpDir.join("tmp/MetalToolchain3.0").str, ], originalToolchain) - try await testExternalToolchainPath(withSetEnv: nil, expecting: [], originalToolchain) - try await testExternalToolchainPath(withSetEnv: "", expecting: [], originalToolchain) + try await testExternalToolchainPath(toolchainPath: tmpDir, withSetEnv: nil, expecting: [], originalToolchain) + try await testExternalToolchainPath(toolchainPath: tmpDir, withSetEnv: "", expecting: [], originalToolchain) // Environment overrides - try await testExternalToolchainPath(withSetEnv: nil, expecting: [], originalToolchain) // Clear + try await testExternalToolchainPath(toolchainPath: tmpDir, withSetEnv: nil, expecting: [], originalToolchain) // Clear - try await testExternalToolchainPath(environmentOverrides: ["Hello":"world"], expecting: [], originalToolchain) - try await testExternalToolchainPath(environmentOverrides: ["EXTERNAL_TOOLCHAINS_DIR": tmpDir.join("tmp/Foobar/MyDir").str], expecting: [tmpDir.join("tmp/Foobar/MyDir").str], originalToolchain) - try await testExternalToolchainPath(environmentOverrides: [:], expecting: [], originalToolchain) - try await testExternalToolchainPath(environmentOverrides: [ + try await testExternalToolchainPath(toolchainPath: tmpDir, environmentOverrides: ["Hello":"world"], expecting: [], originalToolchain) + try await testExternalToolchainPath(toolchainPath: tmpDir, environmentOverrides: ["EXTERNAL_TOOLCHAINS_DIR": tmpDir.join("tmp/Foobar/MyDir").str], expecting: [tmpDir.join("tmp/Foobar/MyDir").str], originalToolchain) + try await testExternalToolchainPath(toolchainPath: tmpDir, environmentOverrides: [:], expecting: [], originalToolchain) + try await testExternalToolchainPath(toolchainPath: tmpDir, environmentOverrides: [ "EXTERNAL_TOOLCHAINS_DIR" : [tmpDir.join("tmp/MetalToolchain1.0").str, tmpDir.join("tmp/MetalToolchain2.0").str, tmpDir.join("tmp/MetalToolchain3.0").str].joined(separator: String(Path.pathEnvironmentSeparator)), ], expecting: [ tmpDir.join("tmp/MetalToolchain1.0").str, tmpDir.join("tmp/MetalToolchain2.0").str, tmpDir.join("tmp/MetalToolchain3.0").str, ], originalToolchain) - try await testExternalToolchainPath(environmentOverrides: [:], expecting: [], originalToolchain) + try await testExternalToolchainPath(toolchainPath: tmpDir, environmentOverrides: [:], expecting: [], originalToolchain) } } func toolchainPathsCount() async throws -> Int { - let delegate = Delegate() - let pluginManager = await PluginManager(skipLoadingPluginIdentifiers: []) - await pluginManager.registerExtensionPoint(DeveloperDirectoryExtensionPoint()) - await pluginManager.registerExtensionPoint(SpecificationsExtensionPoint()) - await pluginManager.registerExtensionPoint(ToolchainRegistryExtensionPoint()) - await pluginManager.register(BuiltinSpecsExtension(), type: SpecificationsExtensionPoint.self) - struct MockDeveloperDirectoryExtensionPoint: DeveloperDirectoryExtension { - func fallbackDeveloperDirectory(hostOperatingSystem: OperatingSystem) async throws -> Path? { - .root + try await withTemporaryDirectory { tmpDir in + try localFS.createDirectory(tmpDir.join("Toolchains")) + let delegate = Delegate() + let pluginManager = await PluginManager(skipLoadingPluginIdentifiers: []) + await pluginManager.registerExtensionPoint(DeveloperDirectoryExtensionPoint()) + await pluginManager.registerExtensionPoint(SpecificationsExtensionPoint()) + await pluginManager.registerExtensionPoint(ToolchainRegistryExtensionPoint()) + await pluginManager.register(BuiltinSpecsExtension(), type: SpecificationsExtensionPoint.self) + struct MockDeveloperDirectoryExtensionPoint: DeveloperDirectoryExtension { + let toolchainPath: Path + func fallbackDeveloperDirectory(hostOperatingSystem: OperatingSystem) async throws -> Core.DeveloperPath? { + .swiftToolchain(toolchainPath, xcodeDeveloperPath: nil) + } } - } - struct MockToolchainExtension: ToolchainRegistryExtension { - func additionalToolchains(context: any ToolchainRegistryExtensionAdditionalToolchainsContext) async throws -> [Toolchain] { - guard context.toolchainRegistry.lookup(ToolchainRegistry.defaultToolchainIdentifier) == nil else { - return [] + struct MockToolchainExtension: ToolchainRegistryExtension { + func additionalToolchains(context: any ToolchainRegistryExtensionAdditionalToolchainsContext) async throws -> [Toolchain] { + guard context.toolchainRegistry.lookup(ToolchainRegistry.defaultToolchainIdentifier) == nil else { + return [] + } + return [Toolchain(identifier: ToolchainRegistry.defaultToolchainIdentifier, displayName: "Mock", version: Version(), aliases: ["default"], path: .root, frameworkPaths: [], libraryPaths: [], defaultSettings: [:], overrideSettings: [:], defaultSettingsWhenPrimary: [:], executableSearchPaths: [], testingLibraryPlatformNames: [], fs: context.fs)] } - return [Toolchain(identifier: ToolchainRegistry.defaultToolchainIdentifier, displayName: "Mock", version: Version(), aliases: ["default"], path: .root, frameworkPaths: [], libraryPaths: [], defaultSettings: [:], overrideSettings: [:], defaultSettingsWhenPrimary: [:], executableSearchPaths: [], testingLibraryPlatformNames: [], fs: context.fs)] } - } - await pluginManager.register(MockDeveloperDirectoryExtensionPoint(), type: DeveloperDirectoryExtensionPoint.self) - await pluginManager.register(MockToolchainExtension(), type: ToolchainRegistryExtensionPoint.self) - let core = await Core.getInitializedCore(delegate, pluginManager: pluginManager, inferiorProductsPath: Path.root.join("invalid"), environment: [:], buildServiceModTime: Date(), connectionMode: .inProcess) - for diagnostic in delegate.diagnostics { - if diagnostic.formatLocalizedDescription(.debug).hasPrefix("warning: found previously-unknown deployment target macro ") { - continue + await pluginManager.register(MockDeveloperDirectoryExtensionPoint(toolchainPath: tmpDir), type: DeveloperDirectoryExtensionPoint.self) + await pluginManager.register(MockToolchainExtension(), type: ToolchainRegistryExtensionPoint.self) + let core = await Core.getInitializedCore(delegate, pluginManager: pluginManager, inferiorProductsPath: Path.root.join("invalid"), environment: [:], buildServiceModTime: Date(), connectionMode: .inProcess) + for diagnostic in delegate.diagnostics { + if diagnostic.formatLocalizedDescription(.debug).hasPrefix("warning: found previously-unknown deployment target macro ") { + continue + } + Issue.record("\(diagnostic.formatLocalizedDescription(.debug))") } - Issue.record("\(diagnostic.formatLocalizedDescription(.debug))") + return try #require(core?.toolchainPaths).count } - return try #require(core?.toolchainPaths).count } - func testExternalToolchainPath(withSetEnv externalToolchainPathsString: String?, expecting expectedPathStrings: [String], _ originalToolchainCount: Int) async throws { + func testExternalToolchainPath(toolchainPath: Path, withSetEnv externalToolchainPathsString: String?, expecting expectedPathStrings: [String], _ originalToolchainCount: Int) async throws { var env = Environment.current.filter { $0.key != .externalToolchainsDir } if let externalToolchainPathsString { env[.externalToolchainsDir] = externalToolchainPathsString @@ -417,11 +424,11 @@ import SWBServiceCore try await withEnvironment(env, clean: true) { #expect(getEnvironmentVariable(.externalToolchainsDir) == externalToolchainPathsString) - try await testExternalToolchainPath(environmentOverrides: [:], expecting: expectedPathStrings, originalToolchainCount) + try await testExternalToolchainPath(toolchainPath: toolchainPath, environmentOverrides: [:], expecting: expectedPathStrings, originalToolchainCount) } } - func testExternalToolchainPath(environmentOverrides: [String:String], expecting expectedPathStrings: [String], _ originalToolchainCount: Int) async throws { + func testExternalToolchainPath(toolchainPath: Path, environmentOverrides: [String:String], expecting expectedPathStrings: [String], _ originalToolchainCount: Int) async throws { let delegate = Delegate() let pluginManager = await PluginManager(skipLoadingPluginIdentifiers: []) await pluginManager.registerExtensionPoint(DeveloperDirectoryExtensionPoint()) @@ -429,8 +436,9 @@ import SWBServiceCore await pluginManager.registerExtensionPoint(ToolchainRegistryExtensionPoint()) await pluginManager.register(BuiltinSpecsExtension(), type: SpecificationsExtensionPoint.self) struct MockDeveloperDirectoryExtensionPoint: DeveloperDirectoryExtension { - func fallbackDeveloperDirectory(hostOperatingSystem: OperatingSystem) async throws -> Path? { - .root + let toolchainPath: Path + func fallbackDeveloperDirectory(hostOperatingSystem: OperatingSystem) async throws -> Core.DeveloperPath? { + .swiftToolchain(toolchainPath, xcodeDeveloperPath: nil) } } struct MockToolchainExtension: ToolchainRegistryExtension { @@ -441,7 +449,7 @@ import SWBServiceCore return [Toolchain(identifier: ToolchainRegistry.defaultToolchainIdentifier, displayName: "Mock", version: Version(), aliases: ["default"], path: .root, frameworkPaths: [], libraryPaths: [], defaultSettings: [:], overrideSettings: [:], defaultSettingsWhenPrimary: [:], executableSearchPaths: [], testingLibraryPlatformNames: [], fs: context.fs)] } } - await pluginManager.register(MockDeveloperDirectoryExtensionPoint(), type: DeveloperDirectoryExtensionPoint.self) + await pluginManager.register(MockDeveloperDirectoryExtensionPoint(toolchainPath: toolchainPath), type: DeveloperDirectoryExtensionPoint.self) await pluginManager.register(MockToolchainExtension(), type: ToolchainRegistryExtensionPoint.self) let core = await Core.getInitializedCore(delegate, pluginManager: pluginManager, inferiorProductsPath: Path.root.join("invalid"), environment: environmentOverrides, buildServiceModTime: Date(), connectionMode: .inProcess) for diagnostic in delegate.diagnostics { diff --git a/Tests/SWBCoreTests/PlatformRegistryTests.swift b/Tests/SWBCoreTests/PlatformRegistryTests.swift index 88dafa57..ae74a237 100644 --- a/Tests/SWBCoreTests/PlatformRegistryTests.swift +++ b/Tests/SWBCoreTests/PlatformRegistryTests.swift @@ -49,7 +49,7 @@ import SWBMacro } var developerPath: Core.DeveloperPath { - .fallback(Path.temporaryDirectory) + .swiftToolchain(.temporaryDirectory, xcodeDeveloperPath: nil) } } diff --git a/Tests/SWBCoreTests/ToolchainRegistryTests.swift b/Tests/SWBCoreTests/ToolchainRegistryTests.swift index e8f8753d..60510119 100644 --- a/Tests/SWBCoreTests/ToolchainRegistryTests.swift +++ b/Tests/SWBCoreTests/ToolchainRegistryTests.swift @@ -30,7 +30,10 @@ import SWBServiceCore infoPlistName: String = "ToolchainInfo.plist", postProcess: (Path) throws -> Void = { _ in }, perform: (ToolchainRegistry, [(String, String)], [(String, String)]) throws -> Void) async throws { - try await withTemporaryDirectory { tmpDirPath in + try await withTemporaryDirectory { baseTmpDirPath in + let tmpDirPath = baseTmpDirPath.join("tmp") + try fs.createDirectory(tmpDirPath) + try fs.createDirectory(baseTmpDirPath.join("Swift").join("Toolchains"), recursive: true) for (name, dataOpt) in inputs { let itemPath = tmpDirPath.join(name).join(infoPlistName) @@ -76,8 +79,9 @@ import SWBServiceCore await pluginManager.registerExtensionPoint(ToolchainRegistryExtensionPoint()) await pluginManager.register(BuiltinSpecsExtension(), type: SpecificationsExtensionPoint.self) struct MockDeveloperDirectoryExtensionPoint: DeveloperDirectoryExtension { - func fallbackDeveloperDirectory(hostOperatingSystem: OperatingSystem) async throws -> Path? { - .root + let toolchainPath: Path + func fallbackDeveloperDirectory(hostOperatingSystem: OperatingSystem) async throws -> Core.DeveloperPath? { + .swiftToolchain(toolchainPath, xcodeDeveloperPath: nil) } } struct MockToolchainExtension: ToolchainRegistryExtension { @@ -88,7 +92,7 @@ import SWBServiceCore return [Toolchain(identifier: ToolchainRegistry.defaultToolchainIdentifier, displayName: "Mock", version: Version(), aliases: ["default"], path: .root, frameworkPaths: [], libraryPaths: [], defaultSettings: [:], overrideSettings: [:], defaultSettingsWhenPrimary: [:], executableSearchPaths: [], testingLibraryPlatformNames: [], fs: context.fs)] } } - await pluginManager.register(MockDeveloperDirectoryExtensionPoint(), type: DeveloperDirectoryExtensionPoint.self) + await pluginManager.register(MockDeveloperDirectoryExtensionPoint(toolchainPath: baseTmpDirPath.join("Swift")), type: DeveloperDirectoryExtensionPoint.self) await pluginManager.register(MockToolchainExtension(), type: ToolchainRegistryExtensionPoint.self) let coreDelegate = TestingCoreDelegate() let core = await Core.getInitializedCore(coreDelegate, pluginManager: pluginManager, inferiorProductsPath: Path.root.join("invalid"), environment: [:], buildServiceModTime: Date(), connectionMode: .inProcess) @@ -172,7 +176,7 @@ import SWBServiceCore ], infoPlistName: "Info.plist") { registry, _, errors in #expect(Set(registry.toolchainsByIdentifier.keys) == Set(["org.swift.3020161114a", "org.swift.3020161115a"] + additionalToolchains)) - #expect(errors.count == 0) + #expect(errors.count == 0, "\(errors)") #expect(registry.lookup("org.swift.3020161115a")?.identifier == "org.swift.3020161115a") #expect(registry.lookup("org.swift.3020161114a")?.identifier == "org.swift.3020161114a") From 4bb431e1e988ae944e0cb0d7b7bcc3db8299bcc0 Mon Sep 17 00:00:00 2001 From: Sam Khouri Date: Wed, 14 May 2025 22:06:09 -0400 Subject: [PATCH 05/54] Add swift version file to record the Swift toolchain to use When working with SwiftBuild, there is a particular version that development is expected to use so that all of the compilers, standard libraries, test frameworks, etc. are expected to be working. The `.swift-version` file establishes a standard way to record this information. Also, there is tooling available through `swiftly` to use that information to automatically use the correct toolchain in any git commit to set the developer's toolchain when they run and test the package functions at desk. Create the `.swift-version` file and set it to the current expected toolchain version: 6.1. --- .swift-version | 1 + 1 file changed, 1 insertion(+) create mode 100644 .swift-version diff --git a/.swift-version b/.swift-version new file mode 100644 index 00000000..dfda3e0b --- /dev/null +++ b/.swift-version @@ -0,0 +1 @@ +6.1.0 From bce6ce72fcf7a78b14e9beead97b722fcbaef5b8 Mon Sep 17 00:00:00 2001 From: Jake Petroules Date: Thu, 29 May 2025 12:25:48 -0700 Subject: [PATCH 06/54] Correct reference to wrong OS in XCFramework tests Noticed this while working on some other tests; probably doesn't matter but may be confusing so correct it. --- Tests/SWBBuildSystemTests/XCFrameworkBuildOperationTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/SWBBuildSystemTests/XCFrameworkBuildOperationTests.swift b/Tests/SWBBuildSystemTests/XCFrameworkBuildOperationTests.swift index 2d0bcaac..4de49c49 100644 --- a/Tests/SWBBuildSystemTests/XCFrameworkBuildOperationTests.swift +++ b/Tests/SWBBuildSystemTests/XCFrameworkBuildOperationTests.swift @@ -1003,7 +1003,7 @@ fileprivate struct XCFrameworkBuildOperationTests: CoreBasedTests { version: Version(1, 0), libraries: [ XCFramework.Library(libraryIdentifier: "arm64-apple-iphoneos\(infoLookup.loadSDK(.iOS).defaultDeploymentTarget)", supportedPlatform: "ios", supportedArchitectures: ["arm64", "arm64e"], platformVariant: nil, libraryPath: Path("Support.framework"), binaryPath: Path("Support.framework/Support"), headersPath: nil), - XCFramework.Library(libraryIdentifier: "arm64-apple-iphoneos\(infoLookup.loadSDK(.watchOS).defaultDeploymentTarget)", supportedPlatform: "watchos", supportedArchitectures: ["arm64", "arm64_32"], platformVariant: nil, libraryPath: Path("Support.framework"), binaryPath: Path("Support.framework/Support"), headersPath: nil), + XCFramework.Library(libraryIdentifier: "arm64-apple-watchos\(infoLookup.loadSDK(.watchOS).defaultDeploymentTarget)", supportedPlatform: "watchos", supportedArchitectures: ["arm64", "arm64_32"], platformVariant: nil, libraryPath: Path("Support.framework"), binaryPath: Path("Support.framework/Support"), headersPath: nil), ]) try fs.createDirectory(supportXCFrameworkPath, recursive: true) From 0d89241bb1cd17901466b20f7606e9fb9b77a0e8 Mon Sep 17 00:00:00 2001 From: Jake Petroules Date: Thu, 29 May 2025 11:14:19 -0700 Subject: [PATCH 07/54] Enforce VALID_ARCHS for simulator platforms This allows projects to build more smoothly across platforms when architectures like arm64e are unconditionally appended to ARCHS. Simulators have never supported arm64e in any capacity. rdar://123839235 --- Sources/SWBApplePlatform/Plugin.swift | 2 +- Sources/SWBApplePlatform/Specs/iOSSimulator.xcspec | 8 -------- Sources/SWBApplePlatform/Specs/tvOSSimulator.xcspec | 8 -------- .../SWBApplePlatform/Specs/watchOSSimulator.xcspec | 8 -------- Sources/SWBApplePlatform/Specs/xrOSSimulator.xcspec | 8 -------- .../Extensions/SettingsBuilderExtension.swift | 2 +- Sources/SWBCore/Settings/Settings.swift | 12 ++++-------- 7 files changed, 6 insertions(+), 42 deletions(-) diff --git a/Sources/SWBApplePlatform/Plugin.swift b/Sources/SWBApplePlatform/Plugin.swift index 2288a849..5bdd8d3b 100644 --- a/Sources/SWBApplePlatform/Plugin.swift +++ b/Sources/SWBApplePlatform/Plugin.swift @@ -239,7 +239,7 @@ struct AppleSettingsBuilderExtension: SettingsBuilderExtension { func addPlatformSDKSettings(_ platform: SWBCore.Platform?, _ sdk: SDK, _ sdkVariant: SDKVariant?) -> [String : String] { [:] } func xcconfigOverrideData(fromParameters: BuildParameters) -> ByteString { ByteString() } func getTargetTestingSwiftPluginFlags(_ scope: MacroEvaluationScope, toolchainRegistry: ToolchainRegistry, sdkRegistry: SDKRegistry, activeRunDestination: RunDestinationInfo?, project: SWBCore.Project?) -> [String] { [] } - func shouldSkipPopulatingValidArchs(platform: SWBCore.Platform) -> Bool { false } + func shouldSkipPopulatingValidArchs(platform: SWBCore.Platform, sdk: SDK?) -> Bool { false } func shouldDisableXOJITPreviews(platformName: String, sdk: SDK?) -> Bool { false } func overridingBuildSettings(_: MacroEvaluationScope, platform: SWBCore.Platform?, productType: ProductTypeSpec) -> [String : String] { [:] } } diff --git a/Sources/SWBApplePlatform/Specs/iOSSimulator.xcspec b/Sources/SWBApplePlatform/Specs/iOSSimulator.xcspec index e0dba20f..0da7e552 100644 --- a/Sources/SWBApplePlatform/Specs/iOSSimulator.xcspec +++ b/Sources/SWBApplePlatform/Specs/iOSSimulator.xcspec @@ -40,14 +40,6 @@ SortNumber = 107; }, - { - _Domain = iphonesimulator; - Type = Architecture; - Identifier = arm64e; - PerArchBuildSettingName = "arm64e"; - SortNumber = 108; - }, - // DEPRECATED { diff --git a/Sources/SWBApplePlatform/Specs/tvOSSimulator.xcspec b/Sources/SWBApplePlatform/Specs/tvOSSimulator.xcspec index 157d5c8d..d881885a 100644 --- a/Sources/SWBApplePlatform/Specs/tvOSSimulator.xcspec +++ b/Sources/SWBApplePlatform/Specs/tvOSSimulator.xcspec @@ -49,14 +49,6 @@ SortNumber = 107; }, - { - _Domain = appletvsimulator; - Type = Architecture; - Identifier = arm64e; - PerArchBuildSettingName = "arm64e"; - SortNumber = 108; - }, - // DEPRECATED { diff --git a/Sources/SWBApplePlatform/Specs/watchOSSimulator.xcspec b/Sources/SWBApplePlatform/Specs/watchOSSimulator.xcspec index 4deabd40..06993c54 100644 --- a/Sources/SWBApplePlatform/Specs/watchOSSimulator.xcspec +++ b/Sources/SWBApplePlatform/Specs/watchOSSimulator.xcspec @@ -48,14 +48,6 @@ SortNumber = 107; }, - { - _Domain = watchsimulator; - Type = Architecture; - Identifier = arm64e; - PerArchBuildSettingName = "arm64e"; - SortNumber = 108; - }, - // DEPRECATED { diff --git a/Sources/SWBApplePlatform/Specs/xrOSSimulator.xcspec b/Sources/SWBApplePlatform/Specs/xrOSSimulator.xcspec index 546a901a..49bb9043 100644 --- a/Sources/SWBApplePlatform/Specs/xrOSSimulator.xcspec +++ b/Sources/SWBApplePlatform/Specs/xrOSSimulator.xcspec @@ -40,14 +40,6 @@ SortNumber = 107; }, - { - _Domain = xrsimulator; - Type = Architecture; - Identifier = arm64e; - PerArchBuildSettingName = "arm64e"; - SortNumber = 108; - }, - // DEPRECATED { diff --git a/Sources/SWBCore/Extensions/SettingsBuilderExtension.swift b/Sources/SWBCore/Extensions/SettingsBuilderExtension.swift index 3cb6dcde..0f1ee377 100644 --- a/Sources/SWBCore/Extensions/SettingsBuilderExtension.swift +++ b/Sources/SWBCore/Extensions/SettingsBuilderExtension.swift @@ -48,7 +48,7 @@ public protocol SettingsBuilderExtension { // Provides a list of flags to configure testing plugins func getTargetTestingSwiftPluginFlags(_ scope: MacroEvaluationScope, toolchainRegistry: ToolchainRegistry, sdkRegistry: SDKRegistry, activeRunDestination: RunDestinationInfo?, project: Project?) -> [String] // Override valid architectures enforcement for a platform - func shouldSkipPopulatingValidArchs(platform: Platform) -> Bool + func shouldSkipPopulatingValidArchs(platform: Platform, sdk: SDK?) -> Bool func shouldDisableXOJITPreviews(platformName: String, sdk: SDK?) -> Bool diff --git a/Sources/SWBCore/Settings/Settings.swift b/Sources/SWBCore/Settings/Settings.swift index bc9b7e62..f98192b0 100644 --- a/Sources/SWBCore/Settings/Settings.swift +++ b/Sources/SWBCore/Settings/Settings.swift @@ -2626,23 +2626,19 @@ private class SettingsBuilder { core.pluginManager.extensions(of: SettingsBuilderExtensionPoint.self) } - func shouldPopulateValidArchs(platform: Platform) -> Bool { + func shouldPopulateValidArchs(platform: Platform, sdk: SDK?) -> Bool { // For now, we only do this for some platforms to avoid behavior changes. // Later, we should extend this to more SDKs via switch platform.name { case "macosx", "iphoneos", - "iphonesimulator", "appletvos", - "appletvsimulator", "watchos", - "watchsimulator", - "xros", - "xrsimulator": + "xros": return false default: for settingsExtension in settingsExtensions() { - if settingsExtension.shouldSkipPopulatingValidArchs(platform: platform) { + if settingsExtension.shouldSkipPopulatingValidArchs(platform: platform, sdk: sdk) { return false } } @@ -2651,7 +2647,7 @@ private class SettingsBuilder { } // VALID_ARCHS should be based on the SDK's SupportedTargets dictionary. - if let archs = sdkVariant?.archs, !archs.isEmpty, let platform, shouldPopulateValidArchs(platform: platform) { + if let archs = sdkVariant?.archs, !archs.isEmpty, let platform, shouldPopulateValidArchs(platform: platform, sdk: sdk) { table.push(BuiltinMacros.VALID_ARCHS, literal: archs) } From e723b3586fe5bfc523a815fa812da06d1e4ab2df Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Fri, 30 May 2025 13:55:12 -0700 Subject: [PATCH 08/54] Improve task backtraces for dynamic tasks - Add a canonical implementation of reconstructing a build backtrace to the SwiftBuild module for use by clients - Adopt it in the unit test infrastructure - Improve backtrace reconstruction for dynamic tasks rdar://152194560 --- Package.swift | 2 +- .../SWBProtocol/BuildOperationMessages.swift | 8 +- .../SWBTestSupport/BuildOperationTester.swift | 105 +---------- .../SWBBuildOperationBacktraceFrame.swift | 70 ++++++- .../TaskBacktraces.swift | 174 ++++++++++++++++++ .../SwiftDriverPerfTests.swift | 1 + .../BuildBacktraceTests.swift | 3 +- .../BuildTaskBehaviorTests.swift | 1 + .../ClangExplicitModulesTests.swift | 1 + .../CodeGenerationToolTests.swift | 1 + .../DependencyCycleDiagnosticsTests.swift | 1 + ...veredDependenciesBuildOperationTests.swift | 1 + .../DsymGenerationBuildOperationTests.swift | 1 + Tests/SWBBuildSystemTests/RebuildTests.swift | 1 + .../StaleFileRemovalTests.swift | 1 + .../SwiftDriverTests.swift | 1 + Tests/SWBBuildSystemTests/UnifdefTests.swift | 1 + 17 files changed, 263 insertions(+), 110 deletions(-) create mode 100644 Sources/SwiftBuildTestSupport/TaskBacktraces.swift diff --git a/Package.swift b/Package.swift index ea2accbc..2cbc3eff 100644 --- a/Package.swift +++ b/Package.swift @@ -369,7 +369,7 @@ let package = Package( // Perf tests .testTarget( name: "SWBBuildSystemPerfTests", - dependencies: ["SWBBuildSystem", "SWBTestSupport"], + dependencies: ["SWBBuildSystem", "SWBTestSupport", "SwiftBuildTestSupport"], swiftSettings: swiftSettings(languageMode: .v6)), .testTarget( name: "SWBCASPerfTests", diff --git a/Sources/SWBProtocol/BuildOperationMessages.swift b/Sources/SWBProtocol/BuildOperationMessages.swift index 51b72ca9..71d51692 100644 --- a/Sources/SWBProtocol/BuildOperationMessages.swift +++ b/Sources/SWBProtocol/BuildOperationMessages.swift @@ -126,7 +126,7 @@ public struct BuildOperationTargetInfo: SerializableCodable, Equatable, Sendable } } -public enum BuildOperationTaskSignature: RawRepresentable, Sendable, Hashable, Codable, CustomDebugStringConvertible { +public enum BuildOperationTaskSignature: RawRepresentable, Sendable, Comparable, Hashable, Codable, CustomDebugStringConvertible { case taskIdentifier(ByteString) case activitySignature(ByteString) case subtaskSignature(ByteString) @@ -155,6 +155,10 @@ public enum BuildOperationTaskSignature: RawRepresentable, Sendable, Hashable, C } } + public static func < (lhs: BuildOperationTaskSignature, rhs: BuildOperationTaskSignature) -> Bool { + lhs.rawValue.lexicographicallyPrecedes(rhs.rawValue) + } + public init(from decoder: any Decoder) throws { let container = try decoder.singleValueContainer() guard let value = BuildOperationTaskSignature(rawValue: ByteString(try container.decode([UInt8].self))) else { @@ -1020,7 +1024,7 @@ public struct BuildOperationDiagnosticEmitted: Message, Equatable, SerializableC } } -public struct BuildOperationBacktraceFrameEmitted: Message, Equatable, SerializableCodable { +public struct BuildOperationBacktraceFrameEmitted: Message, Equatable, Hashable, SerializableCodable { public static let name = "BUILD_BACKTRACE_FRAME_EMITTED" public enum Identifier: Hashable, Equatable, Comparable, SerializableCodable, Sendable { diff --git a/Sources/SWBTestSupport/BuildOperationTester.swift b/Sources/SWBTestSupport/BuildOperationTester.swift index df0f3878..fc090718 100644 --- a/Sources/SWBTestSupport/BuildOperationTester.swift +++ b/Sources/SWBTestSupport/BuildOperationTester.swift @@ -155,7 +155,7 @@ package final class BuildOperationTester { case subtaskDidReportProgress(SubtaskProgressEvent, count: Int) /// The build emitted a backtrace frame. - case emittedBuildBacktraceFrame(identifier: SWBProtocol.BuildOperationBacktraceFrameEmitted.Identifier, previousFrameIdentifier: SWBProtocol.BuildOperationBacktraceFrameEmitted.Identifier?, category: SWBProtocol.BuildOperationBacktraceFrameEmitted.Category, description: String) + case emittedBuildBacktraceFrame(BuildOperationBacktraceFrameEmitted) package var description: String { switch self { @@ -189,8 +189,8 @@ package final class BuildOperationTester { return "activityEmittedData(\(ruleInfo), bytes: \(ByteString(bytes).asString)" case .activityEnded(ruleInfo: let ruleInfo): return "activityEnded(\(ruleInfo))" - case .emittedBuildBacktraceFrame(identifier: let id, previousFrameIdentifier: let previousID, category: let category, description: let description): - return "emittedBuildBacktraceFrame(\(id), previous: \(String(describing: previousID)), category: \(category), description: \(description))" + case .emittedBuildBacktraceFrame(let frame): + return "emittedBuildBacktraceFrame(\(frame.identifier), previous: \(String(describing: frame.previousFrameIdentifier)), category: \(frame.category), description: \(frame.description))" case .previouslyBatchedSubtaskUpToDate(let signature): return "previouslyBatchedSubtaskUpToDate(\(signature))" } @@ -735,18 +735,6 @@ package final class BuildOperationTester { } - package func checkNoTaskWithBacktraces(_ conditions: TaskCondition..., sourceLocation: SourceLocation = #_sourceLocation) { - for matchedTask in findMatchingTasks(conditions) { - Issue.record("found unexpected task matching conditions '\(conditions)', found: \(matchedTask)", sourceLocation: sourceLocation) - - if let frameID = getBacktraceID(matchedTask, sourceLocation: sourceLocation) { - enumerateBacktraces(frameID) { _, category, description in - Issue.record("...", sourceLocation: sourceLocation) - } - } - } - } - /// Check whether the results contains a dependency cycle error. If so, then consume the error and create a `CycleChecking` object and pass it to the block. Otherwise fail. package func checkDependencyCycle(_ pattern: StringPattern, kind: DiagnosticKind = .error, failIfNotFound: Bool = true, sourceLocation: SourceLocation = #_sourceLocation, body: (CycleChecker) async throws -> Void) async throws { guard let message = getDiagnosticMessage(pattern, kind: kind, checkDiagnostic: { _ in true }) else { @@ -1045,55 +1033,6 @@ package final class BuildOperationTester { startedTasks.remove(task) } - private func getBacktraceID(_ task: Task, sourceLocation: SourceLocation = #_sourceLocation) -> BuildOperationBacktraceFrameEmitted.Identifier? { - guard let frameID: BuildOperationBacktraceFrameEmitted.Identifier = events.compactMap ({ (event) -> BuildOperationBacktraceFrameEmitted.Identifier? in - guard case .emittedBuildBacktraceFrame(identifier: let identifier, previousFrameIdentifier: _, category: _, description: _) = event, case .task(let signature) = identifier, BuildOperationTaskSignature.taskIdentifier(ByteString(encodingAsUTF8: task.identifier.rawValue)) == signature else { - return nil - } - return identifier - // Iff the task is a dynamic task, there may be more than one corresponding frame if it was requested multiple times, in which case we choose the first. Non-dynamic tasks always have a 1-1 relationship with frames. - }).sorted().first else { - Issue.record("Did not find a single build backtrace frame for task: \(task.identifier)", sourceLocation: sourceLocation) - return nil - } - return frameID - } - - private func enumerateBacktraces(_ identifier: BuildOperationBacktraceFrameEmitted.Identifier, _ handleFrameInfo: (_ identifier: BuildOperationBacktraceFrameEmitted.Identifier?, _ category: BuildOperationBacktraceFrameEmitted.Category, _ description: String) -> ()) { - var currentFrameID: BuildOperationBacktraceFrameEmitted.Identifier? = identifier - while let id = currentFrameID { - if let frameInfo: (BuildOperationBacktraceFrameEmitted.Identifier?, BuildOperationBacktraceFrameEmitted.Category, String) = events.compactMap({ (event) -> (BuildOperationBacktraceFrameEmitted.Identifier?, BuildOperationBacktraceFrameEmitted.Category, String)? in - guard case .emittedBuildBacktraceFrame(identifier: id, previousFrameIdentifier: let previousFrameIdentifier, category: let category, description: let description) = event else { - return nil - } - return (previousFrameIdentifier, category, description) - // Iff the task is a dynamic task, there may be more than one corresponding frame if it was requested multiple times, in which case we choose the first. Non-dynamic tasks always have a 1-1 relationship with frames. - }).sorted(by: { $0.0 }).first { - handleFrameInfo(frameInfo.0, frameInfo.1, frameInfo.2) - currentFrameID = frameInfo.0 - } else { - currentFrameID = nil - } - } - } - - package func checkBacktrace(_ identifier: BuildOperationBacktraceFrameEmitted.Identifier, _ patterns: [StringPattern], sourceLocation: SourceLocation = #_sourceLocation) { - var frameDescriptions: [String] = [] - enumerateBacktraces(identifier) { (_, category, description) in - frameDescriptions.append("") - } - - XCTAssertMatch(frameDescriptions, patterns, sourceLocation: sourceLocation) - } - - package func checkBacktrace(_ task: Task, _ patterns: [StringPattern], sourceLocation: SourceLocation = #_sourceLocation) { - if let frameID = getBacktraceID(task, sourceLocation: sourceLocation) { - checkBacktrace(frameID, patterns, sourceLocation: sourceLocation) - } else { - // already recorded an issue - } - } - private class TaskDependencyResolver { /// The database schema has to match what `BuildSystemImpl` defines in `getMergedSchemaVersion()`. /// Can be removed once rdar://85336712 is resolved. @@ -1563,42 +1502,6 @@ package final class BuildOperationTester { } } - /// Ensure that the build is a null build. - package func checkNullBuild(_ name: String? = nil, parameters: BuildParameters? = nil, runDestination: RunDestinationInfo?, buildRequest inputBuildRequest: BuildRequest? = nil, buildCommand: BuildCommand? = nil, schemeCommand: SchemeCommand? = .launch, persistent: Bool = false, serial: Bool = false, buildOutputMap: [String:String]? = nil, signableTargets: Set = [], signableTargetInputs: [String: ProvisioningTaskInputs] = [:], clientDelegate: (any ClientDelegate)? = nil, excludedTasks: Set = ["ClangStatCache", "LinkAssetCatalogSignature"], diagnosticsToValidate: Set = [.note, .error, .warning], sourceLocation: SourceLocation = #_sourceLocation) async throws { - - func body(results: BuildResults) throws -> Void { - results.consumeTasksMatchingRuleTypes(excludedTasks) - results.checkNoTaskWithBacktraces(sourceLocation: sourceLocation) - - results.checkNote(.equal("Building targets in dependency order"), failIfNotFound: false) - results.checkNote(.prefix("Target dependency graph"), failIfNotFound: false) - - for kind in diagnosticsToValidate { - switch kind { - case .note: - results.checkNoNotes(sourceLocation: sourceLocation) - - case .warning: - results.checkNoWarnings(sourceLocation: sourceLocation) - - case .error: - results.checkNoErrors(sourceLocation: sourceLocation) - - case .remark: - results.checkNoRemarks(sourceLocation: sourceLocation) - - default: - // other kinds are ignored - break - } - } - } - - try await UserDefaults.withEnvironment(["EnableBuildBacktraceRecording": "true"]) { - try await checkBuild(name, parameters: parameters, runDestination: runDestination, buildRequest: inputBuildRequest, buildCommand: buildCommand, schemeCommand: schemeCommand, persistent: persistent, serial: serial, buildOutputMap: buildOutputMap, signableTargets: signableTargets, signableTargetInputs: signableTargetInputs, clientDelegate: clientDelegate, sourceLocation: sourceLocation, body: body) - } - } - package static func buildRequestForIndexOperation( workspace: Workspace, buildTargets: [any TestTarget]? = nil, @@ -2252,7 +2155,7 @@ private final class BuildOperationTesterDelegate: BuildOperationDelegate { func recordBuildBacktraceFrame(identifier: SWBProtocol.BuildOperationBacktraceFrameEmitted.Identifier, previousFrameIdentifier: SWBProtocol.BuildOperationBacktraceFrameEmitted.Identifier?, category: SWBProtocol.BuildOperationBacktraceFrameEmitted.Category, kind: SWBProtocol.BuildOperationBacktraceFrameEmitted.Kind, description: String) { queue.async { - self.events.append(.emittedBuildBacktraceFrame(identifier: identifier, previousFrameIdentifier: previousFrameIdentifier, category: category, description: description)) + self.events.append(.emittedBuildBacktraceFrame(.init(identifier: identifier, previousFrameIdentifier: previousFrameIdentifier, category: category, kind: kind, description: description))) } } } diff --git a/Sources/SwiftBuild/SWBBuildOperationBacktraceFrame.swift b/Sources/SwiftBuild/SWBBuildOperationBacktraceFrame.swift index 279e8c3d..7bb1fce4 100644 --- a/Sources/SwiftBuild/SWBBuildOperationBacktraceFrame.swift +++ b/Sources/SwiftBuild/SWBBuildOperationBacktraceFrame.swift @@ -15,9 +15,9 @@ import SWBUtil public import Foundation -public struct SWBBuildOperationBacktraceFrame: Hashable, Sendable, Codable, Identifiable { - public struct Identifier: Equatable, Hashable, Sendable, Codable, CustomDebugStringConvertible { - private enum Storage: Equatable, Hashable, Sendable, Codable { +public struct SWBBuildOperationBacktraceFrame: Hashable, Sendable, Codable, Identifiable, Comparable { + public struct Identifier: Equatable, Comparable, Hashable, Sendable, Codable, CustomDebugStringConvertible { + private enum Storage: Equatable, Comparable, Hashable, Sendable, Codable { case task(BuildOperationTaskSignature) case key(String) } @@ -39,6 +39,10 @@ public struct SWBBuildOperationBacktraceFrame: Hashable, Sendable, Codable, Iden self.storage = .task(taskSignature) } + package init(genericBuildKey: String) { + self.storage = .key(genericBuildKey) + } + public var debugDescription: String { switch storage { case .task(let taskSignature): @@ -47,9 +51,13 @@ public struct SWBBuildOperationBacktraceFrame: Hashable, Sendable, Codable, Iden return key } } + + public static func < (lhs: SWBBuildOperationBacktraceFrame.Identifier, rhs: SWBBuildOperationBacktraceFrame.Identifier) -> Bool { + lhs.storage < rhs.storage + } } - public enum Category: Equatable, Hashable, Sendable, Codable { + public enum Category: Equatable, Comparable, Hashable, Sendable, Codable { case ruleNeverBuilt case ruleSignatureChanged case ruleHadInvalidValue @@ -68,7 +76,7 @@ public struct SWBBuildOperationBacktraceFrame: Hashable, Sendable, Codable, Iden } } } - public enum Kind: Equatable, Hashable, Sendable, Codable { + public enum Kind: Equatable, Comparable, Hashable, Sendable, Codable { case genericTask case swiftDriverJob case file @@ -82,6 +90,14 @@ public struct SWBBuildOperationBacktraceFrame: Hashable, Sendable, Codable, Iden public let description: String public let frameKind: Kind + package init(identifier: Identifier, previousFrameIdentifier: Identifier?, category: Category, description: String, frameKind: Kind) { + self.identifier = identifier + self.previousFrameIdentifier = previousFrameIdentifier + self.category = category + self.description = description + self.frameKind = frameKind + } + // The old name collides with the `kind` key used in the SwiftBuildMessage JSON encoding @available(*, deprecated, renamed: "frameKind") public var kind: Kind { @@ -91,6 +107,10 @@ public struct SWBBuildOperationBacktraceFrame: Hashable, Sendable, Codable, Iden public var id: Identifier { identifier } + + public static func < (lhs: SWBBuildOperationBacktraceFrame, rhs: SWBBuildOperationBacktraceFrame) -> Bool { + (lhs.identifier, lhs.previousFrameIdentifier, lhs.category, lhs.description, lhs.frameKind) < (rhs.identifier, rhs.previousFrameIdentifier, rhs.category, rhs.description, rhs.frameKind) + } } extension SWBBuildOperationBacktraceFrame { @@ -134,3 +154,43 @@ extension SWBBuildOperationBacktraceFrame { self.init(identifier: id, previousFrameIdentifier: previousID, category: category, description: message.description, frameKind: kind) } } + +public struct SWBBuildOperationCollectedBacktraceFrames { + fileprivate var frames: [SWBBuildOperationBacktraceFrame.Identifier: Set] + + public init() { + self.frames = [:] + } + + public mutating func add(frame: SWBBuildOperationBacktraceFrame) { + frames[frame.identifier, default: []].insert(frame) + } +} + +public struct SWBTaskBacktrace { + public let frames: [SWBBuildOperationBacktraceFrame] + + public init?(from baseFrameID: SWBBuildOperationBacktraceFrame.Identifier, collectedFrames: SWBBuildOperationCollectedBacktraceFrames) { + var frames: [SWBBuildOperationBacktraceFrame] = [] + var currentFrame = collectedFrames.frames[baseFrameID]?.only + while let frame = currentFrame { + frames.append(frame) + if let previousFrameID = frame.previousFrameIdentifier, let candidatesForNextFrame = collectedFrames.frames[previousFrameID] { + switch frame.category { + case .dynamicTaskRegistration: + currentFrame = candidatesForNextFrame.sorted().first { + $0.category == .dynamicTaskRequest + } + default: + currentFrame = candidatesForNextFrame.sorted().first + } + } else { + currentFrame = nil + } + } + guard !frames.isEmpty else { + return nil + } + self.frames = frames + } +} diff --git a/Sources/SwiftBuildTestSupport/TaskBacktraces.swift b/Sources/SwiftBuildTestSupport/TaskBacktraces.swift new file mode 100644 index 00000000..423937e6 --- /dev/null +++ b/Sources/SwiftBuildTestSupport/TaskBacktraces.swift @@ -0,0 +1,174 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +package import SWBTestSupport +import SwiftBuild +package import SWBProtocol +import SWBTaskConstruction +@_spi(Testing) import SWBUtil +package import SWBCore +import SWBTaskExecution +package import SWBBuildSystem +package import Testing +import Foundation + +extension BuildOperationTester.BuildResults { + private func getBacktraceID(_ task: Task, sourceLocation: SourceLocation = #_sourceLocation) -> BuildOperationBacktraceFrameEmitted.Identifier? { + guard let frameID: BuildOperationBacktraceFrameEmitted.Identifier = events.compactMap ({ (event) -> BuildOperationBacktraceFrameEmitted.Identifier? in + guard case .emittedBuildBacktraceFrame(let frame) = event, case .task(let signature) = frame.identifier, BuildOperationTaskSignature.taskIdentifier(ByteString(encodingAsUTF8: task.identifier.rawValue)) == signature else { + return nil + } + return frame.identifier + // Iff the task is a dynamic task, there may be more than one corresponding frame if it was requested multiple times, in which case we choose the first. Non-dynamic tasks always have a 1-1 relationship with frames. + }).sorted().first else { + Issue.record("Did not find a single build backtrace frame for task: \(task.identifier)", sourceLocation: sourceLocation) + return nil + } + return frameID + } + + private func reconstructBacktrace(for identifier: BuildOperationBacktraceFrameEmitted.Identifier) -> SWBTaskBacktrace? { + var collectedFrames = SWBBuildOperationCollectedBacktraceFrames() + for event in self.events { + if case .emittedBuildBacktraceFrame(let frame) = event { + let wrappedFrame = SWBBuildOperationBacktraceFrame(frame) + collectedFrames.add(frame: wrappedFrame) + } + } + let backtrace = SWBTaskBacktrace(from: SWBBuildOperationBacktraceFrame.Identifier(messageIdentifier: identifier), collectedFrames: collectedFrames) + return backtrace + } + + package func checkBacktrace(_ identifier: BuildOperationBacktraceFrameEmitted.Identifier, _ patterns: [StringPattern], sourceLocation: SourceLocation = #_sourceLocation) { + var frameDescriptions: [String] = [] + guard let backtrace = reconstructBacktrace(for: identifier) else { + Issue.record("unable to reconstruct backtrace for \(identifier)") + return + } + for frame in backtrace.frames { + frameDescriptions.append("") + } + + XCTAssertMatch(frameDescriptions, patterns, sourceLocation: sourceLocation) + } + + package func checkBacktrace(_ task: Task, _ patterns: [StringPattern], sourceLocation: SourceLocation = #_sourceLocation) { + if let frameID = getBacktraceID(task, sourceLocation: sourceLocation) { + checkBacktrace(frameID, patterns, sourceLocation: sourceLocation) + } else { + // already recorded an issue + } + } + + package func checkNoTaskWithBacktraces(_ conditions: TaskCondition..., sourceLocation: SourceLocation = #_sourceLocation) { + for matchedTask in findMatchingTasks(conditions) { + Issue.record("found unexpected task matching conditions '\(conditions)', found: \(matchedTask)", sourceLocation: sourceLocation) + + if let frameID = getBacktraceID(matchedTask, sourceLocation: sourceLocation), let backtrace = reconstructBacktrace(for: frameID) { + for frame in backtrace.frames { + Issue.record("...", sourceLocation: sourceLocation) + } + } + } + } +} + +extension BuildOperationTester { + /// Ensure that the build is a null build. + package func checkNullBuild(_ name: String? = nil, parameters: BuildParameters? = nil, runDestination: RunDestinationInfo?, buildRequest inputBuildRequest: BuildRequest? = nil, buildCommand: BuildCommand? = nil, schemeCommand: SchemeCommand? = .launch, persistent: Bool = false, serial: Bool = false, buildOutputMap: [String:String]? = nil, signableTargets: Set = [], signableTargetInputs: [String: ProvisioningTaskInputs] = [:], clientDelegate: (any ClientDelegate)? = nil, excludedTasks: Set = ["ClangStatCache", "LinkAssetCatalogSignature"], diagnosticsToValidate: Set = [.note, .error, .warning], sourceLocation: SourceLocation = #_sourceLocation) async throws { + + func body(results: BuildResults) throws -> Void { + results.consumeTasksMatchingRuleTypes(excludedTasks) + results.checkNoTaskWithBacktraces(sourceLocation: sourceLocation) + + results.checkNote(.equal("Building targets in dependency order"), failIfNotFound: false) + results.checkNote(.prefix("Target dependency graph"), failIfNotFound: false) + + for kind in diagnosticsToValidate { + switch kind { + case .note: + results.checkNoNotes(sourceLocation: sourceLocation) + + case .warning: + results.checkNoWarnings(sourceLocation: sourceLocation) + + case .error: + results.checkNoErrors(sourceLocation: sourceLocation) + + case .remark: + results.checkNoRemarks(sourceLocation: sourceLocation) + + default: + // other kinds are ignored + break + } + } + } + + try await UserDefaults.withEnvironment(["EnableBuildBacktraceRecording": "true"]) { + try await checkBuild(name, parameters: parameters, runDestination: runDestination, buildRequest: inputBuildRequest, buildCommand: buildCommand, schemeCommand: schemeCommand, persistent: persistent, serial: serial, buildOutputMap: buildOutputMap, signableTargets: signableTargets, signableTargetInputs: signableTargetInputs, clientDelegate: clientDelegate, sourceLocation: sourceLocation, body: body) + } + } +} + +extension SWBBuildOperationBacktraceFrame { + init(_ message: BuildOperationBacktraceFrameEmitted) { + let id = SWBBuildOperationBacktraceFrame.Identifier(messageIdentifier: message.identifier) + let previousID = message.previousFrameIdentifier.map { SWBBuildOperationBacktraceFrame.Identifier(messageIdentifier: $0) } + let category: SWBBuildOperationBacktraceFrame.Category + switch message.category { + case .ruleNeverBuilt: + category = .ruleNeverBuilt + case .ruleSignatureChanged: + category = .ruleSignatureChanged + case .ruleHadInvalidValue: + category = .ruleHadInvalidValue + case .ruleInputRebuilt: + category = .ruleInputRebuilt + case .ruleForced: + category = .ruleForced + case .dynamicTaskRegistration: + category = .dynamicTaskRegistration + case .dynamicTaskRequest: + category = .dynamicTaskRequest + case .none: + category = .none + } + let kind: SWBBuildOperationBacktraceFrame.Kind + switch message.kind { + case .genericTask: + kind = .genericTask + case .swiftDriverJob: + kind = .swiftDriverJob + case .directory: + kind = .directory + case .file: + kind = .file + case .unknown: + kind = .unknown + case nil: + kind = .unknown + } + self.init(identifier: id, previousFrameIdentifier: previousID, category: category, description: message.description, frameKind: kind) + } +} + +extension SWBBuildOperationBacktraceFrame.Identifier { + init(messageIdentifier: BuildOperationBacktraceFrameEmitted.Identifier) { + switch messageIdentifier { + case .task(let signature): + self.init(taskSignatureData: Data(signature.rawValue.bytes))! + case .genericBuildKey(let id): + self.init(genericBuildKey: id) + } + } +} diff --git a/Tests/SWBBuildSystemPerfTests/SwiftDriverPerfTests.swift b/Tests/SWBBuildSystemPerfTests/SwiftDriverPerfTests.swift index 2b16645b..3c019423 100644 --- a/Tests/SWBBuildSystemPerfTests/SwiftDriverPerfTests.swift +++ b/Tests/SWBBuildSystemPerfTests/SwiftDriverPerfTests.swift @@ -17,6 +17,7 @@ import SWBCore import SWBProtocol import SWBUtil import SWBTestSupport +import SwiftBuildTestSupport @Suite(.performance) fileprivate struct SwiftDriverPerfTests: CoreBasedTests, PerfTests { diff --git a/Tests/SWBBuildSystemTests/BuildBacktraceTests.swift b/Tests/SWBBuildSystemTests/BuildBacktraceTests.swift index 12a42d68..a6cbc12e 100644 --- a/Tests/SWBBuildSystemTests/BuildBacktraceTests.swift +++ b/Tests/SWBBuildSystemTests/BuildBacktraceTests.swift @@ -16,6 +16,7 @@ import SWBCore import SWBProtocol @_spi(Testing) import SWBUtil import SWBTestSupport +import SwiftBuildTestSupport @Suite(.userDefaults(["EnableBuildBacktraceRecording": "true"])) fileprivate struct BuildBacktraceTests: CoreBasedTests { @@ -165,7 +166,7 @@ fileprivate struct BuildBacktraceTests: CoreBasedTests { } } - @Test(.requireSDKs(.macOS), .flaky("Single-use task backtraces need rework")) + @Test(.requireSDKs(.macOS)) func singleUseTaskBacktraceRecording() async throws { try await withTemporaryDirectory { tmpDirPath async throws -> Void in let testWorkspace = try await TestWorkspace( diff --git a/Tests/SWBBuildSystemTests/BuildTaskBehaviorTests.swift b/Tests/SWBBuildSystemTests/BuildTaskBehaviorTests.swift index c7ae170f..d8ce8799 100644 --- a/Tests/SWBBuildSystemTests/BuildTaskBehaviorTests.swift +++ b/Tests/SWBBuildSystemTests/BuildTaskBehaviorTests.swift @@ -15,6 +15,7 @@ import Testing import SWBBuildSystem import SWBCore import SWBTestSupport +import SwiftBuildTestSupport import SWBTaskExecution @_spi(Testing) import SWBUtil import SWBLibc diff --git a/Tests/SWBBuildSystemTests/ClangExplicitModulesTests.swift b/Tests/SWBBuildSystemTests/ClangExplicitModulesTests.swift index 70d31af9..d2651f42 100644 --- a/Tests/SWBBuildSystemTests/ClangExplicitModulesTests.swift +++ b/Tests/SWBBuildSystemTests/ClangExplicitModulesTests.swift @@ -17,6 +17,7 @@ import Testing import SWBCore import SWBTaskExecution import SWBTestSupport +import SwiftBuildTestSupport import SWBUtil import SWBProtocol diff --git a/Tests/SWBBuildSystemTests/CodeGenerationToolTests.swift b/Tests/SWBBuildSystemTests/CodeGenerationToolTests.swift index cb46d95a..488338c8 100644 --- a/Tests/SWBBuildSystemTests/CodeGenerationToolTests.swift +++ b/Tests/SWBBuildSystemTests/CodeGenerationToolTests.swift @@ -15,6 +15,7 @@ import class Foundation.Bundle import SWBCore import SWBTaskExecution import SWBTestSupport +import SwiftBuildTestSupport import SWBUtil import Testing import SWBProtocol diff --git a/Tests/SWBBuildSystemTests/DependencyCycleDiagnosticsTests.swift b/Tests/SWBBuildSystemTests/DependencyCycleDiagnosticsTests.swift index 08c6aa38..ef1f3f25 100644 --- a/Tests/SWBBuildSystemTests/DependencyCycleDiagnosticsTests.swift +++ b/Tests/SWBBuildSystemTests/DependencyCycleDiagnosticsTests.swift @@ -14,6 +14,7 @@ import Testing import SWBCore import SWBTestSupport +import SwiftBuildTestSupport @_spi(Testing) import SWBUtil @_spi(Testing) import SWBBuildSystem diff --git a/Tests/SWBBuildSystemTests/DiscoveredDependenciesBuildOperationTests.swift b/Tests/SWBBuildSystemTests/DiscoveredDependenciesBuildOperationTests.swift index 4c01d9ef..398bf185 100644 --- a/Tests/SWBBuildSystemTests/DiscoveredDependenciesBuildOperationTests.swift +++ b/Tests/SWBBuildSystemTests/DiscoveredDependenciesBuildOperationTests.swift @@ -12,6 +12,7 @@ import SWBCore import SWBTestSupport +import SwiftBuildTestSupport import SWBUtil import Testing diff --git a/Tests/SWBBuildSystemTests/DsymGenerationBuildOperationTests.swift b/Tests/SWBBuildSystemTests/DsymGenerationBuildOperationTests.swift index db117150..5fcc6114 100644 --- a/Tests/SWBBuildSystemTests/DsymGenerationBuildOperationTests.swift +++ b/Tests/SWBBuildSystemTests/DsymGenerationBuildOperationTests.swift @@ -14,6 +14,7 @@ import Testing import SWBCore import SWBTestSupport +import SwiftBuildTestSupport import SWBUtil import SWBMacro import SWBProtocol diff --git a/Tests/SWBBuildSystemTests/RebuildTests.swift b/Tests/SWBBuildSystemTests/RebuildTests.swift index 1ed9fb98..d09445d2 100644 --- a/Tests/SWBBuildSystemTests/RebuildTests.swift +++ b/Tests/SWBBuildSystemTests/RebuildTests.swift @@ -14,6 +14,7 @@ import Testing import SWBCore import SWBTestSupport +import SwiftBuildTestSupport import SWBUtil import SWBProtocol diff --git a/Tests/SWBBuildSystemTests/StaleFileRemovalTests.swift b/Tests/SWBBuildSystemTests/StaleFileRemovalTests.swift index e927e72c..1969f1e2 100644 --- a/Tests/SWBBuildSystemTests/StaleFileRemovalTests.swift +++ b/Tests/SWBBuildSystemTests/StaleFileRemovalTests.swift @@ -20,6 +20,7 @@ import SWBUtil import SWBTaskExecution import SWBProtocol +import SwiftBuildTestSupport @Suite fileprivate struct StaleFileRemovalTests: CoreBasedTests { diff --git a/Tests/SWBBuildSystemTests/SwiftDriverTests.swift b/Tests/SWBBuildSystemTests/SwiftDriverTests.swift index 7d7df89f..6d218263 100644 --- a/Tests/SWBBuildSystemTests/SwiftDriverTests.swift +++ b/Tests/SWBBuildSystemTests/SwiftDriverTests.swift @@ -16,6 +16,7 @@ import Testing import SWBProtocol import SWBUtil import SWBTestSupport +import SwiftBuildTestSupport import SWBLLBuild import SWBCore diff --git a/Tests/SWBBuildSystemTests/UnifdefTests.swift b/Tests/SWBBuildSystemTests/UnifdefTests.swift index 0fd25158..aa163cbc 100644 --- a/Tests/SWBBuildSystemTests/UnifdefTests.swift +++ b/Tests/SWBBuildSystemTests/UnifdefTests.swift @@ -16,6 +16,7 @@ import SWBBuildSystem import SWBCore import SWBUtil import SWBTestSupport +import SwiftBuildTestSupport @Suite(.requireXcode16()) fileprivate struct UnifdefTests: CoreBasedTests { From 0506bb2fd1ba441834616f71db03a8a22d68ee85 Mon Sep 17 00:00:00 2001 From: Jake Petroules Date: Thu, 29 May 2025 12:25:48 -0700 Subject: [PATCH 09/54] Enforce a minimum Xcode version for Open Source testing workflows This provides a user-visible error message when attempting to initialize a Core with an older version, and removes the explicit version listing from the README so it doesn't need to be updated in multiple places. --- README.md | 4 ++-- Sources/SWBCore/Core.swift | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 42f81ce9..504b53d8 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,11 @@ When building SwiftPM from sources which include Swift Build integration, passin ### With Xcode -Changes to swift-build can also be tested in Xcode using the `launch-xcode` command plugin provided by the package. Run `swift package --disable-sandbox launch-xcode` from your checkout of swift-build to launch a copy of the currently `xcode-select`ed Xcode.app configured to use your modified copy of the build system service. This workflow is currently supported when using Xcode 16.2. +Changes to swift-build can also be tested in Xcode using the `launch-xcode` command plugin provided by the package. Run `swift package --disable-sandbox launch-xcode` from your checkout of swift-build to launch a copy of the currently `xcode-select`ed Xcode.app configured to use your modified copy of the build system service. This workflow is generally only supported when using the latest available Xcode version. ### With xcodebuild -Changes to swift-build can also be tested in xcodebuild using the `run-xcodebuild` command plugin provided by the package. Run `swift package --disable-sandbox run-xcodebuild` from your checkout of swift-build to run xcodebuild from the currently `xcode-select`ed Xcode.app configured to use your modified copy of the build system service. Arguments followed by `--` will be forwarded to xcodebuild unmodified. This workflow is currently supported when using Xcode 16.2. +Changes to swift-build can also be tested in xcodebuild using the `run-xcodebuild` command plugin provided by the package. Run `swift package --disable-sandbox run-xcodebuild` from your checkout of swift-build to run xcodebuild from the currently `xcode-select`ed Xcode.app configured to use your modified copy of the build system service. Arguments followed by `--` will be forwarded to xcodebuild unmodified. This workflow is generally only supported when using the latest available Xcode version. ### Debugging diff --git a/Sources/SWBCore/Core.swift b/Sources/SWBCore/Core.swift index 238a2e92..ae3fc3a4 100644 --- a/Sources/SWBCore/Core.swift +++ b/Sources/SWBCore/Core.swift @@ -245,6 +245,12 @@ public final class Core: Sendable { // If the ProductBuildVersion key is missing, we use "UNKNOWN" as the value. self.xcodeProductBuildVersion = info.productBuildVersion ?? ProductBuildVersion(major: 0, train: "A", build: 0, buildSuffix: "") self.xcodeProductBuildVersionString = info.productBuildVersion?.description ?? "UNKNOWN" + + // Enforce a minimum Xcode version for Open Source testing workflows + let minimumXcodeVersion = Version(16, 2) + if xcodeVersion < minimumXcodeVersion { + throw StubError.error("This build of Swift Build requires a minimum Xcode version of \(minimumXcodeVersion.description) (current version: \(xcodeVersion.zeroTrimmed.description)).") + } } else { // Set an arbitrary version for testing purposes. self.xcodeVersion = Version(99, 99, 99) From b81c518ed0ec98cfd016870f628c2217ea04563d Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Sun, 1 Jun 2025 18:06:45 -0700 Subject: [PATCH 10/54] License header check should ignore .swift-version --- .licenseignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.licenseignore b/.licenseignore index 21fbce89..e7babf0c 100644 --- a/.licenseignore +++ b/.licenseignore @@ -5,5 +5,6 @@ **/Package.swift .dir-locals.el .editorconfig +.swift-version CODEOWNERS Package.swift From 561e4744e607ad346c50956dd413ab7f2ce8f1c4 Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Mon, 2 Jun 2025 09:06:18 -0700 Subject: [PATCH 11/54] Revert "Update the swift-llbuild in Package.swift to release/6.2 branch (#409)" This reverts commit 2fa722888ff619bedc776d70bd060ad1996d53ae. --- Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index 8d87eeca..89565d48 100644 --- a/Package.swift +++ b/Package.swift @@ -460,6 +460,6 @@ if useLocalDependencies { .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.0.3"), ] if !useLLBuildFramework { - package.dependencies += [.package(url: "https://github.com/swiftlang/swift-llbuild.git", branch: "release/6.2"),] + package.dependencies += [.package(url: "https://github.com/swiftlang/swift-llbuild.git", branch: "main"),] } } From b51f81214322c6b5b9b04ce0950ea883923f14ed Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Mon, 2 Jun 2025 09:07:10 -0700 Subject: [PATCH 12/54] Revert "[6.2] Update the swift-driver to release/6.2 branch (#407)" This reverts commit 65eed717bb19da35f6587613130bb4f9e6801d03. --- Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index 89565d48..ea2accbc 100644 --- a/Package.swift +++ b/Package.swift @@ -455,7 +455,7 @@ if useLocalDependencies { } } else { package.dependencies += [ - .package(url: "https://github.com/swiftlang/swift-driver.git", branch: "release/6.2"), + .package(url: "https://github.com/swiftlang/swift-driver.git", branch: "main"), .package(url: "https://github.com/apple/swift-system.git", .upToNextMajor(from: "1.4.1")), .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.0.3"), ] From 475fde5ad097be6daffb73bdbd61294064f2b359 Mon Sep 17 00:00:00 2001 From: Boris Buegling Date: Wed, 28 May 2025 12:40:22 -0700 Subject: [PATCH 13/54] Introduce a new `MODULE_DEPENDENCIES` build setting This is mostly declarative right now, but does inform implicit dependencies. --- .../DependencyCycleFormatter.swift | 4 +- .../SWBCore/LinkageDependencyResolver.swift | 56 ++++++++++++- Sources/SWBCore/Settings/BuiltinMacros.swift | 2 + Sources/SWBCore/Settings/Settings.swift | 20 +++++ Sources/SWBCore/Specs/CoreBuildSystem.xcspec | 10 +++ .../Specs/en.lproj/CoreBuildSystem.strings | 3 + .../SWBCore/TargetDependencyResolver.swift | 4 +- .../TargetDependencyResolverTests.swift | 78 +++++++++++++++++++ 8 files changed, 171 insertions(+), 6 deletions(-) diff --git a/Sources/SWBBuildSystem/DependencyCycleFormatter.swift b/Sources/SWBBuildSystem/DependencyCycleFormatter.swift index b0d7f6fa..88e83e99 100644 --- a/Sources/SWBBuildSystem/DependencyCycleFormatter.swift +++ b/Sources/SWBBuildSystem/DependencyCycleFormatter.swift @@ -379,7 +379,7 @@ struct DependencyCycleFormatter { message = "Target '\(previousTargetName)' has an explicit dependency on Target '\(targetName)'" case let .implicitBuildPhaseLinkage(filename, _, buildPhase)?: message = "Target '\(previousTargetName)' has an implicit dependency on Target '\(targetName)' because '\(previousTargetName)' references the file '\(filename)' in the build phase '\(buildPhase)'" - case let .implicitBuildSettingLinkage(settingName, options)?: + case let .implicitBuildSetting(settingName, options)?: message = "Target '\(previousTargetName)' has an implicit dependency on Target '\(targetName)' because '\(previousTargetName)' defines the option '\(options.joined(separator: " "))' in the build setting '\(settingName)'" case let .impliedByTransitiveDependencyViaRemovedTargets(intermediateTargetName: intermediateTargetName): message = "Target '\(previousTargetName)' has a dependency on Target '\(targetName)' via its transitive dependency through '\(intermediateTargetName)'" @@ -501,7 +501,7 @@ struct DependencyCycleFormatter { suffix = " via the “Target Dependencies“ build phase" case let .implicitBuildPhaseLinkage(filename, _, buildPhase)?: suffix = " because the scheme has implicit dependencies enabled and the Target '\(lastTargetsName)' references the file '\(filename)' in the build phase '\(buildPhase)'" - case let .implicitBuildSettingLinkage(settingName, options)?: + case let .implicitBuildSetting(settingName, options)?: suffix = " because the scheme has implicit dependencies enabled and the Target '\(lastTargetsName)' defines the options '\(options.joined(separator: " "))' in the build setting '\(settingName)'" case let .impliedByTransitiveDependencyViaRemovedTargets(intermediateTargetName: intermediateTargetName): suffix = " via its transitive dependency through '\(intermediateTargetName)'" diff --git a/Sources/SWBCore/LinkageDependencyResolver.swift b/Sources/SWBCore/LinkageDependencyResolver.swift index 2758f545..f9c87139 100644 --- a/Sources/SWBCore/LinkageDependencyResolver.swift +++ b/Sources/SWBCore/LinkageDependencyResolver.swift @@ -12,6 +12,7 @@ public import SWBUtil import SWBMacro +internal import Foundation /// A completely resolved graph of configured targets for use in a build. public struct TargetLinkageGraph: TargetGraph { @@ -79,11 +80,15 @@ actor LinkageDependencyResolver { /// Sets of targets mapped by product name stem. private let targetsByProductNameStem: [String: Set] + /// Sets of targets mapped by module name (computed using parameters from the build request). + private let targetsByUnconfiguredModuleName: [String: Set] + internal let resolver: DependencyResolver init(workspaceContext: WorkspaceContext, buildRequest: BuildRequest, buildRequestContext: BuildRequestContext, delegate: any TargetDependencyResolverDelegate) { var targetsByProductName = [String: Set]() var targetsByProductNameStem = [String: Set]() + var targetsByUnconfiguredModuleName = [String: Set]() for case let target as StandardTarget in workspaceContext.workspace.allTargets { // FIXME: We are relying on the product reference name being constant here. This is currently true, given how our path resolver works, but it is possible to construct an Xcode project for which this doesn't work (Xcode doesn't, however, handle that situation very well). We should resolve this: Swift Build doesn't support product references with non-constant basenames @@ -95,11 +100,17 @@ actor LinkageDependencyResolver { if let stem = Path(productName).stem, stem != productName { targetsByProductNameStem[stem, default: []].insert(target) } + + let moduleName = buildRequestContext.getCachedSettings(buildRequest.parameters, target: target).globalScope.evaluate(BuiltinMacros.PRODUCT_MODULE_NAME) + if !moduleName.isEmpty { + targetsByUnconfiguredModuleName[moduleName, default: []].insert(target) + } } // Remember the mappings we created. self.targetsByProductName = targetsByProductName self.targetsByProductNameStem = targetsByProductNameStem + self.targetsByUnconfiguredModuleName = targetsByUnconfiguredModuleName resolver = DependencyResolver(workspaceContext: workspaceContext, buildRequest: buildRequest, buildRequestContext: buildRequestContext, delegate: delegate) } @@ -333,7 +344,7 @@ actor LinkageDependencyResolver { // Skip this flag if its corresponding product name is the same as the product of one of our explicit dependencies. This effectively matches the flag to an explicit dependency. if !productNamesOfExplicitDependencies.contains(productName), let implicitDependency = await implicitDependency(forProductName: productName, from: configuredTarget, imposedParameters: imposedParameters, source: .frameworkLinkerFlag(flag: flag, frameworkName: stem, buildSetting: macro)) { - await result.append(ResolvedTargetDependency(target: implicitDependency, reason: .implicitBuildSettingLinkage(settingName: macro.name, options: [flag, stem]))) + await result.append(ResolvedTargetDependency(target: implicitDependency, reason: .implicitBuildSetting(settingName: macro.name, options: [flag, stem]))) return } } addLibrary: { macro, prefix, stem in @@ -349,7 +360,7 @@ actor LinkageDependencyResolver { if productNamesOfExplicitDependencies.intersection(productNames).isEmpty { for productName in productNames { if let implicitDependency = await implicitDependency(forProductName: productName, from: configuredTarget, imposedParameters: imposedParameters, source: .libraryLinkerFlag(flag: prefix, libraryName: stem, buildSetting: macro)) { - await result.append(ResolvedTargetDependency(target: implicitDependency, reason: .implicitBuildSettingLinkage(settingName: macro.name, options: ["\(prefix)\(stem)"]))) + await result.append(ResolvedTargetDependency(target: implicitDependency, reason: .implicitBuildSetting(settingName: macro.name, options: ["\(prefix)\(stem)"]))) // We only match one. return } @@ -360,6 +371,16 @@ actor LinkageDependencyResolver { } } + let moduleNamesOfExplicitDependencies = Set(immediateDependencies.compactMap{ + buildRequestContext.getCachedSettings($0.parameters, target: $0.target).globalScope.evaluate(BuiltinMacros.PRODUCT_MODULE_NAME) + }) + + for moduleDependencyName in configuredTargetSettings.moduleDependencies.map { $0.name } { + if !moduleNamesOfExplicitDependencies.contains(moduleDependencyName), let implicitDependency = await implicitDependency(forModuleName: moduleDependencyName, from: configuredTarget, imposedParameters: imposedParameters, source: .moduleDependency(name: moduleDependencyName, buildSetting: BuiltinMacros.MODULE_DEPENDENCIES)) { + await result.append(ResolvedTargetDependency(target: implicitDependency, reason: .implicitBuildSetting(settingName: BuiltinMacros.MODULE_DEPENDENCIES.name, options: [moduleDependencyName]))) + } + } + return await result.value } @@ -444,6 +465,30 @@ actor LinkageDependencyResolver { return resolver.lookupConfiguredTarget(candidateDependencyTarget, parameters: candidateParameters, imposedParameters: effectiveImposedParameters) } + private func implicitDependency(forModuleName moduleName: String, from configuredTarget: ConfiguredTarget, imposedParameters: SpecializationParameters?, source: ImplicitDependencySource) async -> ConfiguredTarget? { + let candidateConfiguredTargets = await (targetsByUnconfiguredModuleName[moduleName] ?? []).asyncMap { [self] candidateTarget -> ConfiguredTarget? in + // Prefer overriding build parameters from the build request, if present. + let buildParameters = resolver.buildParametersByTarget[candidateTarget] ?? configuredTarget.parameters + + // Validate the module name using concrete parameters. + let configuredModuleName = buildRequestContext.getCachedSettings(buildParameters, target: candidateTarget).globalScope.evaluate(BuiltinMacros.PRODUCT_MODULE_NAME) + if configuredModuleName != moduleName { + return nil + } + + // Get a configured target for this target, and use it as the implicit dependency. + if let candidateConfiguredTarget = await implicitDependency(candidate: candidateTarget, parameters: buildParameters, isValidFor: configuredTarget, imposedParameters: imposedParameters, resolver: resolver) { + return candidateConfiguredTarget + } + + return nil + }.compactMap { $0 }.sorted() + + emitAmbiguousImplicitDependencyWarningIfNeeded(for: configuredTarget, dependencies: candidateConfiguredTargets, from: source) + + return candidateConfiguredTargets.first + } + /// Search for an implicit dependency by full product name. nonisolated private func implicitDependency(forProductName productName: String, from configuredTarget: ConfiguredTarget, imposedParameters: SpecializationParameters?, source: ImplicitDependencySource) async -> ConfiguredTarget? { let candidateConfiguredTargets = await (targetsByProductName[productName] ?? []).asyncMap { [self] candidateTarget -> ConfiguredTarget? in @@ -506,6 +551,9 @@ actor LinkageDependencyResolver { /// The dependency's product name matched the basename of a build file in the target's build phases. case productNameStem(_ stem: String, buildFile: BuildFile, buildPhase: BuildPhase) + /// The dependency's module name matched a declared module dependency of the client target. + case moduleDependency(name: String, buildSetting: MacroDeclaration) + var valueForDisplay: String { switch self { case let .frameworkLinkerFlag(flag, frameworkName, _): @@ -516,6 +564,8 @@ actor LinkageDependencyResolver { return "product reference '\(productName)'" case let .productNameStem(stem, _, _): return "product bundle executable reference '\(stem)'" + case let .moduleDependency(name, _): + return "module dependency \(name)" } } } @@ -530,6 +580,8 @@ actor LinkageDependencyResolver { case let .productReference(_, buildFile, buildPhase), let .productNameStem(_, buildFile, buildPhase): location = .buildFile(buildFileGUID: buildFile.guid, buildPhaseGUID: buildPhase.guid, targetGUID: configuredTarget.target.guid) + case let .moduleDependency(_, buildSetting): + location = .buildSettings([buildSetting]) } delegate.emit(.overrideTarget(configuredTarget), SWBUtil.Diagnostic(behavior: .warning, location: location, data: DiagnosticData("Multiple targets match implicit dependency for \(source.valueForDisplay). Consider adding an explicit dependency on the intended target to resolve this ambiguity.", component: .targetIntegrity), childDiagnostics: candidateConfiguredTargets.map({ dependency -> Diagnostic in diff --git a/Sources/SWBCore/Settings/BuiltinMacros.swift b/Sources/SWBCore/Settings/BuiltinMacros.swift index bf7cc068..bc5da2ac 100644 --- a/Sources/SWBCore/Settings/BuiltinMacros.swift +++ b/Sources/SWBCore/Settings/BuiltinMacros.swift @@ -865,6 +865,7 @@ public final class BuiltinMacros { public static let MODULEMAP_PATH = BuiltinMacros.declareStringMacro("MODULEMAP_PATH") public static let MODULEMAP_PRIVATE_FILE = BuiltinMacros.declareStringMacro("MODULEMAP_PRIVATE_FILE") public static let MODULES_FOLDER_PATH = BuiltinMacros.declarePathMacro("MODULES_FOLDER_PATH") + public static let MODULE_DEPENDENCIES = BuiltinMacros.declareStringListMacro("MODULE_DEPENDENCIES") public static let MODULE_VERIFIER_KIND = BuiltinMacros.declareEnumMacro("MODULE_VERIFIER_KIND") as EnumMacroDeclaration public static let MODULE_VERIFIER_LSV = BuiltinMacros.declareBooleanMacro("MODULE_VERIFIER_LSV") public static let MODULE_VERIFIER_SUPPORTED_LANGUAGES = BuiltinMacros.declareStringListMacro("MODULE_VERIFIER_SUPPORTED_LANGUAGES") @@ -1944,6 +1945,7 @@ public final class BuiltinMacros { MODULEMAP_PRIVATE_FILE, MODULES_FOLDER_PATH, MODULE_CACHE_DIR, + MODULE_DEPENDENCIES, MODULE_NAME, MODULE_START, MODULE_STOP, diff --git a/Sources/SWBCore/Settings/Settings.swift b/Sources/SWBCore/Settings/Settings.swift index f98192b0..e2180bb4 100644 --- a/Sources/SWBCore/Settings/Settings.swift +++ b/Sources/SWBCore/Settings/Settings.swift @@ -5331,3 +5331,23 @@ extension MacroEvaluationScope { } } } + +extension Settings { + public struct ModuleDependencyInfo { + let name: String + let isPublic: Bool + } + + public var moduleDependencies: [ModuleDependencyInfo] { + self.globalScope.evaluate(BuiltinMacros.MODULE_DEPENDENCIES).compactMap { + let components = $0.components(separatedBy: " ") + guard let name = components.last else { + return nil + } + return ModuleDependencyInfo( + name: name, + isPublic: components.count > 1 && components.first == "public" + ) + } + } +} diff --git a/Sources/SWBCore/Specs/CoreBuildSystem.xcspec b/Sources/SWBCore/Specs/CoreBuildSystem.xcspec index d89f2142..2f9992c8 100644 --- a/Sources/SWBCore/Specs/CoreBuildSystem.xcspec +++ b/Sources/SWBCore/Specs/CoreBuildSystem.xcspec @@ -1597,6 +1597,16 @@ When `GENERATE_INFOPLIST_FILE` is enabled, sets the value of the [CFBundleIdenti sdk, ); }, + { + Name = "MODULE_DEPENDENCIES"; + Type = StringList; + Category = BuildOptions; + DefaultValue = ""; + ConditionFlavors = ( + arch, + sdk, + ); + }, { Name = "GENERATE_PRELINK_OBJECT_FILE"; Type = Boolean; diff --git a/Sources/SWBCore/Specs/en.lproj/CoreBuildSystem.strings b/Sources/SWBCore/Specs/en.lproj/CoreBuildSystem.strings index a33cfc4a..7b03e48d 100644 --- a/Sources/SWBCore/Specs/en.lproj/CoreBuildSystem.strings +++ b/Sources/SWBCore/Specs/en.lproj/CoreBuildSystem.strings @@ -397,6 +397,9 @@ Generally you should not specify an order file in Debug or Development configura "[OTHER_LDFLAGS]-name" = "Other Linker Flags"; "[OTHER_LDFLAGS]-description" = "Options defined in this setting are passed to invocations of the linker."; +"[MODULE_DEPENDENCIES]-name" = "Module Dependencies"; +"[MODULE_DEPENDENCIES]-description" = "Other modules this target depends on."; + "[OTHER_LIBTOOLFLAGS]-name" = "Other Librarian Flags"; "[OTHER_LIBTOOLFLAGS]-description" = "Options defined in this setting are passed to all invocations of the archive librarian, which is used to generate static libraries."; diff --git a/Sources/SWBCore/TargetDependencyResolver.swift b/Sources/SWBCore/TargetDependencyResolver.swift index 9a0e4dc0..018abdc2 100644 --- a/Sources/SWBCore/TargetDependencyResolver.swift +++ b/Sources/SWBCore/TargetDependencyResolver.swift @@ -25,7 +25,7 @@ public enum TargetDependencyReason: Sendable { /// - parameter buildPhase: The name of the build phase used to find this linkage. This is used for diagnostics. case implicitBuildPhaseLinkage(filename: String, buildableItem: BuildFile.BuildableItem, buildPhase: String) /// The upstream target has an implicit dependency on the target due to options being passed via a build setting. - case implicitBuildSettingLinkage(settingName: String, options: [String]) + case implicitBuildSetting(settingName: String, options: [String]) /// The upstream target has a transitive dependency on the target via target(s) which were removed from the build graph. case impliedByTransitiveDependencyViaRemovedTargets(intermediateTargetName: String) } @@ -213,7 +213,7 @@ public struct TargetBuildGraph: TargetGraph, Sendable { dependencyString = "Explicit dependency on \(dependencyDescription)" case .implicitBuildPhaseLinkage(filename: let filename, buildableItem: _, buildPhase: let buildPhase): dependencyString = "Implicit dependency on \(dependencyDescription) via file '\(filename)' in build phase '\(buildPhase)'" - case .implicitBuildSettingLinkage(settingName: let settingName, options: let options): + case .implicitBuildSetting(settingName: let settingName, options: let options): dependencyString = "Implicit dependency on \(dependencyDescription) via options '\(options.joined(separator: " "))' in build setting '\(settingName)'" case .impliedByTransitiveDependencyViaRemovedTargets(let intermediateTargetName): dependencyString = "Dependency on \(dependencyDescription) via transitive dependency through '\(intermediateTargetName)'" diff --git a/Tests/SWBCoreTests/TargetDependencyResolverTests.swift b/Tests/SWBCoreTests/TargetDependencyResolverTests.swift index 69a6a6ae..0b43a0dc 100644 --- a/Tests/SWBCoreTests/TargetDependencyResolverTests.swift +++ b/Tests/SWBCoreTests/TargetDependencyResolverTests.swift @@ -4591,6 +4591,84 @@ fileprivate enum TargetPlatformSpecializationMode { XCTAssertEqualSequences(buildGraph.allTargets.map({ $0.target.name }).sorted(), ["AppTarget", "AlwaysUsedDependency"].sorted()) } } + + @Test + func appAndFrameworkModuleDependencies() async throws { + let core = try await getCore() + + let workspace = try TestWorkspace( + "Workspace", + projects: [ + TestProject( + "P1", + groupTree: TestGroup( + "G1", + children: [ + TestFile("aFramework.framework"), + ] + ), + buildConfigurations: [ + TestBuildConfiguration("Debug", buildSettings: [:]), + ], + targets: [ + TestStandardTarget( + "anApp", + type: .application, + buildConfigurations: [ + TestBuildConfiguration("Debug", buildSettings: [ + "PRODUCT_NAME": "anApp", + "MODULE_DEPENDENCIES": "'public aFramework' nonExisting", + ]), + ] + ) + ] + ), + TestProject( + "P2", + groupTree: TestGroup( + "G2", + children:[ + ] + ), + buildConfigurations: [ + TestBuildConfiguration("Debug", buildSettings: [:]), + ], + targets: [ + TestStandardTarget( + "aFramework", + type: .framework, + buildConfigurations: [ + TestBuildConfiguration("Debug", buildSettings: ["PRODUCT_NAME": "aFramework"]), + ] + ), + ] + ), + ] + ).load(core) + let workspaceContext = WorkspaceContext(core: core, workspace: workspace, processExecutionCache: .sharedForTesting) + + // Perform some simple correctness tests. + #expect(workspace.projects.count == 2) + let appProject = workspace.projects[0] + let fwkProject = workspace.projects[1] + + // Configure the targets and create a BuildRequest. + let buildParameters = BuildParameters(configuration: "Debug") + let appTarget = BuildRequest.BuildTargetInfo(parameters: buildParameters, target: appProject.targets[0]) + let fwkTarget = BuildRequest.BuildTargetInfo(parameters: buildParameters, target: fwkProject.targets[0]) + let buildRequest = BuildRequest(parameters: buildParameters, buildTargets: [appTarget], continueBuildingAfterErrors: true, useParallelTargets: false, useImplicitDependencies: true, useDryRun: false) + let buildRequestContext = BuildRequestContext(workspaceContext: workspaceContext) + + let delegate = EmptyTargetDependencyResolverDelegate(workspace: workspaceContext.workspace) + + // Get the dependency closure for the build request and examine it. + let buildGraph = await TargetGraphFactory(workspaceContext: workspaceContext, buildRequest: buildRequest, buildRequestContext: buildRequestContext, delegate: delegate).graph(type: .dependency) + let dependencyClosure = buildGraph.allTargets + #expect(dependencyClosure.map({ $0.target.name }) == ["aFramework", "anApp"]) + #expect(try buildGraph.dependencies(appTarget) == [try buildGraph.target(for: fwkTarget)]) + #expect(try buildGraph.dependencies(fwkTarget) == []) + delegate.checkNoDiagnostics() + } } @Suite fileprivate struct SuperimposedPropertiesTests: CoreBasedTests { From 60b27cb39a34233fcc620f7b3986520b710d07ff Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Mon, 2 Jun 2025 17:34:27 -0700 Subject: [PATCH 14/54] Improve printed descriptions of task backtraces - Clarify frame descriptions based on feedback - expose public API for rendering a backtrace as text rdar://152194638 --- Sources/SWBBuildSystem/BuildOperation.swift | 14 +- .../SWBBuildOperationBacktraceFrame.swift | 11 ++ .../TaskBacktraces.swift | 8 ++ .../BuildBacktraceTests.swift | 123 ++++++++++++++++-- 4 files changed, 139 insertions(+), 17 deletions(-) diff --git a/Sources/SWBBuildSystem/BuildOperation.swift b/Sources/SWBBuildSystem/BuildOperation.swift index 9efc58ee..dc91a5f6 100644 --- a/Sources/SWBBuildSystem/BuildOperation.swift +++ b/Sources/SWBBuildSystem/BuildOperation.swift @@ -2304,9 +2304,9 @@ internal final class OperationSystemAdaptor: SWBLLBuild.BuildSystemDelegate, Act private func inputNounPhraseForBuildKey(_ inputKey: BuildKey) -> String { switch inputKey { case is BuildKey.Command, is BuildKey.CustomTask: - return "the producer" + return "the task producing" case is BuildKey.DirectoryContents, is BuildKey.FilteredDirectoryContents, is BuildKey.DirectoryTreeSignature, is BuildKey.Node: - return "an input" + return "an input of" case is BuildKey.Target, is BuildKey.Stat: return "" default: @@ -2343,15 +2343,15 @@ internal final class OperationSystemAdaptor: SWBLLBuild.BuildSystemDelegate, Act previousFrameID = nil case .signatureChanged: category = .ruleSignatureChanged - description = "signature of \(descriptionForBuildKey(rule)) changed" + description = "arguments, environment, or working directory of \(descriptionForBuildKey(rule)) changed" previousFrameID = nil case .invalidValue: category = .ruleHadInvalidValue previousFrameID = nil if let command = rule as? BuildKey.Command, let task = lookupTask(TaskIdentifier(rawValue: command.name)), task.alwaysExecuteTask { - description = "\(descriptionForBuildKey(rule)) is configured to run in every incremental build" + description = "\(descriptionForBuildKey(rule)) was configured to run in every incremental build" } else if rule is BuildKey.Command || rule is BuildKey.CustomTask { - description = "\(descriptionForBuildKey(rule)) did not have up-to-date outputs" + description = "outputs of \(descriptionForBuildKey(rule)) were missing or modified" } else { description = "\(descriptionForBuildKey(rule)) changed" } @@ -2361,7 +2361,7 @@ internal final class OperationSystemAdaptor: SWBLLBuild.BuildSystemDelegate, Act if isTriggerNode(rule), let mutatedNodeDescription = descriptionOfInputMutatedByBuildKey(inputRule) { description = "\(descriptionForBuildKey(inputRule)) mutated \(mutatedNodeDescription)" } else { - description = "\(inputNounPhraseForBuildKey(inputRule)) of \(descriptionForBuildKey(rule)) \(rebuiltVerbPhraseForBuildKey(inputRule))" + description = "\(inputNounPhraseForBuildKey(inputRule)) \(descriptionForBuildKey(rule)) \(rebuiltVerbPhraseForBuildKey(inputRule))" } previousFrameID = previousFrameIdentifier } else { @@ -2370,7 +2370,7 @@ internal final class OperationSystemAdaptor: SWBLLBuild.BuildSystemDelegate, Act } case .forced: category = .ruleForced - description = "\(descriptionForBuildKey(rule)) was forced to run" + description = "\(descriptionForBuildKey(rule)) was forced to run to break a cycle in the build graph" previousFrameID = nil @unknown default: category = .none diff --git a/Sources/SwiftBuild/SWBBuildOperationBacktraceFrame.swift b/Sources/SwiftBuild/SWBBuildOperationBacktraceFrame.swift index 7bb1fce4..cb0109d4 100644 --- a/Sources/SwiftBuild/SWBBuildOperationBacktraceFrame.swift +++ b/Sources/SwiftBuild/SWBBuildOperationBacktraceFrame.swift @@ -193,4 +193,15 @@ public struct SWBTaskBacktrace { } self.frames = frames } + + public func renderTextualRepresentation() -> String { + var textualBacktrace: String = "" + for (frameNumber, frame) in frames.enumerated() { + guard frame.category.isUserFacing else { + continue + } + textualBacktrace += "#\(frameNumber): \(frame.description)\n" + } + return textualBacktrace + } } diff --git a/Sources/SwiftBuildTestSupport/TaskBacktraces.swift b/Sources/SwiftBuildTestSupport/TaskBacktraces.swift index 423937e6..e0f196fa 100644 --- a/Sources/SwiftBuildTestSupport/TaskBacktraces.swift +++ b/Sources/SwiftBuildTestSupport/TaskBacktraces.swift @@ -80,6 +80,14 @@ extension BuildOperationTester.BuildResults { } } } + + package func checkTextualBacktrace(_ task: Task, _ expected: String, sourceLocation: SourceLocation = #_sourceLocation) { + if let frameID = getBacktraceID(task, sourceLocation: sourceLocation), let backtrace = reconstructBacktrace(for: frameID) { + #expect(backtrace.renderTextualRepresentation() == expected, sourceLocation: sourceLocation) + } else { + // already recorded an issue + } + } } extension BuildOperationTester { diff --git a/Tests/SWBBuildSystemTests/BuildBacktraceTests.swift b/Tests/SWBBuildSystemTests/BuildBacktraceTests.swift index a6cbc12e..1e4dd699 100644 --- a/Tests/SWBBuildSystemTests/BuildBacktraceTests.swift +++ b/Tests/SWBBuildSystemTests/BuildBacktraceTests.swift @@ -121,11 +121,11 @@ fileprivate struct BuildBacktraceTests: CoreBasedTests { results.checkTask(.matchTargetName("TargetBar"), .matchRuleType("Ld")) { task in results.checkBacktrace(task, [ "", - "", + "", "", - "", + "", "", - "", + "", "", "" ]) @@ -139,8 +139,8 @@ fileprivate struct BuildBacktraceTests: CoreBasedTests { results.checkTask(.matchTargetName("TargetFoo"), .matchRuleType("CompileC")) { task in results.checkBacktrace(task, [ "", - "", - "" + "", + "" ]) } if tester.fs.fileSystemMode == .checksumOnly { @@ -155,10 +155,10 @@ fileprivate struct BuildBacktraceTests: CoreBasedTests { results.checkTask(.matchTargetName("TargetBar"), .matchRuleType("Ld")) { task in results.checkBacktrace(task, [ "", - "", + "", "", - "", - "" + "", + "" ]) } } @@ -320,7 +320,7 @@ fileprivate struct BuildBacktraceTests: CoreBasedTests { results.checkNoDiagnostics() results.checkTask(.matchTargetName("TargetFoo"), .matchRuleType("CompileC")) { task in results.checkBacktrace(task, [ - "", + "", ]) } } @@ -374,7 +374,7 @@ fileprivate struct BuildBacktraceTests: CoreBasedTests { results.checkNoDiagnostics() results.checkTask(.matchTargetName("TargetFoo"), .matchRuleType("PhaseScriptExecution")) { task in results.checkBacktrace(task, [ - "" + "" ]) } } @@ -471,4 +471,107 @@ fileprivate struct BuildBacktraceTests: CoreBasedTests { } } } + + @Test(.requireSDKs(.macOS)) + func backtraceTextRendering() async throws { + try await withTemporaryDirectory { tmpDirPath async throws -> Void in + let testWorkspace = TestWorkspace( + "Test", + sourceRoot: tmpDirPath.join("Test"), + projects: [ + TestProject( + "aProject", + groupTree: TestGroup( + "Sources", + path: "Sources", + children: [ + TestFile("foo.c"), + TestFile("bar.c"), + ]), + buildConfigurations: [ + TestBuildConfiguration( + "Debug", + buildSettings: [ + "PRODUCT_NAME": "$(TARGET_NAME)", + ]) + ], + targets: [ + TestStandardTarget( + "TargetFoo", + type: .framework, + buildPhases: [ + TestSourcesBuildPhase([ + "foo.c", + ]), + ]), + TestStandardTarget( + "TargetBar", + type: .framework, + buildPhases: [ + TestSourcesBuildPhase([ + "bar.c", + ]), + TestFrameworksBuildPhase([ + "TargetFoo.framework" + ]) + ], dependencies: ["TargetFoo"]), + ]) + ]) + + let tester = try await BuildOperationTester(getCore(), testWorkspace, simulated: false, fileSystem: localFS) + let parameters = BuildParameters(configuration: "Debug") + let buildRequest = BuildRequest(parameters: parameters, buildTargets: tester.workspace.projects[0].targets.map({ BuildRequest.BuildTargetInfo(parameters: parameters, target: $0) }), dependencyScope: .workspace, continueBuildingAfterErrors: true, useParallelTargets: true, useImplicitDependencies: false, useDryRun: false) + let SRCROOT = testWorkspace.sourceRoot.join("aProject") + + // Create the source files. + try await tester.fs.writeFileContents(SRCROOT.join("Sources/foo.c")) { file in + file <<< + """ + int foo(void) { + return 1; + } + """ + } + try await tester.fs.writeFileContents(SRCROOT.join("Sources/bar.c")) { file in + file <<< + """ + int bar(void) { + return 2; + } + """ + } + + try await tester.checkBuild(runDestination: .macOS, buildRequest: buildRequest, persistent: true) { results in + results.checkNoDiagnostics() + } + + try await tester.checkNullBuild(runDestination: .macOS, buildRequest: buildRequest, persistent: true) + + try await tester.fs.writeFileContents(SRCROOT.join("Sources/foo.c")) { file in + file <<< + """ + int foo2(void) { + return 42; + } + """ + } + + try await tester.checkBuild(runDestination: .macOS, buildRequest: buildRequest, persistent: true) { results in + results.checkNoDiagnostics() + results.checkTask(.matchTargetName("TargetBar"), .matchRuleType("Ld")) { task in + results.checkTextualBacktrace(task, """ + #0: an input of 'Link TargetBar (x86_64)' changed + #1: the task producing file '\(SRCROOT.str)/build/EagerLinkingTBDs/Debug/TargetFoo.framework/Versions/A/TargetFoo.tbd' ran + #2: an input of 'Generate TBD TargetFoo' changed + #3: the task producing file '\(SRCROOT.str)/build/Debug/TargetFoo.framework/Versions/A/TargetFoo' ran + #4: an input of 'Link TargetFoo (x86_64)' changed + #5: the task producing file '\(SRCROOT.str)/build/aProject.build/Debug/TargetFoo.build/Objects-normal/x86_64/foo.o' ran + #6: an input of 'Compile foo.c (x86_64)' changed + #7: file '\(SRCROOT.str)/Sources/foo.c' changed + + """) + } + } + } + } } From 76735f37c2b7bcaeb155fac9441d2f6eee137aa7 Mon Sep 17 00:00:00 2001 From: Paul LeMarquand Date: Tue, 3 Jun 2025 11:30:54 -0400 Subject: [PATCH 15/54] Support Cancellation in AsyncOperationQueue The original `AsyncOperationQueue` was ported in to `swift-package-manager`, where it was updated to support cancellation. This patch backports it to `swift-build` so that the two implementations are the same. --- .../SWBTestSupport/BuildOperationTester.swift | 8 +- Sources/SWBUtil/AsyncOperationQueue.swift | 204 +++++++++++++++--- 2 files changed, 182 insertions(+), 30 deletions(-) diff --git a/Sources/SWBTestSupport/BuildOperationTester.swift b/Sources/SWBTestSupport/BuildOperationTester.swift index df0f3878..a5d1ec09 100644 --- a/Sources/SWBTestSupport/BuildOperationTester.swift +++ b/Sources/SWBTestSupport/BuildOperationTester.swift @@ -1476,7 +1476,7 @@ package final class BuildOperationTester { /// Construct the tasks for the given build parameters, and test the result. @discardableResult package func checkBuild(_ name: String? = nil, parameters: BuildParameters? = nil, runDestination: SWBProtocol.RunDestinationInfo?, buildRequest inputBuildRequest: BuildRequest? = nil, buildCommand: BuildCommand? = nil, schemeCommand: SchemeCommand? = .launch, persistent: Bool = false, serial: Bool = false, buildOutputMap: [String:String]? = nil, signableTargets: Set = [], signableTargetInputs: [String: ProvisioningTaskInputs] = [:], clientDelegate: (any ClientDelegate)? = nil, sourceLocation: SourceLocation = #_sourceLocation, body: (BuildResults) async throws -> T) async throws -> T { - try await checkBuild(name, parameters: parameters, runDestination: runDestination, buildRequest: inputBuildRequest, buildCommand: buildCommand, schemeCommand: schemeCommand, persistent: persistent, serial: serial, buildOutputMap: buildOutputMap, signableTargets: signableTargets, signableTargetInputs: signableTargetInputs, clientDelegate: clientDelegate, sourceLocation: sourceLocation, body: body, performBuild: { await $0.buildWithTimeout() }) + try await checkBuild(name, parameters: parameters, runDestination: runDestination, buildRequest: inputBuildRequest, buildCommand: buildCommand, schemeCommand: schemeCommand, persistent: persistent, serial: serial, buildOutputMap: buildOutputMap, signableTargets: signableTargets, signableTargetInputs: signableTargetInputs, clientDelegate: clientDelegate, sourceLocation: sourceLocation, body: body, performBuild: { try await $0.buildWithTimeout() }) } /// Construct the tasks for the given build parameters, and test the result. @@ -1670,7 +1670,7 @@ package final class BuildOperationTester { let operationParameters = buildRequest.parameters.replacing(activeRunDestination: runDestination, activeArchitecture: nil) let operationBuildRequest = buildRequest.with(parameters: operationParameters, buildTargets: []) - return try await checkBuild(runDestination: nil, buildRequest: buildRequest, operationBuildRequest: operationBuildRequest, persistent: persistent, sourceLocation: sourceLocation, body: body, performBuild: { await $0.buildWithTimeout() }) + return try await checkBuild(runDestination: nil, buildRequest: buildRequest, operationBuildRequest: operationBuildRequest, persistent: persistent, sourceLocation: sourceLocation, body: body, performBuild: { try await $0.buildWithTimeout() }) } package struct BuildGraphResult: Sendable { @@ -2306,8 +2306,8 @@ private let buildSystemOperationQueue = AsyncOperationQueue(concurrentTasks: 6) extension BuildSystemOperation { /// Runs the build system operation -- responds to cooperative cancellation and limited to 6 concurrent operations per process. - func buildWithTimeout() async { - await buildSystemOperationQueue.withOperation { + func buildWithTimeout() async throws { + try await buildSystemOperationQueue.withOperation { do { try await withTimeout(timeout: .seconds(1200), description: "Build system operation 20-minute limit") { await withTaskCancellationHandler { diff --git a/Sources/SWBUtil/AsyncOperationQueue.swift b/Sources/SWBUtil/AsyncOperationQueue.swift index 1146d083..1ef3b936 100644 --- a/Sources/SWBUtil/AsyncOperationQueue.swift +++ b/Sources/SWBUtil/AsyncOperationQueue.swift @@ -10,52 +10,204 @@ // //===----------------------------------------------------------------------===// -public actor AsyncOperationQueue { +import Foundation + +/// A queue for running async operations with a limit on the number of concurrent tasks. +public final class AsyncOperationQueue: @unchecked Sendable { + + // This implementation is identical to the AsyncOperationQueue in swift-package-manager. + // Any modifications made here should also be made there. + // https://github.com/swiftlang/swift-build/blob/main/Sources/SWBUtil/AsyncOperationQueue.swift#L13 + + fileprivate typealias ID = UUID + fileprivate typealias WaitingContinuation = CheckedContinuation + private let concurrentTasks: Int - private var activeTasks: Int = 0 - private var waitingTasks: [CheckedContinuation] = [] + private var waitingTasks: [WorkTask] = [] + private let waitingTasksLock = NSLock() + + fileprivate enum WorkTask { + case creating(ID) + case waiting(ID, WaitingContinuation) + case running(ID) + case cancelled(ID) + + var id: ID { + switch self { + case .creating(let id), .waiting(let id, _), .running(let id), .cancelled(let id): + return id + } + } + + var continuation: WaitingContinuation? { + guard case .waiting(_, let continuation) = self else { + return nil + } + return continuation + } + } + /// Creates an `AsyncOperationQueue` with a specified number of concurrent tasks. + /// - Parameter concurrentTasks: The maximum number of concurrent tasks that can be executed concurrently. public init(concurrentTasks: Int) { self.concurrentTasks = concurrentTasks } deinit { - if !waitingTasks.isEmpty { - preconditionFailure("Deallocated with waiting tasks") + waitingTasksLock.withLock { + if !waitingTasks.isEmpty { + preconditionFailure("Deallocated with waiting tasks") + } } } + /// Executes an asynchronous operation, ensuring that the number of concurrent tasks + // does not exceed the specified limit. + /// - Parameter operation: The asynchronous operation to execute. + /// - Returns: The result of the operation. + /// - Throws: An error thrown by the operation, or a `CancellationError` if the operation is cancelled. public func withOperation( - _ operation: @Sendable () async -> sending ReturnValue - ) async -> ReturnValue { - await waitIfNeeded() - defer { signalCompletion() } - return await operation() - } - - public func withOperation( - _ operation: @Sendable () async throws -> sending ReturnValue + _ operation: () async throws -> sending ReturnValue ) async throws -> ReturnValue { - await waitIfNeeded() - defer { signalCompletion() } + let taskId = try await waitIfNeeded() + defer { signalCompletion(taskId) } return try await operation() } - private func waitIfNeeded() async { - if activeTasks >= concurrentTasks { - await withCheckedContinuation { continuation in - waitingTasks.append(continuation) - } + private func waitIfNeeded() async throws -> ID { + let workTask = waitingTasksLock.withLock({ + let shouldWait = waitingTasks.count >= concurrentTasks + let workTask = shouldWait ? WorkTask.creating(ID()) : .running(ID()) + waitingTasks.append(workTask) + return workTask + }) + + // If we aren't creating a task that needs to wait, we're under the concurrency limit. + guard case .creating(let taskId) = workTask else { + return workTask.id } - activeTasks += 1 + enum TaskAction { + case start(WaitingContinuation) + case cancel(WaitingContinuation) + } + + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: WaitingContinuation) -> Void in + let action: TaskAction? = waitingTasksLock.withLock { + guard let index = waitingTasks.firstIndex(where: { $0.id == taskId }) else { + // The task may have been marked as cancelled already and then removed from + // waitingTasks in `signalCompletion`. + return .cancel(continuation) + } + + switch waitingTasks[index] { + case .cancelled: + // If the task was cancelled in between creating the task cancellation handler and acquiring the lock, + // we should resume the continuation with a `CancellationError`. + waitingTasks.remove(at: index) + return .cancel(continuation) + case .creating, .running, .waiting: + // A task may have completed since we initially checked if we should wait. Check again in this locked + // section and if we can start it, remove it from the waiting tasks and start it immediately. + if waitingTasks.count >= concurrentTasks { + waitingTasks[index] = .waiting(taskId, continuation) + return nil + } else { + waitingTasks.remove(at: index) + return .start(continuation) + } + } + } + + switch action { + case .some(.cancel(let continuation)): + continuation.resume(throwing: _Concurrency.CancellationError()) + case .some(.start(let continuation)): + continuation.resume() + case .none: + return + } + } + } onCancel: { + let continuation: WaitingContinuation? = self.waitingTasksLock.withLock { + guard let taskIndex = self.waitingTasks.firstIndex(where: { $0.id == taskId }) else { + return nil + } + + switch self.waitingTasks[taskIndex] { + case .waiting(_, let continuation): + self.waitingTasks.remove(at: taskIndex) + + // If the parent task is cancelled then we need to manually handle resuming the + // continuation for the waiting task with a `CancellationError`. Return the continuation + // here so it can be resumed once the `waitingTasksLock` is released. + return continuation + case .creating, .running: + // If the task was still being created, mark it as cancelled in `waitingTasks` so that + // the handler for `withCheckedThrowingContinuation` can immediately cancel it. + self.waitingTasks[taskIndex] = .cancelled(taskId) + return nil + case .cancelled: + preconditionFailure("Attempting to cancel a task that was already cancelled") + } + } + + continuation?.resume(throwing: _Concurrency.CancellationError()) + } + return workTask.id } - private func signalCompletion() { - activeTasks -= 1 + private func signalCompletion(_ taskId: ID) { + let continuationToResume = waitingTasksLock.withLock { () -> WaitingContinuation? in + guard !waitingTasks.isEmpty else { + return nil + } - if let continuation = waitingTasks.popLast() { - continuation.resume() + // Remove the completed task from the list to decrement the active task count. + if let taskIndex = self.waitingTasks.firstIndex(where: { $0.id == taskId }) { + waitingTasks.remove(at: taskIndex) + } + + // We cannot remove elements from `waitingTasks` while iterating over it, so we make + // a pass to collect operations and then apply them after the loop. + func createTaskListOperations() -> (CollectionDifference?, WaitingContinuation?) { + var changes: [CollectionDifference.Change] = [] + for (index, task) in waitingTasks.enumerated() { + switch task { + case .running: + // Skip tasks that are already running, looking for the first one that is waiting or creating. + continue + case .creating: + // If the next task is in the process of being created, let the + // creation code in the `withCheckedThrowingContinuation` in `waitIfNeeded` + // handle starting the task. + break + case .waiting: + // Begin the next waiting task + changes.append(.remove(offset: index, element: task, associatedWith: nil)) + return (CollectionDifference(changes), task.continuation) + case .cancelled: + // If the next task is cancelled, continue removing cancelled + // tasks until we find one that hasn't run yet, or we exaust the list of waiting tasks. + changes.append(.remove(offset: index, element: task, associatedWith: nil)) + continue + } + } + return (CollectionDifference(changes), nil) + } + + let (collectionOperations, continuation) = createTaskListOperations() + if let operations = collectionOperations { + guard let appliedDiff = waitingTasks.applying(operations) else { + preconditionFailure("Failed to apply changes to waiting tasks") + } + waitingTasks = appliedDiff + } + + return continuation } + + continuationToResume?.resume() } } From 3a5f0ececd8c30d16d4eaa436a07d60785eaa6f1 Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Mon, 26 May 2025 11:24:25 -0700 Subject: [PATCH 16/54] Expose DYLIB_INSTALL_NAME_BASE and SWIFT_INDEX_STORE_ENABLE in PIF API --- Sources/SwiftBuild/ProjectModel/BuildSettings.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Sources/SwiftBuild/ProjectModel/BuildSettings.swift b/Sources/SwiftBuild/ProjectModel/BuildSettings.swift index a8909c03..ae10d9de 100644 --- a/Sources/SwiftBuild/ProjectModel/BuildSettings.swift +++ b/Sources/SwiftBuild/ProjectModel/BuildSettings.swift @@ -99,6 +99,7 @@ extension ProjectModel { case SUPPORTS_TEXT_BASED_API case SUPPRESS_WARNINGS case SWIFT_ENABLE_BARE_SLASH_REGEX + case SWIFT_INDEX_STORE_ENABLE case SWIFT_INSTALL_MODULE case SWIFT_PACKAGE_NAME case SWIFT_USER_MODULE_VERSION @@ -146,6 +147,7 @@ extension ProjectModel { case SPECIALIZATION_SDK_OPTIONS case SWIFT_VERSION case SWIFT_ACTIVE_COMPILATION_CONDITIONS + case DYLIB_INSTALL_NAME_BASE } public enum Platform: Hashable, CaseIterable, Sendable { From a6b2109b8d1179126618b2e7d2a9c5af8e474500 Mon Sep 17 00:00:00 2001 From: Ian Anderson Date: Mon, 2 Jun 2025 16:34:23 -0700 Subject: [PATCH 17/54] SWIFT_SYSTEM_INCLUDE_PATHS doesn't work everywhere SWIFT_INCLUDE_PATHS does There's a few spots where SWIFT_INCLUDE_PATHS is being used but SWIFT_SYSTEM_INCLUDE_PATHS isn't. Add those. rdar://152384124 --- .../SpecImplementations/Tools/SwiftABICheckerTool.swift | 2 ++ .../SpecImplementations/Tools/SwiftABIGenerationTool.swift | 2 ++ .../SwiftABICheckerTaskConstructionTests.swift | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/Sources/SWBCore/SpecImplementations/Tools/SwiftABICheckerTool.swift b/Sources/SWBCore/SpecImplementations/Tools/SwiftABICheckerTool.swift index d064899a..d3eca2b1 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/SwiftABICheckerTool.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/SwiftABICheckerTool.swift @@ -82,6 +82,8 @@ public final class SwiftABICheckerToolSpec : GenericCommandLineToolSpec, SpecIde for searchPath in SwiftCompilerSpec.collectInputSearchPaths(cbc, toolInfo: toolSpecInfo) { commandLine += ["-I", searchPath] } + // swift-api-digester doesn't support -Isystem or -Fsystem. + commandLine += cbc.scope.evaluate(BuiltinMacros.SWIFT_SYSTEM_INCLUDE_PATHS).flatMap { ["-I", $0] } commandLine += cbc.scope.evaluate(BuiltinMacros.SYSTEM_FRAMEWORK_SEARCH_PATHS).flatMap { ["-F", $0] } delegate.createTask(type: self, payload: ABICheckerPayload(serializedDiagnosticsPath: serializedDiagsPath), diff --git a/Sources/SWBCore/SpecImplementations/Tools/SwiftABIGenerationTool.swift b/Sources/SWBCore/SpecImplementations/Tools/SwiftABIGenerationTool.swift index 21f46270..b407fe1b 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/SwiftABIGenerationTool.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/SwiftABIGenerationTool.swift @@ -40,11 +40,13 @@ public final class SwiftABIGenerationToolSpec : GenericCommandLineToolSpec, Spec var commandLine = await commandLineFromTemplate(cbc, delegate, optionContext: discoveredCommandLineToolSpecInfo(cbc.producer, cbc.scope, delegate)).map(\.asString) commandLine += ["-o", baselineFile.normalize().str] + // swift-api-digester doesn't support -Fsystem or -Isystem. commandLine += cbc.scope.evaluate(BuiltinMacros.SYSTEM_FRAMEWORK_SEARCH_PATHS).flatMap { ["-F", $0] } // Add import search paths for searchPath in SwiftCompilerSpec.collectInputSearchPaths(cbc, toolInfo: toolSpecInfo) { commandLine += ["-I", searchPath] } + commandLine += cbc.scope.evaluate(BuiltinMacros.SWIFT_SYSTEM_INCLUDE_PATHS).flatMap { ["-I", $0] } delegate.createTask(type: self, ruleInfo: defaultRuleInfo(cbc, delegate), commandLine: commandLine, diff --git a/Tests/SWBTaskConstructionTests/SwiftABICheckerTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/SwiftABICheckerTaskConstructionTests.swift index 75c6b9c7..4f47f007 100644 --- a/Tests/SWBTaskConstructionTests/SwiftABICheckerTaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/SwiftABICheckerTaskConstructionTests.swift @@ -40,6 +40,8 @@ fileprivate struct SwiftABICheckerTaskConstructionTests: CoreBasedTests { "SWIFT_VERSION": swiftVersion, "FRAMEWORK_SEARCH_PATHS": "/Target/Framework/Search/Path/A", "SYSTEM_FRAMEWORK_SEARCH_PATHS": "/Target/System/Framework/Search/Path/A", + "SWIFT_INCLUDE_PATHS": "/Target/Import/Search/Path/A", + "SWIFT_SYSTEM_INCLUDE_PATHS": "/Target/System/Import/Search/Path/A", "CODE_SIGNING_ALLOWED": "NO", "BUILD_LIBRARY_FOR_DISTRIBUTION": "YES", "TAPI_EXEC": tapiToolPath.str, @@ -68,6 +70,8 @@ fileprivate struct SwiftABICheckerTaskConstructionTests: CoreBasedTests { "-target", "\(arch)-apple-ios\(core.loadSDK(.iOS).defaultDeploymentTarget)", "-F", "/TEST/build/Debug-iphoneos", "-module", "Fwk", + "-I", "/Target/Import/Search/Path/A", + "-I", "/Target/System/Import/Search/Path/A", "-F", "/Target/System/Framework/Search/Path/A" ]) } From 0135fdb9cf22fd742d7763c0f1af8e4506a7b98d Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Mon, 26 May 2025 11:24:51 -0700 Subject: [PATCH 18/54] Add support for generated unit test runners with xctest discovery --- .../Specs/DarwinProductTypes.xcspec | 13 + Sources/SWBCSupport/IndexStore.h | 194 ++++++ Sources/SWBCSupport/SWBCSupport.h | 1 + Sources/SWBCore/Settings/BuiltinMacros.swift | 2 + .../SpecImplementations/ProductTypes.swift | 10 +- .../Tools/SwiftCompiler.swift | 3 + .../SWBGenericUnixPlatform/Specs/Unix.xcspec | 35 +- .../SWBProjectModel/PIFGenerationModel.swift | 2 + Sources/SWBQNXPlatform/Specs/QNX.xcspec | 33 +- .../ProductPlanning/ProductPlan.swift | 2 +- .../InfoPlistTaskProducer.swift | 2 +- .../SwiftStandardLibrariesTaskProducer.swift | 2 +- Sources/SWBTestSupport/TestWorkspaces.swift | 6 +- .../Specs/ProductTypes.xcspec | 15 + .../TestEntryPointGenerationTaskAction.swift | 603 +++++++++++++++++- .../TestEntryPointGenerationTool.swift | 52 ++ .../TestEntryPointTaskProducer.swift | 51 +- Sources/SWBUtil/CMakeLists.txt | 1 + Sources/SWBUtil/IndexStore.swift | 389 +++++++++++ .../SWBWindowsPlatform/Specs/Windows.xcspec | 25 + Sources/SwiftBuild/ProjectModel/Targets.swift | 1 + .../BuildOperationTests.swift | 52 +- .../UnitTestTaskConstructionTests.swift | 35 +- 23 files changed, 1440 insertions(+), 89 deletions(-) create mode 100644 Sources/SWBCSupport/IndexStore.h create mode 100644 Sources/SWBUtil/IndexStore.swift diff --git a/Sources/SWBApplePlatform/Specs/DarwinProductTypes.xcspec b/Sources/SWBApplePlatform/Specs/DarwinProductTypes.xcspec index e1bc5a17..d6e43e3f 100644 --- a/Sources/SWBApplePlatform/Specs/DarwinProductTypes.xcspec +++ b/Sources/SWBApplePlatform/Specs/DarwinProductTypes.xcspec @@ -461,4 +461,17 @@ ); Platforms = (driverkit); }, + { + _Domain = darwin; + Type = ProductType; + Identifier = com.apple.product-type.tool.swiftpm-test-runner; + Name = "SwiftPM Unit Test Runner"; + Description = "SwiftPM Unit Test Runner"; + DefaultBuildProperties = { + __SKIP_BUILD = YES; + }; + PackageTypes = ( + com.apple.package-type.mach-o-executable + ); + }, ) diff --git a/Sources/SWBCSupport/IndexStore.h b/Sources/SWBCSupport/IndexStore.h new file mode 100644 index 00000000..7d4b77b8 --- /dev/null +++ b/Sources/SWBCSupport/IndexStore.h @@ -0,0 +1,194 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#ifndef INDEXSTORE_H +#define INDEXSTORE_H + +#include +#include +#include +#include + +typedef void *indexstore_error_t; + +typedef struct { + const char *data; + size_t length; +} indexstore_string_ref_t; + +typedef void *indexstore_t; +typedef void *indexstore_symbol_t; + +typedef enum { + INDEXSTORE_SYMBOL_KIND_UNKNOWN = 0, + INDEXSTORE_SYMBOL_KIND_MODULE = 1, + INDEXSTORE_SYMBOL_KIND_NAMESPACE = 2, + INDEXSTORE_SYMBOL_KIND_NAMESPACEALIAS = 3, + INDEXSTORE_SYMBOL_KIND_MACRO = 4, + INDEXSTORE_SYMBOL_KIND_ENUM = 5, + INDEXSTORE_SYMBOL_KIND_STRUCT = 6, + INDEXSTORE_SYMBOL_KIND_CLASS = 7, + INDEXSTORE_SYMBOL_KIND_PROTOCOL = 8, + INDEXSTORE_SYMBOL_KIND_EXTENSION = 9, + INDEXSTORE_SYMBOL_KIND_UNION = 10, + INDEXSTORE_SYMBOL_KIND_TYPEALIAS = 11, + INDEXSTORE_SYMBOL_KIND_FUNCTION = 12, + INDEXSTORE_SYMBOL_KIND_VARIABLE = 13, + INDEXSTORE_SYMBOL_KIND_FIELD = 14, + INDEXSTORE_SYMBOL_KIND_ENUMCONSTANT = 15, + INDEXSTORE_SYMBOL_KIND_INSTANCEMETHOD = 16, + INDEXSTORE_SYMBOL_KIND_CLASSMETHOD = 17, + INDEXSTORE_SYMBOL_KIND_STATICMETHOD = 18, + INDEXSTORE_SYMBOL_KIND_INSTANCEPROPERTY = 19, + INDEXSTORE_SYMBOL_KIND_CLASSPROPERTY = 20, + INDEXSTORE_SYMBOL_KIND_STATICPROPERTY = 21, + INDEXSTORE_SYMBOL_KIND_CONSTRUCTOR = 22, + INDEXSTORE_SYMBOL_KIND_DESTRUCTOR = 23, + INDEXSTORE_SYMBOL_KIND_CONVERSIONFUNCTION = 24, + INDEXSTORE_SYMBOL_KIND_PARAMETER = 25, + INDEXSTORE_SYMBOL_KIND_USING = 26, + + INDEXSTORE_SYMBOL_KIND_COMMENTTAG = 1000, +} indexstore_symbol_kind_t; + +typedef enum { + INDEXSTORE_SYMBOL_PROPERTY_GENERIC = 1 << 0, + INDEXSTORE_SYMBOL_PROPERTY_TEMPLATE_PARTIAL_SPECIALIZATION = 1 << 1, + INDEXSTORE_SYMBOL_PROPERTY_TEMPLATE_SPECIALIZATION = 1 << 2, + INDEXSTORE_SYMBOL_PROPERTY_UNITTEST = 1 << 3, + INDEXSTORE_SYMBOL_PROPERTY_IBANNOTATED = 1 << 4, + INDEXSTORE_SYMBOL_PROPERTY_IBOUTLETCOLLECTION = 1 << 5, + INDEXSTORE_SYMBOL_PROPERTY_GKINSPECTABLE = 1 << 6, + INDEXSTORE_SYMBOL_PROPERTY_LOCAL = 1 << 7, + INDEXSTORE_SYMBOL_PROPERTY_PROTOCOL_INTERFACE = 1 << 8, + INDEXSTORE_SYMBOL_PROPERTY_SWIFT_ASYNC = 1 << 16, +} indexstore_symbol_property_t; + +typedef enum { + INDEXSTORE_SYMBOL_ROLE_DECLARATION = 1 << 0, + INDEXSTORE_SYMBOL_ROLE_DEFINITION = 1 << 1, + INDEXSTORE_SYMBOL_ROLE_REFERENCE = 1 << 2, + INDEXSTORE_SYMBOL_ROLE_READ = 1 << 3, + INDEXSTORE_SYMBOL_ROLE_WRITE = 1 << 4, + INDEXSTORE_SYMBOL_ROLE_CALL = 1 << 5, + INDEXSTORE_SYMBOL_ROLE_DYNAMIC = 1 << 6, + INDEXSTORE_SYMBOL_ROLE_ADDRESSOF = 1 << 7, + INDEXSTORE_SYMBOL_ROLE_IMPLICIT = 1 << 8, + INDEXSTORE_SYMBOL_ROLE_UNDEFINITION = 1 << 19, + + // Relation roles. + INDEXSTORE_SYMBOL_ROLE_REL_CHILDOF = 1 << 9, + INDEXSTORE_SYMBOL_ROLE_REL_BASEOF = 1 << 10, + INDEXSTORE_SYMBOL_ROLE_REL_OVERRIDEOF = 1 << 11, + INDEXSTORE_SYMBOL_ROLE_REL_RECEIVEDBY = 1 << 12, + INDEXSTORE_SYMBOL_ROLE_REL_CALLEDBY = 1 << 13, + INDEXSTORE_SYMBOL_ROLE_REL_EXTENDEDBY = 1 << 14, + INDEXSTORE_SYMBOL_ROLE_REL_ACCESSOROF = 1 << 15, + INDEXSTORE_SYMBOL_ROLE_REL_CONTAINEDBY = 1 << 16, + INDEXSTORE_SYMBOL_ROLE_REL_IBTYPEOF = 1 << 17, + INDEXSTORE_SYMBOL_ROLE_REL_SPECIALIZATIONOF = 1 << 18, +} indexstore_symbol_role_t; + +typedef void *indexstore_unit_dependency_t; + +typedef enum { + INDEXSTORE_UNIT_DEPENDENCY_UNIT = 1, + INDEXSTORE_UNIT_DEPENDENCY_RECORD = 2, + INDEXSTORE_UNIT_DEPENDENCY_FILE = 3, +} indexstore_unit_dependency_kind_t; + +typedef void *indexstore_symbol_relation_t; +typedef void *indexstore_occurrence_t; +typedef void *indexstore_record_reader_t; +typedef void *indexstore_unit_reader_t; + +typedef struct { + const char * + (*error_get_description)(indexstore_error_t); + + void + (*error_dispose)(indexstore_error_t); + + indexstore_t + (*store_create)(const char *store_path, indexstore_error_t *error); + + void + (*store_dispose)(indexstore_t); + + size_t + (*store_get_unit_name_from_output_path)(indexstore_t store, + const char *output_path, + char *name_buf, + size_t buf_size); + + indexstore_symbol_kind_t + (*symbol_get_kind)(indexstore_symbol_t); + + uint64_t + (*symbol_get_properties)(indexstore_symbol_t); + + indexstore_string_ref_t + (*symbol_get_name)(indexstore_symbol_t); + + uint64_t + (*symbol_relation_get_roles)(indexstore_symbol_relation_t); + + indexstore_symbol_t + (*symbol_relation_get_symbol)(indexstore_symbol_relation_t); + + indexstore_symbol_t + (*occurrence_get_symbol)(indexstore_occurrence_t); + + bool + (*occurrence_relations_apply_f)(indexstore_occurrence_t, + void *context, + bool(*applier)(void *context, indexstore_symbol_relation_t symbol_rel)); + + indexstore_record_reader_t + (*record_reader_create)(indexstore_t store, const char *record_name, + indexstore_error_t *error); + + void + (*record_reader_dispose)(indexstore_record_reader_t); + + bool + (*record_reader_occurrences_apply_f)(indexstore_record_reader_t, + void *context, + bool(*applier)(void *context, indexstore_occurrence_t occur)); + + indexstore_unit_reader_t + (*unit_reader_create)(indexstore_t store, const char *unit_name, + indexstore_error_t *error); + + void + (*unit_reader_dispose)(indexstore_unit_reader_t); + + indexstore_string_ref_t + (*unit_reader_get_module_name)(indexstore_unit_reader_t); + + indexstore_unit_dependency_kind_t + (*unit_dependency_get_kind)(indexstore_unit_dependency_t); + + indexstore_string_ref_t + (*unit_dependency_get_name)(indexstore_unit_dependency_t); + + bool + (*unit_reader_dependencies_apply)(indexstore_unit_reader_t, + bool(^applier)(indexstore_unit_dependency_t)); + + bool + (*unit_reader_dependencies_apply_f)(indexstore_unit_reader_t, + void *context, + bool(*applier)(void *context, indexstore_unit_dependency_t)); +} indexstore_functions_t; + +#endif diff --git a/Sources/SWBCSupport/SWBCSupport.h b/Sources/SWBCSupport/SWBCSupport.h index c020472c..18591c94 100644 --- a/Sources/SWBCSupport/SWBCSupport.h +++ b/Sources/SWBCSupport/SWBCSupport.h @@ -21,6 +21,7 @@ #include "CLibclang.h" #include "CLibRemarksHelper.h" +#include "IndexStore.h" #include "PluginAPI.h" #include "PluginAPI_functions.h" #include "PluginAPI_types.h" diff --git a/Sources/SWBCore/Settings/BuiltinMacros.swift b/Sources/SWBCore/Settings/BuiltinMacros.swift index 0ac934ff..52beb989 100644 --- a/Sources/SWBCore/Settings/BuiltinMacros.swift +++ b/Sources/SWBCore/Settings/BuiltinMacros.swift @@ -750,6 +750,7 @@ public final class BuiltinMacros { public static let INDEX_PREPARED_TARGET_MARKER_PATH = BuiltinMacros.declareStringMacro("INDEX_PREPARED_TARGET_MARKER_PATH") public static let INDEX_REGULAR_BUILD_PRODUCTS_DIR = BuiltinMacros.declareStringMacro("INDEX_REGULAR_BUILD_PRODUCTS_DIR") public static let INDEX_REGULAR_BUILD_INTERMEDIATES_DIR = BuiltinMacros.declareStringMacro("INDEX_REGULAR_BUILD_INTERMEDIATES_DIR") + public static let INDEX_STORE_LIBRARY_PATH = BuiltinMacros.declarePathMacro("INDEX_STORE_LIBRARY_PATH") public static let INFOPLIST_ENFORCE_MINIMUM_OS = BuiltinMacros.declareBooleanMacro("INFOPLIST_ENFORCE_MINIMUM_OS") public static let INFOPLIST_EXPAND_BUILD_SETTINGS = BuiltinMacros.declareBooleanMacro("INFOPLIST_EXPAND_BUILD_SETTINGS") public static let INFOPLIST_FILE = BuiltinMacros.declarePathMacro("INFOPLIST_FILE") @@ -1796,6 +1797,7 @@ public final class BuiltinMacros { INDEX_PREPARED_TARGET_MARKER_PATH, INDEX_REGULAR_BUILD_PRODUCTS_DIR, INDEX_REGULAR_BUILD_INTERMEDIATES_DIR, + INDEX_STORE_LIBRARY_PATH, INDEX_ENABLE_DATA_STORE, INDEX_PRECOMPS_DIR, INFOPLIST_ENFORCE_MINIMUM_OS, diff --git a/Sources/SWBCore/SpecImplementations/ProductTypes.swift b/Sources/SWBCore/SpecImplementations/ProductTypes.swift index 7cb6a1d5..28415e4b 100644 --- a/Sources/SWBCore/SpecImplementations/ProductTypes.swift +++ b/Sources/SWBCore/SpecImplementations/ProductTypes.swift @@ -321,7 +321,7 @@ public class ProductTypeSpec : Spec, SpecType, @unchecked Sendable { } /// Returns whether the product type supports embedding Swift standard libraries inside it. - public var supportsEmbeddingSwiftStandardLibraries: Bool { + public func supportsEmbeddingSwiftStandardLibraries(producer: CommandProducer) -> Bool { // Most product types don't support having the Swift libraries embedded in them. return false } @@ -381,7 +381,7 @@ public final class ApplicationProductTypeSpec : BundleProductTypeSpec, @unchecke return "PBXApplicationProductType" } - public override var supportsEmbeddingSwiftStandardLibraries: Bool { + public override func supportsEmbeddingSwiftStandardLibraries(producer: CommandProducer) -> Bool { return true } @@ -602,8 +602,8 @@ public final class XCTestBundleProductTypeSpec : BundleProductTypeSpec, @uncheck super.init(parser, basedOnSpec) } - public override var supportsEmbeddingSwiftStandardLibraries: Bool { - return true + public override func supportsEmbeddingSwiftStandardLibraries(producer: CommandProducer) -> Bool { + return producer.isApplePlatform } public class func usesXCTRunner(_ scope: MacroEvaluationScope) -> Bool { @@ -649,7 +649,7 @@ public final class XCTestBundleProductTypeSpec : BundleProductTypeSpec, @uncheck var (tableOpt, warnings, errors) = super.overridingBuildSettings(scope, platform: platform) var table = tableOpt ?? MacroValueAssignmentTable(namespace: scope.namespace) - let isDeviceBuild = platform?.isDeploymentPlatform == true && platform?.identifier != "com.apple.platform.macosx" + let isDeviceBuild = platform?.isDeploymentPlatform == true && platform?.name != scope.evaluate(BuiltinMacros.HOST_PLATFORM) if isDeviceBuild { // For tests running on devices (not simulators) we always want to generate dSYMs so that symbolication can give file and line information about test failures. table.push(BuiltinMacros.DEBUG_INFORMATION_FORMAT, literal: "dwarf-with-dsym") diff --git a/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift b/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift index 6a495ad6..98213dec 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift @@ -3758,6 +3758,9 @@ public extension BuildPhaseWithBuildFiles { /// - Returns: If the build phase contains any Swift source files that are not filtered out via the platform filter or excluded source file name patterns. func containsSwiftSources(_ referenceLookupContext: any ReferenceLookupContext, _ specLookupContext: any SpecLookupContext, _ scope: MacroEvaluationScope, _ filePathResolver: FilePathResolver) -> Bool { guard let swiftFileType = specLookupContext.lookupFileType(identifier: "sourcecode.swift") else { return false } + if scope.evaluate(BuiltinMacros.GENERATE_TEST_ENTRY_POINT) { + return true + } return containsFiles(ofType: swiftFileType, referenceLookupContext, specLookupContext, scope, filePathResolver) } } diff --git a/Sources/SWBGenericUnixPlatform/Specs/Unix.xcspec b/Sources/SWBGenericUnixPlatform/Specs/Unix.xcspec index eb72ba9c..00bc75d5 100644 --- a/Sources/SWBGenericUnixPlatform/Specs/Unix.xcspec +++ b/Sources/SWBGenericUnixPlatform/Specs/Unix.xcspec @@ -24,41 +24,18 @@ SortNumber = 0; }, - // Test type bundle (bodged to be a tool) { Domain = generic-unix; Type = ProductType; Identifier = com.apple.product-type.bundle.unit-test; - Class = PBXToolProductType; - Name = "Command-line Tool"; - Description = "Standalone command-line tool"; - DefaultTargetName = "Command-line Tool"; + BasedOn = com.apple.product-type.library.dynamic; DefaultBuildProperties = { - FULL_PRODUCT_NAME = "$(EXECUTABLE_NAME)"; - EXECUTABLE_PREFIX = ""; - EXECUTABLE_SUFFIX = ".xctest"; - REZ_EXECUTABLE = YES; - INSTALL_PATH = "/usr/local/bin"; - FRAMEWORK_FLAG_PREFIX = "-framework"; - LIBRARY_FLAG_PREFIX = "-l"; - LIBRARY_FLAG_NOSPACE = YES; - GCC_DYNAMIC_NO_PIC = NO; - LD_NO_PIE = NO; - GCC_SYMBOLS_PRIVATE_EXTERN = YES; - GCC_INLINES_ARE_PRIVATE_EXTERN = YES; - STRIP_STYLE = "all"; - CODE_SIGNING_ALLOWED = NO; - IsUnitTest = YES; - SWIFT_FORCE_DYNAMIC_LINK_STDLIB = YES; - SWIFT_FORCE_STATIC_LINK_STDLIB = NO; - // Avoid warning for executable types - ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = NO; - GENERATE_TEST_ENTRY_POINT = YES; - GENERATED_TEST_ENTRY_POINT_PATH = "$(DERIVED_SOURCES_DIR)/test_entry_point.swift"; + // Index store data is required to discover XCTest tests + COMPILER_INDEX_STORE_ENABLE = YES; + SWIFT_INDEX_STORE_ENABLE = YES; + // Testability is needed to generate code to invoke discovered XCTest tests + SWIFT_ENABLE_TESTABILITY = YES; }; - PackageTypes = ( - com.apple.package-type.mach-o-executable // default - ); }, // Dynamic library (masquerading as a framework to placate Swift's project structure) diff --git a/Sources/SWBProjectModel/PIFGenerationModel.swift b/Sources/SWBProjectModel/PIFGenerationModel.swift index 698a7343..f3db2bce 100644 --- a/Sources/SWBProjectModel/PIFGenerationModel.swift +++ b/Sources/SWBProjectModel/PIFGenerationModel.swift @@ -295,6 +295,7 @@ public enum PIF { case executable = "com.apple.product-type.tool" case hostBuildTool = "com.apple.product-type.tool.host-build" case unitTest = "com.apple.product-type.bundle.unit-test" + case swiftpmTestRunner = "com.apple.product-type.tool.swiftpm-test-runner" case bundle = "com.apple.product-type.bundle" case packageProduct = "packageProduct" public var asString: String { return rawValue } @@ -1022,6 +1023,7 @@ public enum PIF { public var SWIFT_ADD_TOOLCHAIN_SWIFTSYNTAX_SEARCH_PATHS: String? public var SWIFT_FORCE_STATIC_LINK_STDLIB: String? public var SWIFT_FORCE_DYNAMIC_LINK_STDLIB: String? + public var SWIFT_INDEX_STORE_ENABLE: String? public var SWIFT_INSTALL_OBJC_HEADER: String? public var SWIFT_LOAD_BINARY_MACROS: [String]? public var SWIFT_MODULE_ALIASES: [String]? diff --git a/Sources/SWBQNXPlatform/Specs/QNX.xcspec b/Sources/SWBQNXPlatform/Specs/QNX.xcspec index 3c72c620..aea12e3b 100644 --- a/Sources/SWBQNXPlatform/Specs/QNX.xcspec +++ b/Sources/SWBQNXPlatform/Specs/QNX.xcspec @@ -24,39 +24,18 @@ SortNumber = 0; }, - // Test type bundle (bodged to be a tool) { Domain = qnx; Type = ProductType; Identifier = com.apple.product-type.bundle.unit-test; - Class = PBXToolProductType; - Name = "Command-line Tool"; - Description = "Standalone command-line tool"; - DefaultTargetName = "Command-line Tool"; + BasedOn = com.apple.product-type.library.dynamic; DefaultBuildProperties = { - FULL_PRODUCT_NAME = "$(EXECUTABLE_NAME)"; - EXECUTABLE_PREFIX = ""; - EXECUTABLE_SUFFIX = ".xctest"; - REZ_EXECUTABLE = YES; - INSTALL_PATH = "/usr/local/bin"; - FRAMEWORK_FLAG_PREFIX = "-framework"; - LIBRARY_FLAG_PREFIX = "-l"; - LIBRARY_FLAG_NOSPACE = YES; - GCC_DYNAMIC_NO_PIC = NO; - LD_NO_PIE = NO; - GCC_SYMBOLS_PRIVATE_EXTERN = YES; - GCC_INLINES_ARE_PRIVATE_EXTERN = YES; - STRIP_STYLE = "all"; - CODE_SIGNING_ALLOWED = NO; - IsUnitTest = YES; - SWIFT_FORCE_DYNAMIC_LINK_STDLIB = YES; - SWIFT_FORCE_STATIC_LINK_STDLIB = NO; - // Avoid warning for executable types - ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = NO; + // Index store data is required to discover XCTest tests + COMPILER_INDEX_STORE_ENABLE = YES; + SWIFT_INDEX_STORE_ENABLE = YES; + // Testability is needed to generate code to invoke discovered XCTest tests + SWIFT_ENABLE_TESTABILITY = YES; }; - PackageTypes = ( - com.apple.package-type.mach-o-executable // default - ); }, // Dynamic library (masquerading as a framework to placate Swift's project structure) diff --git a/Sources/SWBTaskConstruction/ProductPlanning/ProductPlan.swift b/Sources/SWBTaskConstruction/ProductPlanning/ProductPlan.swift index 88d2edfe..883073b0 100644 --- a/Sources/SWBTaskConstruction/ProductPlanning/ProductPlan.swift +++ b/Sources/SWBTaskConstruction/ProductPlanning/ProductPlan.swift @@ -31,7 +31,7 @@ package protocol GlobalProductPlanDelegate: CoreClientTargetDiagnosticProducingD package final class GlobalProductPlan: GlobalTargetInfoProvider { /// The build plan request. - let planRequest: BuildPlanRequest + package let planRequest: BuildPlanRequest /// The target task info for each configured target. private(set) var targetTaskInfos: [ConfiguredTarget: TargetTaskInfo] diff --git a/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/InfoPlistTaskProducer.swift b/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/InfoPlistTaskProducer.swift index 1f740497..820a55f5 100644 --- a/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/InfoPlistTaskProducer.swift +++ b/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/InfoPlistTaskProducer.swift @@ -57,7 +57,7 @@ private extension ProductTypeSpec break } - fatalError("unknown product type") + fatalError("unknown product type \(type(of: self))") } } diff --git a/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/SwiftStandardLibrariesTaskProducer.swift b/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/SwiftStandardLibrariesTaskProducer.swift index bba8b051..c74e1843 100644 --- a/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/SwiftStandardLibrariesTaskProducer.swift +++ b/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/SwiftStandardLibrariesTaskProducer.swift @@ -41,7 +41,7 @@ final class SwiftStandardLibrariesTaskProducer: PhasedTaskProducer, TaskProducer let buildingAnySwiftSourceFiles = (context.configuredTarget?.target as? BuildPhaseTarget)?.sourcesBuildPhase?.containsSwiftSources(context.workspaceContext.workspace, context, scope, context.filePathResolver) ?? false // Determine whether we want to embed swift libraries. - var shouldEmbedSwiftLibraries = (buildingAnySwiftSourceFiles && productType.supportsEmbeddingSwiftStandardLibraries) + var shouldEmbedSwiftLibraries = (buildingAnySwiftSourceFiles && productType.supportsEmbeddingSwiftStandardLibraries(producer: context)) // If ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES then we will override our earlier reasoning if the product is a wrapper. if !shouldEmbedSwiftLibraries && scope.evaluate(BuiltinMacros.ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES) { diff --git a/Sources/SWBTestSupport/TestWorkspaces.swift b/Sources/SWBTestSupport/TestWorkspaces.swift index c225eb2c..6837bcf7 100644 --- a/Sources/SWBTestSupport/TestWorkspaces.swift +++ b/Sources/SWBTestSupport/TestWorkspaces.swift @@ -924,6 +924,7 @@ package final class TestStandardTarget: TestInternalTarget, Sendable { case extensionKitExtension case xcodeExtension case unitTest + case swiftpmTestRunner case uiTest case multiDeviceUITest case systemExtension @@ -972,6 +973,8 @@ package final class TestStandardTarget: TestInternalTarget, Sendable { return "com.apple.product-type.xcode-extension" case .unitTest: return "com.apple.product-type.bundle.unit-test" + case .swiftpmTestRunner: + return "com.apple.product-type.tool.swiftpm-test-runner" case .uiTest: return "com.apple.product-type.bundle.ui-testing" case .multiDeviceUITest: @@ -1015,7 +1018,8 @@ package final class TestStandardTarget: TestInternalTarget, Sendable { .appClip: return "\(name).app" case .commandLineTool, - .hostBuildTool: + .hostBuildTool, + .swiftpmTestRunner: return "\(name)" case .framework, .staticFramework: diff --git a/Sources/SWBUniversalPlatform/Specs/ProductTypes.xcspec b/Sources/SWBUniversalPlatform/Specs/ProductTypes.xcspec index 0067050f..bd6cf9c8 100644 --- a/Sources/SWBUniversalPlatform/Specs/ProductTypes.xcspec +++ b/Sources/SWBUniversalPlatform/Specs/ProductTypes.xcspec @@ -312,4 +312,19 @@ IsUnitTest = YES; WantsBundleIdentifierEditing = NO; }, + // SwiftPM test runner + { Type = ProductType; + Identifier = com.apple.product-type.tool.swiftpm-test-runner; + BasedOn = com.apple.product-type.tool; + Name = "SwiftPM Unit Test Runner"; + Description = "SwiftPM Unit Test Runner"; + DefaultBuildProperties = { + ENABLE_TESTING_SEARCH_PATHS = YES; + GENERATE_TEST_ENTRY_POINT = YES; + GENERATED_TEST_ENTRY_POINT_PATH = "$(DERIVED_SOURCES_DIR)/test_entry_point.swift"; + }; + PackageTypes = ( + com.apple.package-type.mach-o-executable + ); + }, ) diff --git a/Sources/SWBUniversalPlatform/TestEntryPointGenerationTaskAction.swift b/Sources/SWBUniversalPlatform/TestEntryPointGenerationTaskAction.swift index 57dff473..9765719a 100644 --- a/Sources/SWBUniversalPlatform/TestEntryPointGenerationTaskAction.swift +++ b/Sources/SWBUniversalPlatform/TestEntryPointGenerationTaskAction.swift @@ -23,10 +23,37 @@ class TestEntryPointGenerationTaskAction: TaskAction { override func performTaskAction(_ task: any ExecutableTask, dynamicExecutionDelegate: any DynamicTaskExecutionDelegate, executionDelegate: any TaskExecutionDelegate, clientDelegate: any TaskExecutionClientDelegate, outputDelegate: any TaskOutputDelegate) async -> CommandResult { do { let options = try Options.parse(Array(task.commandLineAsStrings.dropFirst())) - try executionDelegate.fs.write(options.output, contents: #""" + + var tests: [IndexStore.TestCaseClass] = [] + var objects: [Path] = [] + for linkerFilelist in options.linkerFilelist { + let filelistContents = String(String(decoding: try executionDelegate.fs.read(linkerFilelist), as: UTF8.self)) + let entries = filelistContents.split(separator: "\n", omittingEmptySubsequences: true).map { Path($0) }.map { + for indexUnitBasePath in options.indexUnitBasePath { + if let remappedPath = generateIndexOutputPath(from: $0, basePath: indexUnitBasePath) { + return remappedPath + } + } + return $0 + } + objects.append(contentsOf: entries) + } + let indexStoreAPI = try IndexStoreAPI(dylib: options.indexStoreLibraryPath) + for indexStore in options.indexStore { + let store = try IndexStore.open(store: indexStore, api: indexStoreAPI) + let testInfo = try store.listTests(in: objects) + tests.append(contentsOf: testInfo) + } + + try executionDelegate.fs.write(options.output, contents: ByteString(encodingAsUTF8: """ #if canImport(Testing) import Testing #endif + + \(testObservationFragment) + + import XCTest + \(discoveredTestsFragment(tests: tests)) @main @available(macOS 10.15, iOS 11, watchOS 4, tvOS 11, visionOS 1, *) @@ -44,6 +71,16 @@ class TestEntryPointGenerationTaskAction: TaskAction { return "xctest" } + private static func testOutputPath() -> String? { + var iterator = CommandLine.arguments.makeIterator() + while let argument = iterator.next() { + if argument == "--testing-output-path", let outputPath = iterator.next() { + return outputPath + } + } + return nil + } + #if os(Linux) @_silgen_name("$ss13_runAsyncMainyyyyYaKcF") private static func _runAsyncMain(_ asyncFun: @Sendable @escaping () async throws -> ()) @@ -57,6 +94,16 @@ class TestEntryPointGenerationTaskAction: TaskAction { } } #endif + if testingLibrary == "xctest" { + #if !os(Windows) && \(options.enableExperimentalTestOutput) + _ = Self.testOutputPath().map { SwiftPMXCTestObserver(testOutputPath: testOutputPath) } + #endif + #if os(WASI) + await XCTMain(__allDiscoveredTests()) as Never + #else + XCTMain(__allDiscoveredTests()) as Never + #endif + } } #else static func main() async { @@ -66,18 +113,564 @@ class TestEntryPointGenerationTaskAction: TaskAction { await Testing.__swiftPMEntryPoint() as Never } #endif + if testingLibrary == "xctest" { + #if !os(Windows) && \(options.enableExperimentalTestOutput) + _ = Self.testOutputPath().map { SwiftPMXCTestObserver(testOutputPath: testOutputPath) } + #endif + #if os(WASI) + await XCTMain(__allDiscoveredTests()) as Never + #else + XCTMain(__allDiscoveredTests()) as Never + #endif + } } #endif } - """#) + """)) + return .succeeded } catch { outputDelegate.emitError("\(error)") return .failed } } -} -private struct Options: ParsableArguments { - @Option var output: Path + private struct Options: ParsableArguments { + @Option var output: Path + @Option var indexStoreLibraryPath: Path + @Option var linkerFilelist: [Path] + @Option var indexStore: [Path] + @Option var indexUnitBasePath: [Path] + @Flag var enableExperimentalTestOutput: Bool = false + } + + private func discoveredTestsFragment(tests: [IndexStore.TestCaseClass]) -> String { + var fragment = "" + for moduleName in Set(tests.map { $0.module }).sorted() { + fragment += "@testable import \(moduleName)\n" + } + fragment += """ + @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings") + public func __allDiscoveredTests() -> [XCTestCaseEntry] { + return [ + + """ + for testClass in tests { + + let testTuples = testClass.testMethods.map { method in + let basename = method.name.hasSuffix("()") ? String(method.name.dropLast(2)) : method.name + if method.isAsync { + return " (\"\(basename)\", asyncTest(\(testClass.name).\(basename)))" + } else { + return " (\"\(basename)\", \(testClass.name).\(basename))" + } + } + fragment += " testCase([\(testTuples.joined(separator: ",\n"))]),\n" + } + fragment += """ + ] + } + """ + return fragment + } + + private var testObservationFragment: String = + """ + #if !os(Windows) // Test observation is not supported on Windows + import Foundation + import XCTest + + public final class SwiftPMXCTestObserver: NSObject { + let testOutputPath: String + + public init(testOutputPath: String) { + self.testOutputPath = testOutputPath + super.init() + XCTestObservationCenter.shared.addTestObserver(self) + } + } + + extension SwiftPMXCTestObserver: XCTestObservation { + private func write(record: any Encodable) { + let lock = FileLock(at: URL(fileURLWithPath: self.testOutputPath + ".lock")) + _ = try? lock.withLock { + self._write(record: record) + } + } + + private func _write(record: any Encodable) { + if let data = try? JSONEncoder().encode(record) { + if let fileHandle = FileHandle(forWritingAtPath: self.testOutputPath) { + defer { fileHandle.closeFile() } + fileHandle.seekToEndOfFile() + fileHandle.write("\\n".data(using: .utf8)!) + fileHandle.write(data) + } else { + _ = try? data.write(to: URL(fileURLWithPath: self.testOutputPath)) + } + } + } + + public func testBundleWillStart(_ testBundle: Bundle) { + let record = TestBundleEventRecord(bundle: .init(testBundle), event: .start) + write(record: TestEventRecord(bundleEvent: record)) + } + + public func testSuiteWillStart(_ testSuite: XCTestSuite) { + let record = TestSuiteEventRecord(suite: .init(testSuite), event: .start) + write(record: TestEventRecord(suiteEvent: record)) + } + + public func testCaseWillStart(_ testCase: XCTestCase) { + let record = TestCaseEventRecord(testCase: .init(testCase), event: .start) + write(record: TestEventRecord(caseEvent: record)) + } + + #if canImport(Darwin) + public func testCase(_ testCase: XCTestCase, didRecord issue: XCTIssue) { + let record = TestCaseFailureRecord(testCase: .init(testCase), issue: .init(issue), failureKind: .unexpected) + write(record: TestEventRecord(caseFailure: record)) + } + + public func testCase(_ testCase: XCTestCase, didRecord expectedFailure: XCTExpectedFailure) { + let record = TestCaseFailureRecord(testCase: .init(testCase), issue: .init(expectedFailure.issue), failureKind: .expected(failureReason: expectedFailure.failureReason)) + write(record: TestEventRecord(caseFailure: record)) + } + #else + public func testCase(_ testCase: XCTestCase, didFailWithDescription description: String, inFile filePath: String?, atLine lineNumber: Int) { + let issue = TestIssue(description: description, inFile: filePath, atLine: lineNumber) + let record = TestCaseFailureRecord(testCase: .init(testCase), issue: issue, failureKind: .unexpected) + write(record: TestEventRecord(caseFailure: record)) + } + #endif + + public func testCaseDidFinish(_ testCase: XCTestCase) { + let record = TestCaseEventRecord(testCase: .init(testCase), event: .finish) + write(record: TestEventRecord(caseEvent: record)) + } + + #if canImport(Darwin) + public func testSuite(_ testSuite: XCTestSuite, didRecord issue: XCTIssue) { + let record = TestSuiteFailureRecord(suite: .init(testSuite), issue: .init(issue), failureKind: .unexpected) + write(record: TestEventRecord(suiteFailure: record)) + } + + public func testSuite(_ testSuite: XCTestSuite, didRecord expectedFailure: XCTExpectedFailure) { + let record = TestSuiteFailureRecord(suite: .init(testSuite), issue: .init(expectedFailure.issue), failureKind: .expected(failureReason: expectedFailure.failureReason)) + write(record: TestEventRecord(suiteFailure: record)) + } + #else + public func testSuite(_ testSuite: XCTestSuite, didFailWithDescription description: String, inFile filePath: String?, atLine lineNumber: Int) { + let issue = TestIssue(description: description, inFile: filePath, atLine: lineNumber) + let record = TestSuiteFailureRecord(suite: .init(testSuite), issue: issue, failureKind: .unexpected) + write(record: TestEventRecord(suiteFailure: record)) + } + #endif + + public func testSuiteDidFinish(_ testSuite: XCTestSuite) { + let record = TestSuiteEventRecord(suite: .init(testSuite), event: .finish) + write(record: TestEventRecord(suiteEvent: record)) + } + + public func testBundleDidFinish(_ testBundle: Bundle) { + let record = TestBundleEventRecord(bundle: .init(testBundle), event: .finish) + write(record: TestEventRecord(bundleEvent: record)) + } + } + + // FIXME: Copied from `Lock.swift` in TSCBasic, would be nice if we had a better way + + #if canImport(Glibc) + @_exported import Glibc + #elseif canImport(Musl) + @_exported import Musl + #elseif os(Windows) + @_exported import CRT + @_exported import WinSDK + #elseif os(WASI) + @_exported import WASILibc + #elseif canImport(Android) + @_exported import Android + #else + @_exported import Darwin.C + #endif + + import Foundation + + public final class FileLock { + #if os(Windows) + private var handle: HANDLE? + #else + private var fileDescriptor: CInt? + #endif + + private let lockFile: URL + + public init(at lockFile: URL) { + self.lockFile = lockFile + } + + public func lock() throws { + #if os(Windows) + if handle == nil { + let h: HANDLE = lockFile.path.withCString(encodedAs: UTF16.self, { + CreateFileW( + $0, + UInt32(GENERIC_READ) | UInt32(GENERIC_WRITE), + UInt32(FILE_SHARE_READ) | UInt32(FILE_SHARE_WRITE), + nil, + DWORD(OPEN_ALWAYS), + DWORD(FILE_ATTRIBUTE_NORMAL), + nil + ) + }) + if h == INVALID_HANDLE_VALUE { + throw FileSystemError(errno: Int32(GetLastError()), lockFile) + } + self.handle = h + } + var overlapped = OVERLAPPED() + overlapped.Offset = 0 + overlapped.OffsetHigh = 0 + overlapped.hEvent = nil + if !LockFileEx(handle, DWORD(LOCKFILE_EXCLUSIVE_LOCK), 0, + UInt32.max, UInt32.max, &overlapped) { + throw ProcessLockError.unableToAquireLock(errno: Int32(GetLastError())) + } + #elseif os(WASI) + // WASI doesn't support flock + #else + if fileDescriptor == nil { + let fd = open(lockFile.path, O_WRONLY | O_CREAT | O_CLOEXEC, 0o666) + if fd == -1 { + fatalError("errno: \\(errno), lockFile: \\(lockFile)") + } + self.fileDescriptor = fd + } + while true { + if flock(fileDescriptor!, LOCK_EX) == 0 { + break + } + if errno == EINTR { continue } + fatalError("unable to acquire lock, errno: \\(errno)") + } + #endif + } + + public func unlock() { + #if os(Windows) + var overlapped = OVERLAPPED() + overlapped.Offset = 0 + overlapped.OffsetHigh = 0 + overlapped.hEvent = nil + UnlockFileEx(handle, 0, UInt32.max, UInt32.max, &overlapped) + #elseif os(WASI) + // WASI doesn't support flock + #else + guard let fd = fileDescriptor else { return } + flock(fd, LOCK_UN) + #endif + } + + deinit { + #if os(Windows) + guard let handle = handle else { return } + CloseHandle(handle) + #elseif os(WASI) + // WASI doesn't support flock + #else + guard let fd = fileDescriptor else { return } + close(fd) + #endif + } + + public func withLock(_ body: () throws -> T) throws -> T { + try lock() + defer { unlock() } + return try body() + } + + public func withLock(_ body: () async throws -> T) async throws -> T { + try lock() + defer { unlock() } + return try await body() + } + } + + // FIXME: Copied from `XCTEvents.swift`, would be nice if we had a better way + + struct TestEventRecord: Codable { + let caseFailure: TestCaseFailureRecord? + let suiteFailure: TestSuiteFailureRecord? + + let bundleEvent: TestBundleEventRecord? + let suiteEvent: TestSuiteEventRecord? + let caseEvent: TestCaseEventRecord? + + init( + caseFailure: TestCaseFailureRecord? = nil, + suiteFailure: TestSuiteFailureRecord? = nil, + bundleEvent: TestBundleEventRecord? = nil, + suiteEvent: TestSuiteEventRecord? = nil, + caseEvent: TestCaseEventRecord? = nil + ) { + self.caseFailure = caseFailure + self.suiteFailure = suiteFailure + self.bundleEvent = bundleEvent + self.suiteEvent = suiteEvent + self.caseEvent = caseEvent + } + } + + // MARK: - Records + + struct TestAttachment: Codable { + let name: String? + // TODO: Handle `userInfo: [AnyHashable : Any]?` + let uniformTypeIdentifier: String + let payload: Data? + } + + struct TestBundleEventRecord: Codable { + let bundle: TestBundle + let event: TestEvent + } + + struct TestCaseEventRecord: Codable { + let testCase: TestCase + let event: TestEvent + } + + struct TestCaseFailureRecord: Codable, CustomStringConvertible { + let testCase: TestCase + let issue: TestIssue + let failureKind: TestFailureKind + + var description: String { + return "\\(issue.sourceCodeContext.description)\\(testCase) \\(issue.compactDescription)" + } + } + + struct TestSuiteEventRecord: Codable { + let suite: TestSuiteRecord + let event: TestEvent + } + + struct TestSuiteFailureRecord: Codable { + let suite: TestSuiteRecord + let issue: TestIssue + let failureKind: TestFailureKind + } + + // MARK: Primitives + + struct TestBundle: Codable { + let bundleIdentifier: String? + let bundlePath: String + } + + struct TestCase: Codable { + let name: String + } + + struct TestErrorInfo: Codable { + let description: String + let type: String + } + + enum TestEvent: Codable { + case start + case finish + } + + enum TestFailureKind: Codable, Equatable { + case unexpected + case expected(failureReason: String?) + + var isExpected: Bool { + switch self { + case .expected: return true + case .unexpected: return false + } + } + } + + struct TestIssue: Codable { + let type: TestIssueType + let compactDescription: String + let detailedDescription: String? + let associatedError: TestErrorInfo? + let sourceCodeContext: TestSourceCodeContext + let attachments: [TestAttachment] + } + + enum TestIssueType: Codable { + case assertionFailure + case performanceRegression + case system + case thrownError + case uncaughtException + case unmatchedExpectedFailure + case unknown + } + + struct TestLocation: Codable, CustomStringConvertible { + let file: String + let line: Int + + var description: String { + return "\\(file):\\(line) " + } + } + + struct TestSourceCodeContext: Codable, CustomStringConvertible { + let callStack: [TestSourceCodeFrame] + let location: TestLocation? + + var description: String { + return location?.description ?? "" + } + } + + struct TestSourceCodeFrame: Codable { + let address: UInt64 + let symbolInfo: TestSourceCodeSymbolInfo? + let symbolicationError: TestErrorInfo? + } + + struct TestSourceCodeSymbolInfo: Codable { + let imageName: String + let symbolName: String + let location: TestLocation? + } + + struct TestSuiteRecord: Codable { + let name: String + } + + // MARK: XCTest compatibility + + extension TestIssue { + init(description: String, inFile filePath: String?, atLine lineNumber: Int) { + let location: TestLocation? + if let filePath = filePath { + location = .init(file: filePath, line: lineNumber) + } else { + location = nil + } + self.init(type: .assertionFailure, compactDescription: description, detailedDescription: description, associatedError: nil, sourceCodeContext: .init(callStack: [], location: location), attachments: []) + } + } + + import XCTest + + #if canImport(Darwin) // XCTAttachment is unavailable in swift-corelibs-xctest. + extension TestAttachment { + init(_ attachment: XCTAttachment) { + self.init( + name: attachment.name, + uniformTypeIdentifier: attachment.uniformTypeIdentifier, + payload: attachment.value(forKey: "payload") as? Data + ) + } + } + #endif + + extension TestBundle { + init(_ testBundle: Bundle) { + self.init( + bundleIdentifier: testBundle.bundleIdentifier, + bundlePath: testBundle.bundlePath + ) + } + } + + extension TestCase { + init(_ testCase: XCTestCase) { + self.init(name: testCase.name) + } + } + + extension TestErrorInfo { + init(_ error: any Swift.Error) { + self.init(description: "\\(error)", type: "\\(Swift.type(of: error))") + } + } + + #if canImport(Darwin) // XCTIssue is unavailable in swift-corelibs-xctest. + extension TestIssue { + init(_ issue: XCTIssue) { + self.init( + type: .init(issue.type), + compactDescription: issue.compactDescription, + detailedDescription: issue.detailedDescription, + associatedError: issue.associatedError.map { .init($0) }, + sourceCodeContext: .init(issue.sourceCodeContext), + attachments: issue.attachments.map { .init($0) } + ) + } + } + + extension TestIssueType { + init(_ type: XCTIssue.IssueType) { + switch type { + case .assertionFailure: self = .assertionFailure + case .thrownError: self = .thrownError + case .uncaughtException: self = .uncaughtException + case .performanceRegression: self = .performanceRegression + case .system: self = .system + case .unmatchedExpectedFailure: self = .unmatchedExpectedFailure + @unknown default: self = .unknown + } + } + } + #endif + + #if canImport(Darwin) // XCTSourceCodeLocation/XCTSourceCodeContext/XCTSourceCodeFrame/XCTSourceCodeSymbolInfo is unavailable in swift-corelibs-xctest. + extension TestLocation { + init(_ location: XCTSourceCodeLocation) { + self.init( + file: location.fileURL.absoluteString, + line: location.lineNumber + ) + } + } + + extension TestSourceCodeContext { + init(_ context: XCTSourceCodeContext) { + self.init( + callStack: context.callStack.map { .init($0) }, + location: context.location.map { .init($0) } + ) + } + } + + extension TestSourceCodeFrame { + init(_ frame: XCTSourceCodeFrame) { + self.init( + address: frame.address, + symbolInfo: (try? frame.symbolInfo()).map { .init($0) }, + symbolicationError: frame.symbolicationError.map { .init($0) } + ) + } + } + + extension TestSourceCodeSymbolInfo { + init(_ symbolInfo: XCTSourceCodeSymbolInfo) { + self.init( + imageName: symbolInfo.imageName, + symbolName: symbolInfo.symbolName, + location: symbolInfo.location.map { .init($0) } + ) + } + } + #endif + + extension TestSuiteRecord { + init(_ testSuite: XCTestSuite) { + self.init(name: testSuite.name) + } + } + #endif + """ } diff --git a/Sources/SWBUniversalPlatform/TestEntryPointGenerationTool.swift b/Sources/SWBUniversalPlatform/TestEntryPointGenerationTool.swift index 007611ad..5ee78857 100644 --- a/Sources/SWBUniversalPlatform/TestEntryPointGenerationTool.swift +++ b/Sources/SWBUniversalPlatform/TestEntryPointGenerationTool.swift @@ -17,7 +17,59 @@ import SWBCore final class TestEntryPointGenerationToolSpec: GenericCommandLineToolSpec, SpecIdentifierType, @unchecked Sendable { static let identifier = "org.swift.test-entry-point-generator" + override func commandLineFromTemplate(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, optionContext: (any DiscoveredCommandLineToolSpecInfo)?, specialArgs: [String] = [], lookup: ((MacroDeclaration) -> MacroExpression?)? = nil) -> [CommandLineArgument] { + var args = super.commandLineFromTemplate(cbc, delegate, optionContext: optionContext, specialArgs: specialArgs, lookup: lookup) + for (toolchainPath, toolchainLibrarySearchPath) in cbc.producer.toolchains.map({ ($0.path, $0.librarySearchPaths) }) { + if let path = toolchainLibrarySearchPath.findLibrary(operatingSystem: cbc.producer.hostOperatingSystem, basename: "IndexStore") { + args.append(contentsOf: ["--index-store-library-path", .path(path)]) + } + for input in cbc.inputs { + if input.fileType.conformsTo(identifier: "text") { + args.append(contentsOf: ["--linker-filelist", .path(input.absolutePath)]) + } else if input.fileType.conformsTo(identifier: "compiled.mach-o") { + // Do nothing + } else { + delegate.error("Unexpected input of type '\(input.fileType)' to test entry point generation") + } + } + } + return args + } + override func createTaskAction(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate) -> (any PlannedTaskAction)? { TestEntryPointGenerationTaskAction() } + + public func constructTasks(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, indexStorePaths: [Path], indexUnitBasePaths: [Path]) async { + var commandLine = commandLineFromTemplate(cbc, delegate, optionContext: nil) + + for indexStorePath in indexStorePaths { + commandLine.append(contentsOf: ["--index-store", .path(indexStorePath)]) + } + + for basePath in indexUnitBasePaths { + commandLine.append(contentsOf: ["--index-unit-base-path", .path(basePath)]) + } + + delegate.createTask( + type: self, + dependencyData: nil, + payload: nil, + ruleInfo: defaultRuleInfo(cbc, delegate), + additionalSignatureData: "", + commandLine: commandLine, + additionalOutput: [], + environment: environmentFromSpec(cbc, delegate), + workingDirectory: cbc.producer.defaultWorkingDirectory, + inputs: cbc.inputs.map { delegate.createNode($0.absolutePath) }, + outputs: cbc.outputs.map { delegate.createNode($0) }, + mustPrecede: [], + action: createTaskAction(cbc, delegate), + execDescription: resolveExecutionDescription(cbc, delegate), + preparesForIndexing: true, + enableSandboxing: enableSandboxing, + llbuildControlDisabled: true, + additionalTaskOrderingOptions: [] + ) + } } diff --git a/Sources/SWBUniversalPlatform/TestEntryPointTaskProducer.swift b/Sources/SWBUniversalPlatform/TestEntryPointTaskProducer.swift index fe4b56ef..23ec56f0 100644 --- a/Sources/SWBUniversalPlatform/TestEntryPointTaskProducer.swift +++ b/Sources/SWBUniversalPlatform/TestEntryPointTaskProducer.swift @@ -13,6 +13,7 @@ import SWBCore import SWBTaskConstruction import SWBMacro +import SWBUtil class TestEntryPointTaskProducer: PhasedTaskProducer, TaskProducer { func generateTasks() async -> [any PlannedTask] { @@ -21,8 +22,54 @@ class TestEntryPointTaskProducer: PhasedTaskProducer, TaskProducer { await self.appendGeneratedTasks(&tasks) { delegate in let scope = context.settings.globalScope let outputPath = scope.evaluate(BuiltinMacros.GENERATED_TEST_ENTRY_POINT_PATH) - let cbc = CommandBuildContext(producer: context, scope: scope, inputs: [], outputs: [outputPath]) - await context.testEntryPointGenerationToolSpec.constructTasks(cbc, delegate) + + guard let configuredTarget = context.configuredTarget else { + context.error("Cannot generate a test entry point without a target") + return + } + var indexStoreDirectories: OrderedSet = [] + var linkerFileLists: OrderedSet = [] + var indexUnitBasePaths: OrderedSet = [] + var binaryPaths: OrderedSet = [] + for directDependency in context.globalProductPlan.dependencies(of: configuredTarget) { + let settings = context.globalProductPlan.planRequest.buildRequestContext.getCachedSettings(directDependency.parameters, target: directDependency.target) + guard settings.productType?.conformsTo(identifier: "com.apple.product-type.bundle.unit-test") == true else { + continue + } + guard settings.globalScope.evaluate(BuiltinMacros.SWIFT_INDEX_STORE_ENABLE) else { + context.error("Cannot perform test discovery for '\(directDependency.target.name)' because index while building is disabled") + continue + } + let path = settings.globalScope.evaluate(BuiltinMacros.SWIFT_INDEX_STORE_PATH) + guard !path.isEmpty else { + continue + } + indexStoreDirectories.append(path) + + for arch in settings.globalScope.evaluate(BuiltinMacros.ARCHS) { + for variant in settings.globalScope.evaluate(BuiltinMacros.BUILD_VARIANTS) { + let innerScope = settings.globalScope + .subscope(binding: BuiltinMacros.archCondition, to: arch) + .subscope(binding: BuiltinMacros.variantCondition, to: variant) + let linkerFileListPath = innerScope.evaluate(BuiltinMacros.__INPUT_FILE_LIST_PATH__) + if !linkerFileListPath.isEmpty { + linkerFileLists.append(linkerFileListPath) + } + let objroot = innerScope.evaluate(BuiltinMacros.OBJROOT) + if !objroot.isEmpty { + indexUnitBasePaths.append(objroot) + } + + let binaryPath = innerScope.evaluate(BuiltinMacros.TARGET_BUILD_DIR).join(innerScope.evaluate(BuiltinMacros.EXECUTABLE_PATH)).normalize() + binaryPaths.append(binaryPath) + } + } + } + + let inputs: [FileToBuild] = linkerFileLists.map { FileToBuild(absolutePath: $0, fileType: self.context.workspaceContext.core.specRegistry.getSpec("text") as! FileTypeSpec) } + binaryPaths.map { FileToBuild(absolutePath: $0, fileType: self.context.workspaceContext.core.specRegistry.getSpec("compiled.mach-o") as! FileTypeSpec) } + + let cbc = CommandBuildContext(producer: context, scope: scope, inputs: inputs, outputs: [outputPath]) + await context.testEntryPointGenerationToolSpec.constructTasks(cbc, delegate, indexStorePaths: indexStoreDirectories.elements, indexUnitBasePaths: indexUnitBasePaths.elements) } } return tasks diff --git a/Sources/SWBUtil/CMakeLists.txt b/Sources/SWBUtil/CMakeLists.txt index 91348fd9..9d2d611d 100644 --- a/Sources/SWBUtil/CMakeLists.txt +++ b/Sources/SWBUtil/CMakeLists.txt @@ -46,6 +46,7 @@ add_library(SWBUtil HashContext.swift Headermap.swift HeavyCache.swift + IndexStore.swift Int.swift InterningArena.swift IO.swift diff --git a/Sources/SWBUtil/IndexStore.swift b/Sources/SWBUtil/IndexStore.swift new file mode 100644 index 00000000..46cbd8bc --- /dev/null +++ b/Sources/SWBUtil/IndexStore.swift @@ -0,0 +1,389 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SWBCSupport +import Foundation + +public final class IndexStore { + + public struct TestCaseClass { + public struct TestMethod: Hashable, Comparable { + public let name: String + public let isAsync: Bool + + public static func < (lhs: IndexStore.TestCaseClass.TestMethod, rhs: IndexStore.TestCaseClass.TestMethod) -> Bool { + return (lhs.name, (lhs.isAsync ? 1 : 0)) < (rhs.name, (rhs.isAsync ? 1 : 0)) + } + } + + public var name: String + public var module: String + public var testMethods: [TestMethod] + @available(*, deprecated, message: "use testMethods instead") public var methods: [String] + } + + fileprivate var impl: IndexStoreImpl { _impl as! IndexStoreImpl } + private let _impl: Any + + fileprivate init(_ impl: IndexStoreImpl) { + self._impl = impl + } + + static public func open(store path: Path, api: IndexStoreAPI) throws -> IndexStore { + let impl = try IndexStoreImpl.open(store: path, api: api.impl) + return IndexStore(impl) + } + + public func listTests(in objectFiles: [Path]) throws -> [TestCaseClass] { + return try impl.listTests(in: objectFiles) + } + + @available(*, deprecated, message: "use listTests(in:) instead") + public func listTests(inObjectFile object: Path) throws -> [TestCaseClass] { + return try impl.listTests(inObjectFile: object) + } +} + +public final class IndexStoreAPI { + fileprivate var impl: IndexStoreAPIImpl { + _impl as! IndexStoreAPIImpl + } + private let _impl: Any + + public init(dylib path: Path) throws { + self._impl = try IndexStoreAPIImpl(dylib: path) + } +} + +private final class IndexStoreImpl { + typealias TestCaseClass = IndexStore.TestCaseClass + + let api: IndexStoreAPIImpl + + let store: indexstore_t + + private init(store: indexstore_t, api: IndexStoreAPIImpl) { + self.store = store + self.api = api + } + + static public func open(store path: Path, api: IndexStoreAPIImpl) throws -> IndexStoreImpl { + if let store = try api.call({ api.fn.store_create(path.str, &$0) }) { + return IndexStoreImpl(store: store, api: api) + } + throw StubError.error("Unable to open store at \(path.str)") + } + + public func listTests(in objectFiles: [Path]) throws -> [TestCaseClass] { + var inheritance = [String: [String: String]]() + var testMethods = [String: [String: [(name: String, async: Bool)]]]() + + for objectFile in objectFiles { + // Get the records of this object file. + guard let unitReader = try? self.api.call ({ self.api.fn.unit_reader_create(store, unitName(object: objectFile), &$0) }) else { + continue + } + let records = try getRecords(unitReader: unitReader) + let moduleName = self.api.fn.unit_reader_get_module_name(unitReader).str + for record in records { + // get tests info + let testsInfo = try self.getTestsInfo(record: record) + // merge results across module + for (className, parentClassName) in testsInfo.inheritance { + inheritance[moduleName, default: [:]][className] = parentClassName + } + for (className, classTestMethods) in testsInfo.testMethods { + testMethods[moduleName, default: [:]][className, default: []].append(contentsOf: classTestMethods) + } + } + } + + // merge across inheritance in module boundries + func flatten(moduleName: String, className: String) -> [String: (name: String, async: Bool)] { + var allMethods = [String: (name: String, async: Bool)]() + + if let parentClassName = inheritance[moduleName]?[className] { + let parentMethods = flatten(moduleName: moduleName, className: parentClassName) + allMethods.merge(parentMethods, uniquingKeysWith: { (lhs, _) in lhs }) + } + + for method in testMethods[moduleName]?[className] ?? [] { + allMethods[method.name] = (name: method.name, async: method.async) + } + + return allMethods + } + + var testCaseClasses = [TestCaseClass]() + for (moduleName, classMethods) in testMethods { + for className in classMethods.keys { + let methods = flatten(moduleName: moduleName, className: className) + .map { (name, info) in TestCaseClass.TestMethod(name: name, isAsync: info.async) } + .sorted() + testCaseClasses.append(TestCaseClass(name: className, module: moduleName, testMethods: methods, methods: methods.map(\.name))) + } + } + + return testCaseClasses + } + + + @available(*, deprecated, message: "use listTests(in:) instead") + public func listTests(inObjectFile object: Path) throws -> [TestCaseClass] { + // Get the records of this object file. + let unitReader = try api.call{ self.api.fn.unit_reader_create(store, unitName(object: object), &$0) } + let records = try getRecords(unitReader: unitReader) + + // Get the test classes. + var inheritance = [String: String]() + var testMethods = [String: [(name: String, async: Bool)]]() + + for record in records { + let testsInfo = try self.getTestsInfo(record: record) + inheritance.merge(testsInfo.inheritance, uniquingKeysWith: { (lhs, _) in lhs }) + testMethods.merge(testsInfo.testMethods, uniquingKeysWith: { (lhs, _) in lhs }) + } + + func flatten(className: String) -> [(method: String, async: Bool)] { + var results = [(String, Bool)]() + if let parentClassName = inheritance[className] { + let parentMethods = flatten(className: parentClassName) + results.append(contentsOf: parentMethods) + } + if let methods = testMethods[className] { + results.append(contentsOf: methods) + } + return results + } + + let moduleName = self.api.fn.unit_reader_get_module_name(unitReader).str + + var testCaseClasses = [TestCaseClass]() + for className in testMethods.keys { + let methods = flatten(className: className) + .map { TestCaseClass.TestMethod(name: $0.method, isAsync: $0.async) } + .sorted() + testCaseClasses.append(TestCaseClass(name: className, module: moduleName, testMethods: methods, methods: methods.map(\.name))) + } + + return testCaseClasses + } + + private func getTestsInfo(record: String) throws -> (inheritance: [String: String], testMethods: [String: [(name: String, async: Bool)]] ) { + let recordReader = try api.call{ self.api.fn.record_reader_create(store, record, &$0) } + + // scan for inheritance + + let inheritanceStoreRef = StoreRef([String: String](), api: self.api) + let inheritancePointer = unsafeBitCast(Unmanaged.passUnretained(inheritanceStoreRef), to: UnsafeMutableRawPointer.self) + + _ = self.api.fn.record_reader_occurrences_apply_f(recordReader, inheritancePointer) { inheritancePointer , occ -> Bool in + let inheritanceStoreRef = Unmanaged>.fromOpaque(inheritancePointer!).takeUnretainedValue() + let fn = inheritanceStoreRef.api.fn + + // Get the symbol. + let sym = fn.occurrence_get_symbol(occ) + let symbolProperties = fn.symbol_get_properties(sym) + // We only care about symbols that are marked unit tests and are instance methods. + if symbolProperties & UInt64(INDEXSTORE_SYMBOL_PROPERTY_UNITTEST.rawValue) == 0 { + return true + } + if fn.symbol_get_kind(sym) != INDEXSTORE_SYMBOL_KIND_CLASS{ + return true + } + + let parentClassName = fn.symbol_get_name(sym).str + + let childClassNameStoreRef = StoreRef("", api: inheritanceStoreRef.api) + let childClassNamePointer = unsafeBitCast(Unmanaged.passUnretained(childClassNameStoreRef), to: UnsafeMutableRawPointer.self) + _ = fn.occurrence_relations_apply_f(occ!, childClassNamePointer) { childClassNamePointer, relation in + guard let relation = relation else { return true } + let childClassNameStoreRef = Unmanaged>.fromOpaque(childClassNamePointer!).takeUnretainedValue() + let fn = childClassNameStoreRef.api.fn + + // Look for the base class. + if fn.symbol_relation_get_roles(relation) != UInt64(INDEXSTORE_SYMBOL_ROLE_REL_BASEOF.rawValue) { + return true + } + + let childClassNameSym = fn.symbol_relation_get_symbol(relation) + childClassNameStoreRef.instance = fn.symbol_get_name(childClassNameSym).str + return true + } + + if !childClassNameStoreRef.instance.isEmpty { + inheritanceStoreRef.instance[childClassNameStoreRef.instance] = parentClassName + } + + return true + } + + // scan for methods + + let testMethodsStoreRef = StoreRef([String: [(name: String, async: Bool)]](), api: api) + let testMethodsPointer = unsafeBitCast(Unmanaged.passUnretained(testMethodsStoreRef), to: UnsafeMutableRawPointer.self) + + _ = self.api.fn.record_reader_occurrences_apply_f(recordReader, testMethodsPointer) { testMethodsPointer , occ -> Bool in + let testMethodsStoreRef = Unmanaged>.fromOpaque(testMethodsPointer!).takeUnretainedValue() + let fn = testMethodsStoreRef.api.fn + + // Get the symbol. + let sym = fn.occurrence_get_symbol(occ) + let symbolProperties = fn.symbol_get_properties(sym) + // We only care about symbols that are marked unit tests and are instance methods. + if symbolProperties & UInt64(INDEXSTORE_SYMBOL_PROPERTY_UNITTEST.rawValue) == 0 { + return true + } + if fn.symbol_get_kind(sym) != INDEXSTORE_SYMBOL_KIND_INSTANCEMETHOD { + return true + } + + let classNameStoreRef = StoreRef("", api: testMethodsStoreRef.api) + let classNamePointer = unsafeBitCast(Unmanaged.passUnretained(classNameStoreRef), to: UnsafeMutableRawPointer.self) + + _ = fn.occurrence_relations_apply_f(occ!, classNamePointer) { classNamePointer, relation in + guard let relation = relation else { return true } + let classNameStoreRef = Unmanaged>.fromOpaque(classNamePointer!).takeUnretainedValue() + let fn = classNameStoreRef.api.fn + + // Look for the class. + if fn.symbol_relation_get_roles(relation) != UInt64(INDEXSTORE_SYMBOL_ROLE_REL_CHILDOF.rawValue) { + return true + } + + let classNameSym = fn.symbol_relation_get_symbol(relation) + classNameStoreRef.instance = fn.symbol_get_name(classNameSym).str + return true + } + + if !classNameStoreRef.instance.isEmpty { + let methodName = fn.symbol_get_name(sym).str + let isAsync = symbolProperties & UInt64(INDEXSTORE_SYMBOL_PROPERTY_SWIFT_ASYNC.rawValue) != 0 + testMethodsStoreRef.instance[classNameStoreRef.instance, default: []].append((name: methodName, async: isAsync)) + } + + return true + } + + return ( + inheritance: inheritanceStoreRef.instance, + testMethods: testMethodsStoreRef.instance + ) + + } + + private func getRecords(unitReader: indexstore_unit_reader_t?) throws -> [String] { + let builder = StoreRef([String](), api: api) + + let ctx = unsafeBitCast(Unmanaged.passUnretained(builder), to: UnsafeMutableRawPointer.self) + _ = self.api.fn.unit_reader_dependencies_apply_f(unitReader, ctx) { ctx , unit -> Bool in + let store = Unmanaged>.fromOpaque(ctx!).takeUnretainedValue() + let fn = store.api.fn + if fn.unit_dependency_get_kind(unit) == INDEXSTORE_UNIT_DEPENDENCY_RECORD { + store.instance.append(fn.unit_dependency_get_name(unit).str) + } + return true + } + + return builder.instance + } + + private func unitName(object: Path) -> String { + let initialSize = 64 + var buf = UnsafeMutablePointer.allocate(capacity: initialSize) + let len = self.api.fn.store_get_unit_name_from_output_path(store, object.str, buf, initialSize) + + if len + 1 > initialSize { + buf.deallocate() + buf = UnsafeMutablePointer.allocate(capacity: len + 1) + _ = self.api.fn.store_get_unit_name_from_output_path(store, object.str, buf, len + 1) + } + + defer { + buf.deallocate() + } + + return String(cString: buf) + } +} + +private class StoreRef { + let api: IndexStoreAPIImpl + var instance: T + init(_ instance: T, api: IndexStoreAPIImpl) { + self.instance = instance + self.api = api + } +} + +private final class IndexStoreAPIImpl { + + /// The path of the index store dylib. + private let path: Path + + /// Handle of the dynamic library. + private let dylib: LibraryHandle + + /// The index store API functions. + fileprivate let fn: indexstore_functions_t + + fileprivate func call(_ fn: (inout indexstore_error_t?) -> T) throws -> T { + var error: indexstore_error_t? = nil + let ret = fn(&error) + + if let error = error { + if let desc = self.fn.error_get_description(error) { + throw StubError.error(String(cString: desc)) + } + throw StubError.error("Unable to get description for error: \(error)") + } + + return ret + } + + public init(dylib path: Path) throws { + self.path = path + self.dylib = try Library.open(path) + + var api = indexstore_functions_t() + api.store_create = Library.lookup(dylib, "indexstore_store_create") + api.store_get_unit_name_from_output_path = Library.lookup(dylib, "indexstore_store_get_unit_name_from_output_path") + api.unit_reader_create = Library.lookup(dylib, "indexstore_unit_reader_create") + api.error_get_description = Library.lookup(dylib, "indexstore_error_get_description") + api.unit_reader_dependencies_apply_f = Library.lookup(dylib, "indexstore_unit_reader_dependencies_apply_f") + api.unit_reader_get_module_name = Library.lookup(dylib, "indexstore_unit_reader_get_module_name") + api.unit_dependency_get_kind = Library.lookup(dylib, "indexstore_unit_dependency_get_kind") + api.unit_dependency_get_name = Library.lookup(dylib, "indexstore_unit_dependency_get_name") + api.record_reader_create = Library.lookup(dylib, "indexstore_record_reader_create") + api.symbol_get_name = Library.lookup(dylib, "indexstore_symbol_get_name") + api.symbol_get_properties = Library.lookup(dylib, "indexstore_symbol_get_properties") + api.symbol_get_kind = Library.lookup(dylib, "indexstore_symbol_get_kind") + api.record_reader_occurrences_apply_f = Library.lookup(dylib, "indexstore_record_reader_occurrences_apply_f") + api.occurrence_get_symbol = Library.lookup(dylib, "indexstore_occurrence_get_symbol") + api.occurrence_relations_apply_f = Library.lookup(dylib, "indexstore_occurrence_relations_apply_f") + api.symbol_relation_get_symbol = Library.lookup(dylib, "indexstore_symbol_relation_get_symbol") + api.symbol_relation_get_roles = Library.lookup(dylib, "indexstore_symbol_relation_get_roles") + + self.fn = api + } +} + +extension indexstore_string_ref_t { + fileprivate var str: String { + return String( + bytesNoCopy: UnsafeMutableRawPointer(mutating: data), + length: length, + encoding: .utf8, + freeWhenDone: false + )! + } +} diff --git a/Sources/SWBWindowsPlatform/Specs/Windows.xcspec b/Sources/SWBWindowsPlatform/Specs/Windows.xcspec index df09990f..9c58f4f4 100644 --- a/Sources/SWBWindowsPlatform/Specs/Windows.xcspec +++ b/Sources/SWBWindowsPlatform/Specs/Windows.xcspec @@ -42,6 +42,31 @@ BasedOn = com.apple.product-type.tool; }, + { + Domain = windows; + Type = ProductType; + Identifier = com.apple.product-type.bundle.unit-test; + BasedOn = com.apple.product-type.library.dynamic; + DefaultBuildProperties = { + // Index store data is required to discover XCTest tests + COMPILER_INDEX_STORE_ENABLE = YES; + SWIFT_INDEX_STORE_ENABLE = YES; + // Testability is needed to generate code to invoke discovered XCTest tests + SWIFT_ENABLE_TESTABILITY = YES; + }; + }, + + { + Domain = windows; + Type = ProductType; + Identifier = com.apple.product-type.tool.swiftpm-test-runner; + BasedOn = default:com.apple.product-type.tool.swiftpm-test-runner; + DefaultBuildProperties = { + EXECUTABLE_SUFFIX = ".$(EXECUTABLE_EXTENSION)"; + EXECUTABLE_EXTENSION = "exe"; + }; + }, + { Domain = windows; Type = ProductType; diff --git a/Sources/SwiftBuild/ProjectModel/Targets.swift b/Sources/SwiftBuild/ProjectModel/Targets.swift index 0f8a201a..efd4749c 100644 --- a/Sources/SwiftBuild/ProjectModel/Targets.swift +++ b/Sources/SwiftBuild/ProjectModel/Targets.swift @@ -316,6 +316,7 @@ extension ProjectModel { case executable = "com.apple.product-type.tool" case hostBuildTool = "com.apple.product-type.tool.host-build" case unitTest = "com.apple.product-type.bundle.unit-test" + case swiftpmTestRunner = "com.apple.product-type.tool.swiftpm-test-runner" case bundle = "com.apple.product-type.bundle" case packageProduct = "packageProduct" } diff --git a/Tests/SWBBuildSystemTests/BuildOperationTests.swift b/Tests/SWBBuildSystemTests/BuildOperationTests.swift index 70778e4a..f76b30b3 100644 --- a/Tests/SWBBuildSystemTests/BuildOperationTests.swift +++ b/Tests/SWBBuildSystemTests/BuildOperationTests.swift @@ -399,7 +399,7 @@ fileprivate struct BuildOperationTests: CoreBasedTests { @Test(.requireSDKs(.host), .skipHostOS(.macOS), .skipHostOS(.windows, "cannot find testing library"), .requireThreadSafeWorkingDirectory) func unitTestWithGeneratedEntryPoint() async throws { - try await withTemporaryDirectory { (tmpDir: Path) in + try await withTemporaryDirectory(removeTreeOnDeinit: false) { (tmpDir: Path) in let testProject = try await TestProject( "TestProject", sourceRoot: tmpDir, @@ -417,14 +417,32 @@ fileprivate struct BuildOperationTests: CoreBasedTests { "SDKROOT": "$(HOST_PLATFORM)", "SUPPORTED_PLATFORMS": "$(HOST_PLATFORM)", "SWIFT_VERSION": swiftVersion, + "INDEX_DATA_STORE_DIR": "\(tmpDir.join("index").str)", + "LINKER_DRIVER": "swiftc" ]) ], targets: [ TestStandardTarget( - "test", + "UnitTestRunner", + type: .swiftpmTestRunner, + buildConfigurations: [ + TestBuildConfiguration("Debug", + buildSettings: [:]), + ], + buildPhases: [ + TestSourcesBuildPhase(), + TestFrameworksBuildPhase([ + "MyTests.so" + ]) + ], + dependencies: ["MyTests"] + ), + TestStandardTarget( + "MyTests", type: .unitTest, buildConfigurations: [ TestBuildConfiguration("Debug", buildSettings: [ + "DYLIB_INSTALL_NAME_BASE": "$ORIGIN", "LD_RUNPATH_SEARCH_PATHS": "@loader_path/", ]) ], @@ -433,10 +451,10 @@ fileprivate struct BuildOperationTests: CoreBasedTests { TestFrameworksBuildPhase([ TestBuildFile(.target("library")), ]) - ], - dependencies: [ + ], dependencies: [ "library" - ] + ], + productReferenceName: "MyTests.so" ), TestStandardTarget( "library", @@ -444,6 +462,7 @@ fileprivate struct BuildOperationTests: CoreBasedTests { buildConfigurations: [ TestBuildConfiguration("Debug", buildSettings: [ "DYLIB_INSTALL_NAME_BASE": "$ORIGIN", + "LD_RUNPATH_SEARCH_PATHS": "@loader_path/", // FIXME: Find a way to make these default "EXECUTABLE_PREFIX": "lib", @@ -457,7 +476,7 @@ fileprivate struct BuildOperationTests: CoreBasedTests { ]) let core = try await getCore() let tester = try await BuildOperationTester(core, testProject, simulated: false) - + try localFS.createDirectory(tmpDir.join("index")) let projectDir = tester.workspace.projects[0].sourceRoot try await tester.fs.writeFileContents(projectDir.join("library.swift")) { stream in @@ -467,12 +486,19 @@ fileprivate struct BuildOperationTests: CoreBasedTests { try await tester.fs.writeFileContents(projectDir.join("test.swift")) { stream in stream <<< """ import Testing + import XCTest import library @Suite struct MySuite { - @Test func myTest() async throws { + @Test func myTest() { #expect(foo() == 42) } } + + final class MYXCTests: XCTestCase { + func testFoo() { + XCTAssertTrue(true) + } + } """ } @@ -483,13 +509,19 @@ fileprivate struct BuildOperationTests: CoreBasedTests { let toolchain = try #require(try await getCore().toolchainRegistry.defaultToolchain) let environment: Environment if destination.platform == "linux" { - environment = ["LD_LIBRARY_PATH": toolchain.path.join("usr/lib/swift/linux").str] + environment = ["LD_LIBRARY_PATH": "\(toolchain.path.join("usr/lib/swift/linux").str):\(projectDir.join("build").join("Debug\(destination.builtProductsDirSuffix)"))"] } else { environment = .init() } - let executionResult = try await Process.getOutput(url: URL(fileURLWithPath: projectDir.join("build").join("Debug\(destination.builtProductsDirSuffix)").join(core.hostOperatingSystem.imageFormat.executableName(basename: "test.xctest")).str), arguments: ["--testing-library", "swift-testing"], environment: environment) - #expect(String(decoding: executionResult.stderr, as: UTF8.self).contains("Test run started")) + do { + let executionResult = try await Process.getOutput(url: URL(fileURLWithPath: projectDir.join("build").join("Debug\(destination.builtProductsDirSuffix)").join(core.hostOperatingSystem.imageFormat.executableName(basename: "UnitTestRunner")).str), arguments: [], environment: environment) + #expect(String(decoding: executionResult.stdout, as: UTF8.self).contains("Executed 1 test, with 0 failures")) + } + do { + let executionResult = try await Process.getOutput(url: URL(fileURLWithPath: projectDir.join("build").join("Debug\(destination.builtProductsDirSuffix)").join(core.hostOperatingSystem.imageFormat.executableName(basename: "UnitTestRunner")).str), arguments: ["--testing-library", "swift-testing"], environment: environment) + #expect(String(decoding: executionResult.stderr, as: UTF8.self).contains("Test run with 1 test in 1 suite passed")) + } } } } diff --git a/Tests/SWBTaskConstructionTests/UnitTestTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/UnitTestTaskConstructionTests.swift index d40bc363..ce4515df 100644 --- a/Tests/SWBTaskConstructionTests/UnitTestTaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/UnitTestTaskConstructionTests.swift @@ -300,7 +300,7 @@ fileprivate struct UnitTestTaskConstructionTests: CoreBasedTests { } @Test(.requireSDKs(.linux)) - func unitTestTarget_linux() async throws { + func unitTestRunnerTarget_linux() async throws { let swiftCompilerPath = try await self.swiftCompilerPath let swiftVersion = try await self.swiftVersion let testProject = TestProject( @@ -319,9 +319,26 @@ fileprivate struct UnitTestTaskConstructionTests: CoreBasedTests { "PRODUCT_NAME": "$(TARGET_NAME)", "SDKROOT": "linux", "SWIFT_VERSION": swiftVersion, + "INDEX_DATA_STORE_DIR": "/index", + "LINKER_DRIVER": "swiftc" ]), ], targets: [ + TestStandardTarget( + "UnitTestRunner", + type: .swiftpmTestRunner, + buildConfigurations: [ + TestBuildConfiguration("Debug", + buildSettings: [:]), + ], + buildPhases: [ + TestSourcesBuildPhase(), + TestFrameworksBuildPhase([ + "UnitTestTarget.so" + ]) + ], + dependencies: ["UnitTestTarget"], + ), TestStandardTarget( "UnitTestTarget", type: .unitTest, @@ -335,7 +352,8 @@ fileprivate struct UnitTestTaskConstructionTests: CoreBasedTests { "TestTwo.swift", ]), ], - dependencies: [] + dependencies: [], + productReferenceName: "UnitTestTarget.so" ), ]) let core = try await getCore() @@ -346,14 +364,17 @@ fileprivate struct UnitTestTaskConstructionTests: CoreBasedTests { try await fs.writeFileContents(swiftCompilerPath) { $0 <<< "binary" } await tester.checkBuild(runDestination: .linux, fs: fs) { results in - results.checkTarget("UnitTestTarget") { target in + results.checkTarget("UnitTestRunner") { target in results.checkTask(.matchTarget(target), .matchRuleType("GenerateTestEntryPoint")) { task in - task.checkCommandLineMatches([.suffix("builtin-generateTestEntryPoint"), "--output", .suffix("test_entry_point.swift")]) + task.checkCommandLineMatches([.suffix("builtin-generateTestEntryPoint"), "--output", .suffix("test_entry_point.swift"), "--index-store-library-path", .suffix("libIndexStore.so"), "--linker-filelist", .suffix("UnitTestTarget.LinkFileList"), "--index-store", "/index", "--index-unit-base-path", "/tmp/Test/aProject/build"]) + task.checkInputs([ + .pathPattern(.suffix("UnitTestTarget.LinkFileList")), + .pathPattern(.suffix("UnitTestTarget.so")), + .namePattern(.any), + .namePattern(.any) + ]) task.checkOutputs([.pathPattern(.suffix("test_entry_point.swift"))]) } - results.checkTask(.matchTarget(target), .matchRuleType("SwiftDriver Compilation")) { task in - task.checkInputs(contain: [.pathPattern(.suffix("test_entry_point.swift"))]) - } } results.checkNoDiagnostics() From d435ca630055febead4987ff200650459ff88e38 Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Tue, 3 Jun 2025 13:42:21 -0700 Subject: [PATCH 19/54] Fix BuildRequest encoding of the recordBuildBacktraces key --- Sources/SWBProtocol/MessageSupport.swift | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Sources/SWBProtocol/MessageSupport.swift b/Sources/SWBProtocol/MessageSupport.swift index 19ccc2bf..44186a05 100644 --- a/Sources/SWBProtocol/MessageSupport.swift +++ b/Sources/SWBProtocol/MessageSupport.swift @@ -197,6 +197,7 @@ public struct BuildRequestMessagePayload: SerializableCodable, Equatable, Sendab case hideShellScriptEnvironment case useParallelTargets case useImplicitDependencies + case recordBuildBacktraces case generatePrecompiledModulesReport case useDryRun case showNonLoggedProgress @@ -219,6 +220,7 @@ public struct BuildRequestMessagePayload: SerializableCodable, Equatable, Sendab self.hideShellScriptEnvironment = try container.decode(Bool.self, forKey: BuildRequestMessagePayload.CodingKeys.hideShellScriptEnvironment) self.useParallelTargets = try container.decode(Bool.self, forKey: BuildRequestMessagePayload.CodingKeys.useParallelTargets) self.useImplicitDependencies = try container.decode(Bool.self, forKey: BuildRequestMessagePayload.CodingKeys.useImplicitDependencies) + self.recordBuildBacktraces = try container.decodeIfPresent(Bool.self, forKey: .recordBuildBacktraces) self.generatePrecompiledModulesReport = try container.decodeIfPresent(Bool.self, forKey: .generatePrecompiledModulesReport) self.useDryRun = try container.decode(Bool.self, forKey: BuildRequestMessagePayload.CodingKeys.useDryRun) self.showNonLoggedProgress = try container.decode(Bool.self, forKey: BuildRequestMessagePayload.CodingKeys.showNonLoggedProgress) @@ -242,6 +244,7 @@ public struct BuildRequestMessagePayload: SerializableCodable, Equatable, Sendab try container.encode(self.hideShellScriptEnvironment, forKey: BuildRequestMessagePayload.CodingKeys.hideShellScriptEnvironment) try container.encode(self.useParallelTargets, forKey: BuildRequestMessagePayload.CodingKeys.useParallelTargets) try container.encode(self.useImplicitDependencies, forKey: BuildRequestMessagePayload.CodingKeys.useImplicitDependencies) + try container.encodeIfPresent(self.recordBuildBacktraces, forKey: .recordBuildBacktraces) try container.encodeIfPresent(self.generatePrecompiledModulesReport, forKey: .generatePrecompiledModulesReport) try container.encode(self.useDryRun, forKey: BuildRequestMessagePayload.CodingKeys.useDryRun) try container.encode(self.showNonLoggedProgress, forKey: BuildRequestMessagePayload.CodingKeys.showNonLoggedProgress) From 7e6e3bdee364d41945ebd3188d750b0a5d1e901b Mon Sep 17 00:00:00 2001 From: Jake Petroules Date: Mon, 2 Jun 2025 10:56:53 -0700 Subject: [PATCH 20/54] Unskip tests blocked by requireThreadSafeWorkingDirectory Now that llbuild has fork/exec support, the tests can be enabled on Amazon Linux 2, OpenBSD, etc. --- .../AssetCatalogCompiler.swift | 2 +- .../SWBApplePlatform/CoreDataCompiler.swift | 2 +- Sources/SWBApplePlatform/CoreMLCompiler.swift | 2 +- .../SWBApplePlatform/IntentsCompiler.swift | 2 +- .../SWBApplePlatform/XCStringsCompiler.swift | 2 +- .../SWBBuildService/BuildDependencyInfo.swift | 2 +- .../ClientExchangeDelegate.swift | 2 +- Sources/SWBBuildSystem/BuildOperation.swift | 4 +- Sources/SWBCore/ProcessExecutionCache.swift | 12 +- .../CommandLineToolSpec.swift | 4 +- Sources/SWBCore/TaskGeneration.swift | 8 +- .../SWBProtocol/ClientExchangeMessages.swift | 4 +- .../TaskActions/ClangCompileTaskAction.swift | 2 +- .../TaskActions/CodeSignTaskAction.swift | 2 +- .../TaskActions/CopyTiffTaskAction.swift | 2 +- .../DeferredExecutionTaskAction.swift | 4 +- .../EmbedSwiftStdLibTaskAction.swift | 4 +- .../TaskActions/FileCopyTaskAction.swift | 2 +- .../GenericCachingTaskAction.swift | 2 +- .../TaskActions/LSRegisterURLTaskAction.swift | 2 +- .../PrecompileClangModuleTaskAction.swift | 2 +- .../SwiftDriverJobTaskAction.swift | 2 +- .../TaskActions/TaskAction.swift | 2 +- .../SWBTestSupport/BuildOperationTester.swift | 4 +- .../CapturingTaskGenerationDelegate.swift | 2 +- .../CommandLineToolSpecDiscoveredInfo.swift | 2 +- Sources/SWBTestSupport/CoreBasedTests.swift | 2 +- .../SWBTestSupport/LibraryGeneration.swift | 10 +- Sources/SWBTestSupport/Misc.swift | 10 +- Sources/SWBTestSupport/PerfTestSupport.swift | 6 +- .../SWBTestSupport/SkippedTestSupport.swift | 12 +- .../TaskPlanningTestSupport.swift | 4 +- Sources/SWBTestSupport/Xcode.swift | 2 +- Sources/SWBUtil/PbxCp.swift | 4 +- Sources/SWBUtil/Process.swift | 118 +++++++++++++----- Sources/SWBUtil/URL.swift | 9 +- .../SwiftBuild/SWBClientExchangeSupport.swift | 2 +- .../SWBAndroidPlatformTests.swift | 2 +- .../BuildCommandTests.swift | 17 ++- .../BuildOperationTests.swift | 26 ++-- .../BuildTaskBehaviorTests.swift | 16 +-- .../CustomTaskBuildOperationTests.swift | 2 +- Tests/SWBBuildSystemTests/LinkerTests.swift | 2 +- .../SwiftBuildTraceTests.swift | 3 +- .../CommandLineSpecPerfTests.swift | 2 +- .../ClangSerializedDiagnosticsTests.swift | 10 +- .../MessageSerializationTests.swift | 2 +- .../SWBQNXPlatformTests.swift | 2 +- .../BuildToolTaskConstructionTests.swift | 6 +- .../SWBTaskConstructionTests/ClangTests.swift | 2 +- .../EagerCompilationTests.swift | 2 +- .../InstallAPITaskConstructionTests.swift | 2 +- .../ModuleVerifierTaskConstructionTests.swift | 2 +- ...DemandResourcesTaskConstructionTests.swift | 2 +- .../PackageProductConstructionTests.swift | 2 +- .../PreviewsTaskConstructionTests.swift | 8 +- .../TaskConstructionTests.swift | 4 +- .../XCStringsTaskConstructionTests.swift | 2 +- .../InProcessTaskTestSupport.swift | 2 +- .../TaskTestSupport.swift | 4 +- Tests/SWBUtilTests/MachOTests.swift | 6 +- .../SWBWebAssemblyPlatformTests.swift | 3 +- .../BuildOperationPerfTests.swift | 2 +- .../SwiftBuildTests/BuildOperationTests.swift | 7 +- .../SwiftBuildTests/DocumentationTests.swift | 2 +- Tests/SwiftBuildTests/ServiceTests.swift | 2 +- Tests/SwiftBuildTests/ValidationTests.swift | 2 +- 67 files changed, 237 insertions(+), 168 deletions(-) diff --git a/Sources/SWBApplePlatform/AssetCatalogCompiler.swift b/Sources/SWBApplePlatform/AssetCatalogCompiler.swift index 5c589030..45bb476a 100644 --- a/Sources/SWBApplePlatform/AssetCatalogCompiler.swift +++ b/Sources/SWBApplePlatform/AssetCatalogCompiler.swift @@ -55,7 +55,7 @@ public final class ActoolCompilerSpec : GenericCompilerSpec, SpecIdentifierType, } private func assetTagCombinations(catalogInputs inputs: [FileToBuild], _ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate) async throws -> Set> { - return try await executeExternalTool(cbc, delegate, commandLine: [resolveExecutablePath(cbc, cbc.scope.actoolExecutablePath()).str, "--print-asset-tag-combinations", "--output-format", "xml1"] + inputs.map { $0.absolutePath.str }, workingDirectory: cbc.producer.defaultWorkingDirectory.str, environment: environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute asset tag combinations") { output in + return try await executeExternalTool(cbc, delegate, commandLine: [resolveExecutablePath(cbc, cbc.scope.actoolExecutablePath()).str, "--print-asset-tag-combinations", "--output-format", "xml1"] + inputs.map { $0.absolutePath.str }, workingDirectory: cbc.producer.defaultWorkingDirectory, environment: environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute asset tag combinations") { output in struct AssetCatalogToolOutput: Decodable { struct Diagnostic: Decodable { let description: String diff --git a/Sources/SWBApplePlatform/CoreDataCompiler.swift b/Sources/SWBApplePlatform/CoreDataCompiler.swift index 1a324cf8..9e3bc3ea 100644 --- a/Sources/SWBApplePlatform/CoreDataCompiler.swift +++ b/Sources/SWBApplePlatform/CoreDataCompiler.swift @@ -67,7 +67,7 @@ public final class CoreDataModelCompilerSpec : GenericCompilerSpec, SpecIdentifi // Mark the entire directory structure as being watched by the build system. delegate.access(path: input.absolutePath) - generatedFiles = try await generatedFilePaths(cbc, delegate, commandLine: [commandLine[0]] + ["--dry-run"] + commandLine[1...], workingDirectory: cbc.producer.defaultWorkingDirectory.str, environment: self.environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute data model \(input.absolutePath.basename) code generation output paths") { output in + generatedFiles = try await generatedFilePaths(cbc, delegate, commandLine: [commandLine[0]] + ["--dry-run"] + commandLine[1...], workingDirectory: cbc.producer.defaultWorkingDirectory, environment: self.environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute data model \(input.absolutePath.basename) code generation output paths") { output in return output.unsafeStringValue.split(separator: "\n").map(Path.init).map { $0.prependingPrivatePrefixIfNeeded(otherPath: outputDir) } } guard !generatedFiles.isEmpty else { diff --git a/Sources/SWBApplePlatform/CoreMLCompiler.swift b/Sources/SWBApplePlatform/CoreMLCompiler.swift index dfa14648..17225c0a 100644 --- a/Sources/SWBApplePlatform/CoreMLCompiler.swift +++ b/Sources/SWBApplePlatform/CoreMLCompiler.swift @@ -241,7 +241,7 @@ public final class CoreMLCompilerSpec : GenericCompilerSpec, SpecIdentifierType, // Mark the file as being watched by the build system to invalidate the build description. delegate.access(path: input.absolutePath) - generatedFiles = try await generatedFilePaths(cbc, delegate, commandLine: commandLine[0...3] + ["--dry-run", "yes"] + commandLine[4...], workingDirectory: cbc.producer.defaultWorkingDirectory.str, environment: self.environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute CoreML model \(input.absolutePath.basename) code generation output paths") { output in + generatedFiles = try await generatedFilePaths(cbc, delegate, commandLine: commandLine[0...3] + ["--dry-run", "yes"] + commandLine[4...], workingDirectory: cbc.producer.defaultWorkingDirectory, environment: self.environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute CoreML model \(input.absolutePath.basename) code generation output paths") { output in return output.unsafeStringValue.split(separator: "\n").map(Path.init) } guard !generatedFiles.isEmpty else { diff --git a/Sources/SWBApplePlatform/IntentsCompiler.swift b/Sources/SWBApplePlatform/IntentsCompiler.swift index e31cf5a4..6d879dd1 100644 --- a/Sources/SWBApplePlatform/IntentsCompiler.swift +++ b/Sources/SWBApplePlatform/IntentsCompiler.swift @@ -194,7 +194,7 @@ public final class IntentsCompilerSpec : GenericCompilerSpec, SpecIdentifierType // Mark the file as being watched by the build system to invalidate the build description. delegate.access(path: input.absolutePath) - generatedFiles = try await generatedFilePaths(cbc, delegate, commandLine: commandLine[0...1] + ["-dryRun"] + commandLine[2...], workingDirectory: cbc.producer.defaultWorkingDirectory.str, environment: self.environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute Intent Definition \(input.absolutePath.basename) code generation output paths") { output in + generatedFiles = try await generatedFilePaths(cbc, delegate, commandLine: commandLine[0...1] + ["-dryRun"] + commandLine[2...], workingDirectory: cbc.producer.defaultWorkingDirectory, environment: self.environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute Intent Definition \(input.absolutePath.basename) code generation output paths") { output in return output.unsafeStringValue.split(separator: "\n").map(Path.init).map { $0.prependingPrivatePrefixIfNeeded(otherPath: outputDir) } } guard !generatedFiles.isEmpty else { diff --git a/Sources/SWBApplePlatform/XCStringsCompiler.swift b/Sources/SWBApplePlatform/XCStringsCompiler.swift index 07f6dd8b..2a6eccd7 100644 --- a/Sources/SWBApplePlatform/XCStringsCompiler.swift +++ b/Sources/SWBApplePlatform/XCStringsCompiler.swift @@ -62,7 +62,7 @@ public final class XCStringsCompilerSpec: GenericCompilerSpec, SpecIdentifierTyp // xcstringstool compile --dry-run dryRunCommandLine.insert("--dry-run", at: 2) - outputs = try await generatedFilePaths(cbc, delegate, commandLine: dryRunCommandLine, workingDirectory: cbc.producer.defaultWorkingDirectory.str, environment: environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute XCStrings \(cbc.input.absolutePath.basename) output paths") { output in + outputs = try await generatedFilePaths(cbc, delegate, commandLine: dryRunCommandLine, workingDirectory: cbc.producer.defaultWorkingDirectory, environment: environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute XCStrings \(cbc.input.absolutePath.basename) output paths") { output in return output.unsafeStringValue.split(separator: "\n").map(Path.init) } } catch { diff --git a/Sources/SWBBuildService/BuildDependencyInfo.swift b/Sources/SWBBuildService/BuildDependencyInfo.swift index 223fa75f..09386ef3 100644 --- a/Sources/SWBBuildService/BuildDependencyInfo.swift +++ b/Sources/SWBBuildService/BuildDependencyInfo.swift @@ -469,7 +469,7 @@ extension BuildDependencyInfo { /// Special `CoreClientDelegate`-conforming struct because our use of `GlobalProductPlan` here should never be running external tools. fileprivate struct UnsupportedCoreClientDelegate: CoreClientDelegate { - func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { throw StubError.error("Running external tools is not supported when computing build dependency target info.") } } diff --git a/Sources/SWBBuildService/ClientExchangeDelegate.swift b/Sources/SWBBuildService/ClientExchangeDelegate.swift index 57ffc94e..b391870e 100644 --- a/Sources/SWBBuildService/ClientExchangeDelegate.swift +++ b/Sources/SWBBuildService/ClientExchangeDelegate.swift @@ -33,7 +33,7 @@ final class ClientExchangeDelegate: ClientDelegate { self.session = session } - func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { // Create a synchronous client exchange which the session uses to handle the response from the client, to make the communication synchronous from the point of view of our caller. let exchange = SynchronousClientExchange(session) diff --git a/Sources/SWBBuildSystem/BuildOperation.swift b/Sources/SWBBuildSystem/BuildOperation.swift index dc91a5f6..3f2afc02 100644 --- a/Sources/SWBBuildSystem/BuildOperation.swift +++ b/Sources/SWBBuildSystem/BuildOperation.swift @@ -1122,14 +1122,14 @@ private struct OperatorSystemAdaptorDynamicContext: DynamicTaskExecutionDelegate } @discardableResult - func spawn(commandLine: [String], environment: [String: String], workingDirectory: String, processDelegate: any ProcessDelegate) async throws -> Bool { + func spawn(commandLine: [String], environment: [String: String], workingDirectory: Path, processDelegate: any ProcessDelegate) async throws -> Bool { guard let jobContext else { throw StubError.error("API misuse. Spawning processes is only allowed from `performTaskAction`.") } // This calls into llb_buildsystem_command_interface_spawn, which can block, so ensure it's shunted to a new thread so as not to block the Swift Concurrency thread pool. This shouldn't risk thread explosion because this function is only allowed to be called from performTaskAction, which in turn should be bounded to ncores based on the number of active llbuild lane threads. return await _Concurrency.Task.detachNewThread(name: "llb_buildsystem_command_interface_spawn") { [commandInterface, jobContext, processDelegate] in - commandInterface.spawn(jobContext, commandLine: commandLine, environment: environment, workingDirectory: workingDirectory, processDelegate: processDelegate) + commandInterface.spawn(jobContext, commandLine: commandLine, environment: environment, workingDirectory: workingDirectory.str, processDelegate: processDelegate) } } diff --git a/Sources/SWBCore/ProcessExecutionCache.swift b/Sources/SWBCore/ProcessExecutionCache.swift index ef49b02a..1030561c 100644 --- a/Sources/SWBCore/ProcessExecutionCache.swift +++ b/Sources/SWBCore/ProcessExecutionCache.swift @@ -14,12 +14,20 @@ public import SWBUtil public final class ProcessExecutionCache: Sendable { private let cache = AsyncCache<[String], Processes.ExecutionResult>() + private let workingDirectory: Path? - public init() { } + public init(workingDirectory: Path? = .root) { + // FIXME: Work around lack of thread-safe working directory support in Foundation (Amazon Linux 2, OpenBSD). Executing processes in the current working directory is less deterministic, but all of the clients which use this class are generally not expected to be sensitive to the working directory anyways. This workaround can be removed once we drop support for Amazon Linux 2 and/or adopt swift-subprocess and/or Foundation.Process's working directory support is made thread safe. + if try! Process.hasUnsafeWorkingDirectorySupport { + self.workingDirectory = nil + return + } + self.workingDirectory = workingDirectory + } public func run(_ delegate: any CoreClientTargetDiagnosticProducingDelegate, _ commandLine: [String], executionDescription: String?) async throws -> Processes.ExecutionResult { try await cache.value(forKey: commandLine) { - try await delegate.executeExternalTool(commandLine: commandLine, workingDirectory: "/", environment: [:], executionDescription: executionDescription) + try await delegate.executeExternalTool(commandLine: commandLine, workingDirectory: workingDirectory, environment: [:], executionDescription: executionDescription) } } } diff --git a/Sources/SWBCore/SpecImplementations/CommandLineToolSpec.swift b/Sources/SWBCore/SpecImplementations/CommandLineToolSpec.swift index 866dd2f5..65072bc4 100644 --- a/Sources/SWBCore/SpecImplementations/CommandLineToolSpec.swift +++ b/Sources/SWBCore/SpecImplementations/CommandLineToolSpec.swift @@ -1393,7 +1393,7 @@ open class CommandLineToolSpec : PropertyDomainSpec, SpecType, TaskTypeDescripti return Base.instance.shouldStart(task, buildCommand: buildCommand) } - public func executeExternalTool(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, commandLine: [String], workingDirectory: String?, environment: [String: String], executionDescription: String?, _ parse: @escaping (ByteString) throws -> T) async throws -> T { + public func executeExternalTool(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, commandLine: [String], workingDirectory: Path?, environment: [String: String], executionDescription: String?, _ parse: @escaping (ByteString) throws -> T) async throws -> T { let executionResult = try await delegate.executeExternalTool(commandLine: commandLine, workingDirectory: workingDirectory, environment: environment, executionDescription: executionDescription) guard executionResult.exitStatus.isSuccess else { throw RunProcessNonZeroExitError(args: commandLine, workingDirectory: workingDirectory, environment: .init(environment), status: executionResult.exitStatus, stdout: ByteString(executionResult.stdout), stderr: ByteString(executionResult.stderr)) @@ -1401,7 +1401,7 @@ open class CommandLineToolSpec : PropertyDomainSpec, SpecType, TaskTypeDescripti return try parse(ByteString(executionResult.stdout)) } - public func generatedFilePaths(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, commandLine: [String], workingDirectory: String?, environment: [String: String], executionDescription: String?, _ parse: @escaping (ByteString) throws -> [Path]) async throws -> [Path] { + public func generatedFilePaths(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, commandLine: [String], workingDirectory: Path?, environment: [String: String], executionDescription: String?, _ parse: @escaping (ByteString) throws -> [Path]) async throws -> [Path] { return try await executeExternalTool(cbc, delegate, commandLine: commandLine, workingDirectory: workingDirectory, environment: environment, executionDescription: executionDescription, parse) } } diff --git a/Sources/SWBCore/TaskGeneration.swift b/Sources/SWBCore/TaskGeneration.swift index 44a2ff15..511db51e 100644 --- a/Sources/SWBCore/TaskGeneration.swift +++ b/Sources/SWBCore/TaskGeneration.swift @@ -647,7 +647,7 @@ public struct PlannedTaskBuilder { /// Interface by which core classes can request information from the client. public protocol CoreClientDelegate { - func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String: String]) async throws -> ExternalToolResult + func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String: String]) async throws -> ExternalToolResult } public protocol CoreClientTargetDiagnosticProducingDelegate: AnyObject, TargetDiagnosticProducingDelegate, ActivityReporter { @@ -757,7 +757,7 @@ extension TaskGenerationDelegate { } extension CoreClientTargetDiagnosticProducingDelegate { - public func executeExternalTool(commandLine: [String], workingDirectory: String? = nil, environment: [String: String] = [:], executionDescription: String?) async throws -> Processes.ExecutionResult { + public func executeExternalTool(commandLine: [String], workingDirectory: Path? = nil, environment: [String: String] = [:], executionDescription: String?) async throws -> Processes.ExecutionResult { try await withActivity(ruleInfo: "ExecuteExternalTool " + commandLine.joined(separator: " "), executionDescription: executionDescription ?? CommandLineToolSpec.fallbackExecutionDescription, signature: ByteString(encodingAsUTF8: "\(commandLine) \(String(describing: workingDirectory)) \(environment)"), target: nil, parentActivity: ActivityID.buildDescriptionActivity) { activity in try await coreClientDelegate.executeExternalTool(commandLine: commandLine, workingDirectory: workingDirectory, environment: environment) } @@ -768,7 +768,7 @@ extension CoreClientTargetDiagnosticProducingDelegate { private let externalToolExecutionQueue = AsyncOperationQueue(concurrentTasks: ProcessInfo.processInfo.activeProcessorCount) extension CoreClientDelegate { - package func executeExternalTool(commandLine: [String], workingDirectory: String? = nil, environment: [String: String] = [:]) async throws -> Processes.ExecutionResult { + package func executeExternalTool(commandLine: [String], workingDirectory: Path? = nil, environment: [String: String] = [:]) async throws -> Processes.ExecutionResult { switch try await executeExternalTool(commandLine: commandLine, workingDirectory: workingDirectory, environment: environment) { case .deferred: guard let url = commandLine.first.map(URL.init(fileURLWithPath:)) else { @@ -776,7 +776,7 @@ extension CoreClientDelegate { } return try await externalToolExecutionQueue.withOperation { - try await Process.getOutput(url: url, arguments: Array(commandLine.dropFirst()), currentDirectoryURL: workingDirectory.map(URL.init(fileURLWithPath:)), environment: Environment.current.addingContents(of: .init(environment))) + try await Process.getOutput(url: url, arguments: Array(commandLine.dropFirst()), currentDirectoryURL: workingDirectory.map { URL(fileURLWithPath: $0.str) }, environment: Environment.current.addingContents(of: .init(environment))) } case let .result(status, stdout, stderr): return Processes.ExecutionResult(exitStatus: status, stdout: stdout, stderr: stderr) diff --git a/Sources/SWBProtocol/ClientExchangeMessages.swift b/Sources/SWBProtocol/ClientExchangeMessages.swift index 12922e07..8ee73a58 100644 --- a/Sources/SWBProtocol/ClientExchangeMessages.swift +++ b/Sources/SWBProtocol/ClientExchangeMessages.swift @@ -21,10 +21,10 @@ public struct ExternalToolExecutionRequest: ClientExchangeMessage, Equatable { public let exchangeHandle: String public let commandLine: [String] - public let workingDirectory: String? + public let workingDirectory: Path? public let environment: [String: String] - public init(sessionHandle: String, exchangeHandle: String, commandLine: [String], workingDirectory: String?, environment: [String: String]) { + public init(sessionHandle: String, exchangeHandle: String, commandLine: [String], workingDirectory: Path?, environment: [String: String]) { self.sessionHandle = sessionHandle self.exchangeHandle = exchangeHandle self.commandLine = commandLine diff --git a/Sources/SWBTaskExecution/TaskActions/ClangCompileTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/ClangCompileTaskAction.swift index 5f26bde7..18caead1 100644 --- a/Sources/SWBTaskExecution/TaskActions/ClangCompileTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/ClangCompileTaskAction.swift @@ -266,7 +266,7 @@ public final class ClangCompileTaskAction: TaskAction, BuildValueValidatingTaskA let commandLine = command.arguments let delegate = TaskProcessDelegate(outputDelegate: outputDelegate) // The frontend invocations should be unaffected by the environment, pass an empty one. - try await spawn(commandLine: commandLine, environment: [:], workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: delegate) + try await spawn(commandLine: commandLine, environment: [:], workingDirectory: task.workingDirectory, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: delegate) lastResult = delegate.commandResult if lastResult == .succeeded { diff --git a/Sources/SWBTaskExecution/TaskActions/CodeSignTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/CodeSignTaskAction.swift index aeb49157..c059f6a8 100644 --- a/Sources/SWBTaskExecution/TaskActions/CodeSignTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/CodeSignTaskAction.swift @@ -37,7 +37,7 @@ public final class CodeSignTaskAction: TaskAction { commandLine.insert(preEncryptHashesFlag, at: 1) } - try await spawn(commandLine: commandLine, environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) + try await spawn(commandLine: commandLine, environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) } catch { outputDelegate.error(error.localizedDescription) return .failed diff --git a/Sources/SWBTaskExecution/TaskActions/CopyTiffTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/CopyTiffTaskAction.swift index cac4975e..df1675af 100644 --- a/Sources/SWBTaskExecution/TaskActions/CopyTiffTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/CopyTiffTaskAction.swift @@ -145,7 +145,7 @@ public final class CopyTiffTaskAction: TaskAction { let processDelegate = TaskProcessDelegate(outputDelegate: outputDelegate) do { - try await spawn(commandLine: tiffutilCommand, environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) + try await spawn(commandLine: tiffutilCommand, environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) } catch { outputDelegate.error(error.localizedDescription) return .failed diff --git a/Sources/SWBTaskExecution/TaskActions/DeferredExecutionTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/DeferredExecutionTaskAction.swift index a0307227..3af329c9 100644 --- a/Sources/SWBTaskExecution/TaskActions/DeferredExecutionTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/DeferredExecutionTaskAction.swift @@ -24,7 +24,7 @@ public final class DeferredExecutionTaskAction: TaskAction { public override func performTaskAction(_ task: any ExecutableTask, dynamicExecutionDelegate: any DynamicTaskExecutionDelegate, executionDelegate: any TaskExecutionDelegate, clientDelegate: any TaskExecutionClientDelegate, outputDelegate: any TaskOutputDelegate) async -> CommandResult { let processDelegate = TaskProcessDelegate(outputDelegate: outputDelegate) do { - try await spawn(commandLine: Array(task.commandLineAsStrings), environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) + try await spawn(commandLine: Array(task.commandLineAsStrings), environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) } catch { outputDelegate.error(error.localizedDescription) return .failed @@ -50,7 +50,7 @@ fileprivate extension CommandResult { } extension TaskAction { - func spawn(commandLine: [String], environment: [String: String], workingDirectory: String, dynamicExecutionDelegate: any DynamicTaskExecutionDelegate, clientDelegate: any TaskExecutionClientDelegate, processDelegate: any ProcessDelegate) async throws { + func spawn(commandLine: [String], environment: [String: String], workingDirectory: Path, dynamicExecutionDelegate: any DynamicTaskExecutionDelegate, clientDelegate: any TaskExecutionClientDelegate, processDelegate: any ProcessDelegate) async throws { guard dynamicExecutionDelegate.allowsExternalToolExecution else { try await dynamicExecutionDelegate.spawn(commandLine: commandLine, environment: environment, workingDirectory: workingDirectory, processDelegate: processDelegate) return diff --git a/Sources/SWBTaskExecution/TaskActions/EmbedSwiftStdLibTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/EmbedSwiftStdLibTaskAction.swift index f2ded2fd..4c995f5d 100644 --- a/Sources/SWBTaskExecution/TaskActions/EmbedSwiftStdLibTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/EmbedSwiftStdLibTaskAction.swift @@ -515,7 +515,7 @@ public final class EmbedSwiftStdLibTaskAction: TaskAction { let capturingDelegate = CapturingOutputDelegate(outputDelegate: outputDelegate) let processDelegate = TaskProcessDelegate(outputDelegate: capturingDelegate) - try await taskAction.spawn(commandLine: args, environment: effectiveEnvironment, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) + try await taskAction.spawn(commandLine: args, environment: effectiveEnvironment, workingDirectory: task.workingDirectory, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) if let error = processDelegate.executionError { throw StubError.error(error) } @@ -526,7 +526,7 @@ public final class EmbedSwiftStdLibTaskAction: TaskAction { } guard !failed else { - throw RunProcessNonZeroExitError(args: args, workingDirectory: task.workingDirectory.str, environment: .init(effectiveEnvironment), status: { + throw RunProcessNonZeroExitError(args: args, workingDirectory: task.workingDirectory, environment: .init(effectiveEnvironment), status: { if case let .exit(exitStatus, _) = processDelegate.outputDelegate.result { return exitStatus } diff --git a/Sources/SWBTaskExecution/TaskActions/FileCopyTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/FileCopyTaskAction.swift index 94b2899d..f3bbb3f6 100644 --- a/Sources/SWBTaskExecution/TaskActions/FileCopyTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/FileCopyTaskAction.swift @@ -132,7 +132,7 @@ public final class FileCopyTaskAction: TaskAction } for commandLine in commandLine.compileAndLink.flatMap({ [$0.compile, $0.link] }) + [commandLine.lipo] { - try await spawn(commandLine: commandLine, environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) + try await spawn(commandLine: commandLine, environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) } } } diff --git a/Sources/SWBTaskExecution/TaskActions/GenericCachingTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/GenericCachingTaskAction.swift index bb6291c2..466859d9 100644 --- a/Sources/SWBTaskExecution/TaskActions/GenericCachingTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/GenericCachingTaskAction.swift @@ -197,7 +197,7 @@ public final class GenericCachingTaskAction: TaskAction { } emitCacheDebuggingRemark("running sandboxed command") - try await spawn(commandLine: sandboxArgs + remappedCommandLine, environment: remappedEnvironment.bindingsDictionary, workingDirectory: cacheKey.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) + try await spawn(commandLine: sandboxArgs + remappedCommandLine, environment: remappedEnvironment.bindingsDictionary, workingDirectory: cacheKey.workingDirectory, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) if processDelegate.commandResult == .succeeded { try await withThrowingTaskGroup(of: Void.self) { group in diff --git a/Sources/SWBTaskExecution/TaskActions/LSRegisterURLTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/LSRegisterURLTaskAction.swift index 93f1d0c4..0a924145 100644 --- a/Sources/SWBTaskExecution/TaskActions/LSRegisterURLTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/LSRegisterURLTaskAction.swift @@ -23,7 +23,7 @@ public final class LSRegisterURLTaskAction: TaskAction { override public func performTaskAction(_ task: any ExecutableTask, dynamicExecutionDelegate: any DynamicTaskExecutionDelegate, executionDelegate: any TaskExecutionDelegate, clientDelegate: any TaskExecutionClientDelegate, outputDelegate: any TaskOutputDelegate) async -> CommandResult { let processDelegate = TaskProcessDelegate(outputDelegate: outputDelegate) do { - try await spawn(commandLine: Array(task.commandLineAsStrings), environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) + try await spawn(commandLine: Array(task.commandLineAsStrings), environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate) } catch { outputDelegate.error(error.localizedDescription) return .failed diff --git a/Sources/SWBTaskExecution/TaskActions/PrecompileClangModuleTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/PrecompileClangModuleTaskAction.swift index 3c308b9e..ebab37b8 100644 --- a/Sources/SWBTaskExecution/TaskActions/PrecompileClangModuleTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/PrecompileClangModuleTaskAction.swift @@ -198,7 +198,7 @@ final public class PrecompileClangModuleTaskAction: TaskAction, BuildValueValida let delegate = TaskProcessDelegate(outputDelegate: outputDelegate) // The frontend invocations should be unaffected by the environment, pass an empty one. - try await spawn(commandLine: commandLine, environment: [:], workingDirectory: dependencyInfo.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: delegate) + try await spawn(commandLine: commandLine, environment: [:], workingDirectory: dependencyInfo.workingDirectory, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: delegate) let result = delegate.commandResult ?? .failed if result == .succeeded { diff --git a/Sources/SWBTaskExecution/TaskActions/SwiftDriverJobTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/SwiftDriverJobTaskAction.swift index 08d57daa..84eee161 100644 --- a/Sources/SWBTaskExecution/TaskActions/SwiftDriverJobTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/SwiftDriverJobTaskAction.swift @@ -520,7 +520,7 @@ public final class SwiftDriverJobTaskAction: TaskAction, BuildValueValidatingTas return .succeeded } - try await spawn(commandLine: options.commandLine, environment: environment, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: delegate) + try await spawn(commandLine: options.commandLine, environment: environment, workingDirectory: task.workingDirectory, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: delegate) if let error = delegate.executionError { outputDelegate.error(error) diff --git a/Sources/SWBTaskExecution/TaskActions/TaskAction.swift b/Sources/SWBTaskExecution/TaskActions/TaskAction.swift index ba5e3f1d..6b7fd761 100644 --- a/Sources/SWBTaskExecution/TaskActions/TaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/TaskAction.swift @@ -229,7 +229,7 @@ public protocol DynamicTaskExecutionDelegate: ActivityReporter { func spawn( commandLine: [String], environment: [String: String], - workingDirectory: String, + workingDirectory: Path, processDelegate: any ProcessDelegate ) async throws -> Bool diff --git a/Sources/SWBTestSupport/BuildOperationTester.swift b/Sources/SWBTestSupport/BuildOperationTester.swift index fc090718..02c05f16 100644 --- a/Sources/SWBTestSupport/BuildOperationTester.swift +++ b/Sources/SWBTestSupport/BuildOperationTester.swift @@ -1680,7 +1680,7 @@ extension BuildOperationTester.BuildDescriptionResults: Sendable { } package final class MockTestClientDelegate: ClientDelegate, Sendable { package init() {} - package func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String: String]) async throws -> ExternalToolResult { + package func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String: String]) async throws -> ExternalToolResult { return .deferred } } @@ -1876,7 +1876,7 @@ private final class BuildOperationTesterDelegate: BuildOperationDelegate { if !self.hadErrors { switch result { case let .exit(exitStatus, _) where !exitStatus.isSuccess && !exitStatus.wasCanceled: - self.delegate.events.append(.buildHadDiagnostic(Diagnostic(behavior: .error, location: .unknown, data: DiagnosticData("Command \(task.ruleInfo[0]) failed. \(RunProcessNonZeroExitError(args: Array(task.commandLineAsStrings), workingDirectory: task.workingDirectory.str, environment: .init(task.environment.bindingsDictionary), status: exitStatus, mergedOutput: output).description)")))) + self.delegate.events.append(.buildHadDiagnostic(Diagnostic(behavior: .error, location: .unknown, data: DiagnosticData("Command \(task.ruleInfo[0]) failed. \(RunProcessNonZeroExitError(args: Array(task.commandLineAsStrings), workingDirectory: task.workingDirectory, environment: .init(task.environment.bindingsDictionary), status: exitStatus, mergedOutput: output).description)")))) case .failedSetup: self.delegate.events.append(.buildHadDiagnostic(Diagnostic(behavior: .error, location: .unknown, data: DiagnosticData("Command \(task.ruleInfo[0]) failed setup.")))) case .exit, .skipped: diff --git a/Sources/SWBTestSupport/CapturingTaskGenerationDelegate.swift b/Sources/SWBTestSupport/CapturingTaskGenerationDelegate.swift index 516931d9..b4689254 100644 --- a/Sources/SWBTestSupport/CapturingTaskGenerationDelegate.swift +++ b/Sources/SWBTestSupport/CapturingTaskGenerationDelegate.swift @@ -106,7 +106,7 @@ package class CapturingTaskGenerationDelegate: TaskGenerationDelegate, CoreClien package func fileExists(at path: Path) -> Bool { return true } package var taskActionCreationDelegate: any TaskActionCreationDelegate { return self } package var clientDelegate: any CoreClientDelegate { return self } - package func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String: String]) async throws -> ExternalToolResult { + package func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String: String]) async throws -> ExternalToolResult { return .deferred } } diff --git a/Sources/SWBTestSupport/CommandLineToolSpecDiscoveredInfo.swift b/Sources/SWBTestSupport/CommandLineToolSpecDiscoveredInfo.swift index 3676dd9b..b35fb1d3 100644 --- a/Sources/SWBTestSupport/CommandLineToolSpecDiscoveredInfo.swift +++ b/Sources/SWBTestSupport/CommandLineToolSpecDiscoveredInfo.swift @@ -61,7 +61,7 @@ private class ToolSpecCapturingTaskGenerationDelegate: CapturingTaskGenerationDe try super.init(producer: producer, userPreferences: userPreferences) } - override func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String: String]) async throws -> ExternalToolResult { + override func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String: String]) async throws -> ExternalToolResult { result } } diff --git a/Sources/SWBTestSupport/CoreBasedTests.swift b/Sources/SWBTestSupport/CoreBasedTests.swift index 2cfbeb6f..7c3e6f42 100644 --- a/Sources/SWBTestSupport/CoreBasedTests.swift +++ b/Sources/SWBTestSupport/CoreBasedTests.swift @@ -365,7 +365,7 @@ private final class AlwaysDeferredCoreClientDelegate: CoreClientDelegate, CoreCl _diagnosticsEngine.hasErrors } - func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { .deferred } } diff --git a/Sources/SWBTestSupport/LibraryGeneration.swift b/Sources/SWBTestSupport/LibraryGeneration.swift index 02cc269d..1a238077 100644 --- a/Sources/SWBTestSupport/LibraryGeneration.swift +++ b/Sources/SWBTestSupport/LibraryGeneration.swift @@ -64,28 +64,28 @@ extension InstalledXcode { let linkerArgs = linkerOptions.map({ $0.args }).reduce([], +) if buildLibraryForDistribution { distributionArgs = ["-enable-library-evolution"] - _ = try await xcrun(["-sdk", platform.sdkName, "swiftc", "-target", target] + targetVariantArgs + distributionArgs as [String] + ["-emit-module-interface", "-emit-module-interface-path", swiftModuleDir.join("\(name).swiftinterface").str, "-c", sourcePath.str], workingDirectory: workingDirectory.str) + _ = try await xcrun(["-sdk", platform.sdkName, "swiftc", "-target", target] + targetVariantArgs + distributionArgs as [String] + ["-emit-module-interface", "-emit-module-interface-path", swiftModuleDir.join("\(name).swiftinterface").str, "-c", sourcePath.str], workingDirectory: workingDirectory) } else { distributionArgs = [] } // Generate the macho file. let machoArgs = [["-sdk", platform.sdkName, "swiftc", "-target", target], targetVariantArgs, distributionArgs, ["-emit-module", "-emit-module-path", swiftModuleDir.join("\(arch).swiftmodule").str, "-module-name", name, "-c", sourcePath.str]].reduce([], +) - _ = try await xcrun(machoArgs, workingDirectory: workingDirectory.str) + _ = try await xcrun(machoArgs, workingDirectory: workingDirectory) // Generate the swiftmodule file. let swiftmoduleArgs = [["-sdk", platform.sdkName, "swiftc", "-target", target], targetVariantArgs, distributionArgs, ["-module-name", name, "-o", objectPath.str, "-c", sourcePath.str]].reduce([], +) - _ = try await xcrun(swiftmoduleArgs, workingDirectory: workingDirectory.str) + _ = try await xcrun(swiftmoduleArgs, workingDirectory: workingDirectory) if `static` { return objectPath } else if object { let linkArgs = [["-sdk", platform.sdkName, "clang", "-r", "-target", target], targetVariantArgs, linkerArgs, ["-L" + swiftRuntimeLibraryDirectoryPath(name: platform.sdkName), "-L/usr/lib/swift", "-o", machoPath.str, objectPath.str]].reduce([], +) - _ = try await xcrun(linkArgs, workingDirectory: workingDirectory.str) + _ = try await xcrun(linkArgs, workingDirectory: workingDirectory) return machoPath } else { let linkArgs = [["-sdk", platform.sdkName, "clang", "-dynamiclib", "-target", target], targetVariantArgs, linkerArgs, ["-L" + swiftRuntimeLibraryDirectoryPath(name: platform.sdkName), "-L/usr/lib/swift", "-o", machoPath.str, objectPath.str]].reduce([], +) - _ = try await xcrun(linkArgs, workingDirectory: workingDirectory.str) + _ = try await xcrun(linkArgs, workingDirectory: workingDirectory) if needSigned { _ = try await runProcessWithDeveloperDirectory(["/usr/bin/codesign", "-s", "-", machoPath.str]) } diff --git a/Sources/SWBTestSupport/Misc.swift b/Sources/SWBTestSupport/Misc.swift index 85394475..9e77ad92 100644 --- a/Sources/SWBTestSupport/Misc.swift +++ b/Sources/SWBTestSupport/Misc.swift @@ -39,7 +39,7 @@ package extension Sequence where Element: Equatable { /// - throws: ``StubError`` if the arguments list is an empty array. /// - throws: ``RunProcessNonZeroExitError`` if the process exited with a nonzero status code or uncaught signal. @discardableResult -package func runProcess(_ args: [String], workingDirectory: String? = nil, environment: Environment = .init(), interruptible: Bool = true, redirectStderr: Bool = false) async throws -> String { +package func runProcess(_ args: [String], workingDirectory: Path? = nil, environment: Environment = .init(), interruptible: Bool = true, redirectStderr: Bool = false) async throws -> String { guard let first = args.first else { throw StubError.error("Invalid number of arguments") } @@ -48,13 +48,13 @@ package func runProcess(_ args: [String], workingDirectory: String? = nil, envir } let arguments = Array(args.dropFirst()) if redirectStderr { - let (exitStatus, output) = try await Process.getMergedOutput(url: url, arguments: arguments, currentDirectoryURL: workingDirectory.map(URL.init(fileURLWithPath:)), environment: environment, interruptible: interruptible) + let (exitStatus, output) = try await Process.getMergedOutput(url: url, arguments: arguments, currentDirectoryURL: workingDirectory.map { URL(fileURLWithPath: $0.str) }, environment: environment, interruptible: interruptible) guard exitStatus.isSuccess else { throw RunProcessNonZeroExitError(args: args, workingDirectory: workingDirectory, environment: environment, status: exitStatus, mergedOutput: ByteString(output)) } return String(decoding: output, as: UTF8.self) } else { - let executionResult = try await Process.getOutput(url: url, arguments: arguments, currentDirectoryURL: workingDirectory.map(URL.init(fileURLWithPath:)), environment: environment, interruptible: interruptible) + let executionResult = try await Process.getOutput(url: url, arguments: arguments, currentDirectoryURL: workingDirectory.map { URL(fileURLWithPath: $0.str) }, environment: environment, interruptible: interruptible) guard executionResult.exitStatus.isSuccess else { throw RunProcessNonZeroExitError(args: args, workingDirectory: workingDirectory, environment: environment, status: executionResult.exitStatus, stdout: ByteString(executionResult.stdout), stderr: ByteString(executionResult.stderr)) } @@ -65,14 +65,14 @@ package func runProcess(_ args: [String], workingDirectory: String? = nil, envir /// Runs the command specified by `args` with the `DEVELOPER_DIR` environment variable set. /// /// This method will use the current value of `DEVELOPER_DIR` in the environment by default, or the value of `overrideDeveloperDirectory` if specified. -package func runProcessWithDeveloperDirectory(_ args: [String], workingDirectory: String? = nil, overrideDeveloperDirectory: String? = nil, interruptible: Bool = true, redirectStderr: Bool = true) async throws -> String { +package func runProcessWithDeveloperDirectory(_ args: [String], workingDirectory: Path? = nil, overrideDeveloperDirectory: String? = nil, interruptible: Bool = true, redirectStderr: Bool = true) async throws -> String { let environment = Environment.current .filter(keys: ["DEVELOPER_DIR", "LLVM_PROFILE_FILE"]) .addingContents(of: overrideDeveloperDirectory.map { Environment(["DEVELOPER_DIR": $0]) } ?? .init()) return try await runProcess(args, workingDirectory: workingDirectory, environment: environment, interruptible: interruptible, redirectStderr: redirectStderr) } -package func runHostProcess(_ args: [String], workingDirectory: String? = nil, interruptible: Bool = true, redirectStderr: Bool = true) async throws -> String { +package func runHostProcess(_ args: [String], workingDirectory: Path? = nil, interruptible: Bool = true, redirectStderr: Bool = true) async throws -> String { switch try ProcessInfo.processInfo.hostOperatingSystem() { case .macOS: return try await InstalledXcode.currentlySelected().xcrun(args, workingDirectory: workingDirectory, redirectStderr: redirectStderr) diff --git a/Sources/SWBTestSupport/PerfTestSupport.swift b/Sources/SWBTestSupport/PerfTestSupport.swift index 0cfaa6ea..f05532ad 100644 --- a/Sources/SWBTestSupport/PerfTestSupport.swift +++ b/Sources/SWBTestSupport/PerfTestSupport.swift @@ -39,11 +39,11 @@ extension PerfTests { extension Trait where Self == Testing.ConditionTrait { package static var performance: Self { - disabled("Skipping performance test") { + enabled("Skipping performance test") { #if DEBUG - return true + return getEnvironmentVariable("SWB_PERF_TESTS_ENABLE")?.boolValue ?? false #else - return false + return true #endif } } diff --git a/Sources/SWBTestSupport/SkippedTestSupport.swift b/Sources/SWBTestSupport/SkippedTestSupport.swift index 08ef7190..27040e16 100644 --- a/Sources/SWBTestSupport/SkippedTestSupport.swift +++ b/Sources/SWBTestSupport/SkippedTestSupport.swift @@ -146,17 +146,9 @@ extension Trait where Self == Testing.ConditionTrait { }) } - /// Constructs a condition trait that causes a test to be disabled if the Foundation process spawning implementation is not using `posix_spawn_file_actions_addchdir`. + /// Constructs a condition trait that causes a test to be disabled if the Foundation process spawning implementation is not thread-safe. package static var requireThreadSafeWorkingDirectory: Self { - disabled("Thread-safe process working directory support is unavailable.", { - switch try ProcessInfo.processInfo.hostOperatingSystem() { - case .linux: - // Amazon Linux 2 has glibc 2.26, and glibc 2.29 is needed for posix_spawn_file_actions_addchdir_np support - FileManager.default.contents(atPath: "/etc/system-release").map { String(decoding: $0, as: UTF8.self) == "Amazon Linux release 2 (Karoo)\n" } ?? false - default: - false - } - }) + disabled(if: try Process.hasUnsafeWorkingDirectorySupport, "Foundation.Process working directory support is not thread-safe.") } /// Constructs a condition trait that causes a test to be disabled if the specified llbuild API version requirement is not met. diff --git a/Sources/SWBTestSupport/TaskPlanningTestSupport.swift b/Sources/SWBTestSupport/TaskPlanningTestSupport.swift index ae67ca1c..f615bdc7 100644 --- a/Sources/SWBTestSupport/TaskPlanningTestSupport.swift +++ b/Sources/SWBTestSupport/TaskPlanningTestSupport.swift @@ -15,7 +15,7 @@ public import enum SWBProtocol.ExternalToolResult public import struct SWBProtocol.BuildOperationTaskEnded public import SWBTaskConstruction import SWBTaskExecution -package import SWBUtil +public import SWBUtil import Testing package import SWBMacro import Foundation @@ -192,7 +192,7 @@ package extension Array where Element == TaskCondition { open class MockTestTaskPlanningClientDelegate: TaskPlanningClientDelegate, @unchecked Sendable { package init() {} - open func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String: String]) async throws -> ExternalToolResult { + open func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String: String]) async throws -> ExternalToolResult { let args = commandLine.dropFirst() switch commandLine.first.map(Path.init)?.basenameWithoutSuffix { case "actool" where args == ["--version", "--output-format", "xml1"]: diff --git a/Sources/SWBTestSupport/Xcode.swift b/Sources/SWBTestSupport/Xcode.swift index 00448733..969fc923 100644 --- a/Sources/SWBTestSupport/Xcode.swift +++ b/Sources/SWBTestSupport/Xcode.swift @@ -31,7 +31,7 @@ package struct InstalledXcode: Sendable { return try await Path(xcrun(["-f", tool] + toolchainArgs).trimmingCharacters(in: .whitespacesAndNewlines)) } - package func xcrun(_ args: [String], workingDirectory: String? = nil, redirectStderr: Bool = true) async throws -> String { + package func xcrun(_ args: [String], workingDirectory: Path? = nil, redirectStderr: Bool = true) async throws -> String { return try await runProcessWithDeveloperDirectory(["/usr/bin/xcrun"] + args, workingDirectory: workingDirectory, overrideDeveloperDirectory: self.developerDirPath.str, redirectStderr: redirectStderr) } diff --git a/Sources/SWBUtil/PbxCp.swift b/Sources/SWBUtil/PbxCp.swift index c917f411..f5be8a02 100644 --- a/Sources/SWBUtil/PbxCp.swift +++ b/Sources/SWBUtil/PbxCp.swift @@ -80,7 +80,7 @@ fileprivate func xSecCodePathIsSigned(_ path: Path) throws -> Bool { // FIXME: Move this fully to Swift Concurrency and execute the process via llbuild after PbxCp is fully converted to Swift /// Spawns a process and waits for it to finish, closing stdin and redirecting stdout and stderr to fdout. Failure to launch, non-zero exit code, or exit with a signal will throw an error. -fileprivate func spawnTaskAndWait(_ launchPath: Path, _ arguments: [String]?, _ environment: Environment?, _ workingDirPath: String?, _ dryRun: Bool, _ stream: OutputByteStream) async throws { +fileprivate func spawnTaskAndWait(_ launchPath: Path, _ arguments: [String]?, _ environment: Environment?, _ workingDirPath: Path?, _ dryRun: Bool, _ stream: OutputByteStream) async throws { stream <<< launchPath.str for arg in arguments ?? [] { stream <<< " \(arg)" @@ -91,7 +91,7 @@ fileprivate func spawnTaskAndWait(_ launchPath: Path, _ arguments: [String]?, _ return } - let (exitStatus, output) = try await Process.getMergedOutput(url: URL(fileURLWithPath: launchPath.str), arguments: arguments ?? [], currentDirectoryURL: workingDirPath.map { URL(fileURLWithPath: $0, isDirectory: true) }, environment: environment) + let (exitStatus, output) = try await Process.getMergedOutput(url: URL(fileURLWithPath: launchPath.str), arguments: arguments ?? [], currentDirectoryURL: workingDirPath.map { URL(fileURLWithPath: $0.str, isDirectory: true) }, environment: environment) // Copy the process output to the output stream. stream <<< "\(String(decoding: output, as: UTF8.self))" diff --git a/Sources/SWBUtil/Process.swift b/Sources/SWBUtil/Process.swift index 634c3f0d..d07f8e9f 100644 --- a/Sources/SWBUtil/Process.swift +++ b/Sources/SWBUtil/Process.swift @@ -63,6 +63,20 @@ public final class Process { public typealias Process = Foundation.Process #endif +extension Process { + public static var hasUnsafeWorkingDirectorySupport: Bool { + get throws { + switch try ProcessInfo.processInfo.hostOperatingSystem() { + case .linux: + // Amazon Linux 2 has glibc 2.26, and glibc 2.29 is needed for posix_spawn_file_actions_addchdir_np support + FileManager.default.contents(atPath: "/etc/system-release").map { String(decoding: $0, as: UTF8.self) == "Amazon Linux release 2 (Karoo)\n" } ?? false + default: + false + } + } + } +} + extension Process { public static func getOutput(url: URL, arguments: [String], currentDirectoryURL: URL? = nil, environment: Environment? = nil, interruptible: Bool = true) async throws -> Processes.ExecutionResult { if #available(macOS 15, iOS 18, tvOS 18, watchOS 11, visionOS 2, *) { @@ -125,13 +139,8 @@ extension Process { } private static func _getOutput(url: URL, arguments: [String], currentDirectoryURL: URL?, environment: Environment?, interruptible: Bool, setup: (Process) -> T, collect: (T) async throws -> U) async throws -> (exitStatus: Processes.ExitStatus, output: U) { - if !url.isFileURL { - throw StubError.error("\(url) is not an absolute file URL") - } let executableFilePath = try url.standardizedFileURL.filePath - if try !localFS.isExecutable(executableFilePath) { - throw StubError.error("\(executableFilePath.str) is not an executable file") - } + let process = Process() process.executableURL = url process.arguments = arguments @@ -140,6 +149,14 @@ extension Process { } process.environment = environment.map { .init($0) } ?? nil + if try currentDirectoryURL != nil && hasUnsafeWorkingDirectorySupport { + throw try RunProcessLaunchError(process, context: "Foundation.Process working directory support is not thread-safe") + } + + if try !localFS.isExecutable(executableFilePath) { + throw try RunProcessLaunchError(process, context: "\(executableFilePath.str) is not an executable file") + } + let streams = setup(process) try await process.run(interruptible: interruptible) @@ -300,9 +317,68 @@ extension Processes.ExitStatus: CustomStringConvertible { } } -public struct RunProcessNonZeroExitError: Error { +public protocol RunProcessError: Sendable { + var args: [String] { get } + var workingDirectory: Path? { get } + var environment: Environment { get } +} + +extension RunProcessError { + fileprivate var commandIdentityPrefixString: String { + let fullArgs: [String] + if !environment.isEmpty { + fullArgs = ["env"] + [String: String](environment).sorted(byKey: <).map { key, value in "\(key)=\(value)" } + args + } else { + fullArgs = args + } + + let commandString = UNIXShellCommandCodec(encodingStrategy: .singleQuotes, encodingBehavior: .fullCommandLine).encode(fullArgs) + let fullCommandString: String + if let workingDirectory { + let directoryCommandString = UNIXShellCommandCodec(encodingStrategy: .singleQuotes, encodingBehavior: .fullCommandLine).encode(["cd", workingDirectory.str]) + fullCommandString = "(\([directoryCommandString, commandString].joined(separator: " && ")))" + } else { + fullCommandString = commandString + } + + return "The command `\(fullCommandString)`" + } +} + +public struct RunProcessLaunchError: Error, RunProcessError { public let args: [String] - public let workingDirectory: String? + public let workingDirectory: Path? + public let environment: Environment + public let context: String + + public init(args: [String], workingDirectory: Path?, environment: Environment, context: String) { + self.args = args + self.workingDirectory = workingDirectory + self.environment = environment + self.context = context + } + + public init(_ process: Process, context: String) throws { + self.args = ((process.executableURL?.path).map { [$0] } ?? []) + (process.arguments ?? []) + self.workingDirectory = try process.currentDirectoryURL?.filePath + self.environment = process.environment.map { .init($0) } ?? .init() + self.context = context + } +} + +extension RunProcessLaunchError: CustomStringConvertible, LocalizedError { + public var description: String { + return "\(commandIdentityPrefixString) failed to launch. \(context)." + } + + public var errorDescription: String? { + return description + } +} + +public struct RunProcessNonZeroExitError: Error, RunProcessError { + public let args: [String] + public let workingDirectory: Path? public let environment: Environment public let status: Processes.ExitStatus @@ -313,15 +389,15 @@ public struct RunProcessNonZeroExitError: Error { public let output: Output? - public init(args: [String], workingDirectory: String?, environment: Environment, status: Processes.ExitStatus, mergedOutput: ByteString) { + public init(args: [String], workingDirectory: Path?, environment: Environment, status: Processes.ExitStatus, mergedOutput: ByteString) { self.init(args: args, workingDirectory: workingDirectory, environment: environment, status: status, output: .merged(mergedOutput)) } - public init(args: [String], workingDirectory: String?, environment: Environment, status: Processes.ExitStatus, stdout: ByteString, stderr: ByteString) { + public init(args: [String], workingDirectory: Path?, environment: Environment, status: Processes.ExitStatus, stdout: ByteString, stderr: ByteString) { self.init(args: args, workingDirectory: workingDirectory, environment: environment, status: status, output: .separate(stdout: stdout, stderr: stderr)) } - public init(args: [String], workingDirectory: String?, environment: Environment, status: Processes.ExitStatus, output: Output) { + public init(args: [String], workingDirectory: Path?, environment: Environment, status: Processes.ExitStatus, output: Output) { self.args = args self.workingDirectory = workingDirectory self.environment = environment @@ -331,7 +407,7 @@ public struct RunProcessNonZeroExitError: Error { public init?(_ process: Process) throws { self.args = ((process.executableURL?.path).map { [$0] } ?? []) + (process.arguments ?? []) - self.workingDirectory = process.currentDirectoryURL?.path + self.workingDirectory = try process.currentDirectoryURL?.filePath self.environment = process.environment.map { .init($0) } ?? .init() self.status = try .init(process) self.output = nil @@ -343,23 +419,7 @@ public struct RunProcessNonZeroExitError: Error { extension RunProcessNonZeroExitError: CustomStringConvertible, LocalizedError { public var description: String { - let fullArgs: [String] - if !environment.isEmpty { - fullArgs = ["env"] + [String: String](environment).sorted(byKey: <).map { key, value in "\(key)=\(value)" } + args - } else { - fullArgs = args - } - - let commandString = UNIXShellCommandCodec(encodingStrategy: .singleQuotes, encodingBehavior: .fullCommandLine).encode(fullArgs) - let fullCommandString: String - if let workingDirectory { - let directoryCommandString = UNIXShellCommandCodec(encodingStrategy: .singleQuotes, encodingBehavior: .fullCommandLine).encode(["cd", workingDirectory]) - fullCommandString = "(\([directoryCommandString, commandString].joined(separator: " && ")))" - } else { - fullCommandString = commandString - } - - let message = "The command `\(fullCommandString)` \(status)." + let message = "\(commandIdentityPrefixString) \(status)." switch output { case let .separate(stdout, stderr) where !stdout.isEmpty || !stderr.isEmpty: return message + [ diff --git a/Sources/SWBUtil/URL.swift b/Sources/SWBUtil/URL.swift index 264a792d..725f7ac5 100644 --- a/Sources/SWBUtil/URL.swift +++ b/Sources/SWBUtil/URL.swift @@ -35,6 +35,13 @@ extension URL { } } -fileprivate enum FileURLError: Error { +fileprivate enum FileURLError: Error, CustomStringConvertible { case notRepresentable(URL) + + var description: String { + switch self { + case .notRepresentable(let url): + return "URL \(url) cannot be represented as an absolute file path" + } + } } diff --git a/Sources/SwiftBuild/SWBClientExchangeSupport.swift b/Sources/SwiftBuild/SWBClientExchangeSupport.swift index f93c7af7..730caa20 100644 --- a/Sources/SwiftBuild/SWBClientExchangeSupport.swift +++ b/Sources/SwiftBuild/SWBClientExchangeSupport.swift @@ -42,7 +42,7 @@ fileprivate extension Processes.ExitStatus { return await session.service.send(ErrorResponse("No delegate for response.")) } - let result = await Result.catching { try await delegate.executeExternalTool(commandLine: message.commandLine, workingDirectory: message.workingDirectory, environment: message.environment) } + let result = await Result.catching { try await delegate.executeExternalTool(commandLine: message.commandLine, workingDirectory: message.workingDirectory?.str, environment: message.environment) } let reply = ExternalToolExecutionResponse(sessionHandle: message.sessionHandle, exchangeHandle: message.exchangeHandle, value: result.map(ExternalToolResult.init).mapError { .error("\($0)") }) return await session.service.send(reply) } diff --git a/Tests/SWBAndroidPlatformTests/SWBAndroidPlatformTests.swift b/Tests/SWBAndroidPlatformTests/SWBAndroidPlatformTests.swift index 2299add3..74ef48e4 100644 --- a/Tests/SWBAndroidPlatformTests/SWBAndroidPlatformTests.swift +++ b/Tests/SWBAndroidPlatformTests/SWBAndroidPlatformTests.swift @@ -19,7 +19,7 @@ import SWBCore @Suite fileprivate struct AndroidBuildOperationTests: CoreBasedTests { - @Test(.requireSDKs(.android), .requireThreadSafeWorkingDirectory, arguments: ["armv7", "aarch64", "riscv64", "i686", "x86_64"]) + @Test(.requireSDKs(.android), arguments: ["armv7", "aarch64", "riscv64", "i686", "x86_64"]) func androidCommandLineTool(arch: String) async throws { try await withTemporaryDirectory { (tmpDir: Path) in let testProject = TestProject( diff --git a/Tests/SWBBuildSystemTests/BuildCommandTests.swift b/Tests/SWBBuildSystemTests/BuildCommandTests.swift index 995e8a43..2cd6a109 100644 --- a/Tests/SWBBuildSystemTests/BuildCommandTests.swift +++ b/Tests/SWBBuildSystemTests/BuildCommandTests.swift @@ -22,7 +22,7 @@ import Foundation @Suite fileprivate struct BuildCommandTests: CoreBasedTests { /// Check compilation of a single file in C, ObjC and Swift, including the `uniquingSuffix` behavior. - @Test(.requireSDKs(.host), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host)) func singleFileCompile() async throws { try await withTemporaryDirectory { tmpDirPath async throws -> Void in let testWorkspace = try await TestWorkspace( @@ -84,8 +84,13 @@ fileprivate struct BuildCommandTests: CoreBasedTests { try await tester.checkBuild(parameters: parameters, runDestination: runDestination, persistent: true, buildOutputMap: [cOutputPath: cFile.str]) { results in results.consumeTasksMatchingRuleTypes(excludedTypes) results.checkTaskExists(.matchRule(["CompileC", tmpDirPath.join("Test/aProject/build/aProject.build/Debug\(runDestination.builtProductsDirSuffix)/aLibrary.build/Objects-normal/\(results.runDestinationTargetArchitecture)/CFile.o").str, cFile.str, "normal", results.runDestinationTargetArchitecture, "c", "com.apple.compilers.llvm.clang.1_0.compiler"])) - if runDestination == .linux { // FIXME: This needs to be investigated... iIs not clear why this task is added when building a C file, and only on Linux. - results.checkTaskExists(.matchRule(["SwiftEmitModule", "normal", results.runDestinationTargetArchitecture, "Emitting module for aLibrary"])) + if runDestination == .linux { + // FIXME: This needs to be investigated... iIs not clear why this task is added when building a C file, and only on Linux. It's also nondeterministic. + let tasks = results.findMatchingTasks([.matchRule(["SwiftEmitModule", "normal", results.runDestinationTargetArchitecture, "Emitting module for aLibrary"])]) + for task in tasks { + results.removeMatchedTask(task) + } + #expect(tasks.count == 0 || tasks.count == 1) } results.checkNoTask() } @@ -164,7 +169,7 @@ fileprivate struct BuildCommandTests: CoreBasedTests { } /// Check analyze of a single file. - @Test(.requireSDKs(.host), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host)) func singleFileAnalyze() async throws { try await runSingleFileTask(BuildParameters(configuration: "Debug", activeRunDestination: .host, overrides: ["RUN_CLANG_STATIC_ANALYZER": "YES"]), buildCommand: .singleFileBuild(buildOnlyTheseFiles: [Path("")]), fileName: "File.m") { results, excludedTypes, _, _ in results.consumeTasksMatchingRuleTypes(excludedTypes) @@ -174,7 +179,7 @@ fileprivate struct BuildCommandTests: CoreBasedTests { } /// Check preprocessing of a single file. - @Test(.requireSDKs(.host), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host)) func preprocessSingleFile() async throws { try await runSingleFileTask(BuildParameters(configuration: "Debug", activeRunDestination: .host), buildCommand: .generatePreprocessedFile(buildOnlyTheseFiles: [Path("")]), fileName: "File.m") { results, excludedTypes, inputs, outputs in results.consumeTasksMatchingRuleTypes(excludedTypes) @@ -255,7 +260,7 @@ fileprivate struct BuildCommandTests: CoreBasedTests { } /// Check behavior of the skip dependencies flag. - @Test(.requireSDKs(.host), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host)) func skipDependenciesFlag() async throws { func runTest(skipDependencies: Bool, checkAuxiliaryTarget: (_ results: BuildOperationTester.BuildResults) throws -> Void) async throws { try await withTemporaryDirectory { tmpDirPath async throws -> Void in diff --git a/Tests/SWBBuildSystemTests/BuildOperationTests.swift b/Tests/SWBBuildSystemTests/BuildOperationTests.swift index 70778e4a..e6c0caaf 100644 --- a/Tests/SWBBuildSystemTests/BuildOperationTests.swift +++ b/Tests/SWBBuildSystemTests/BuildOperationTests.swift @@ -30,7 +30,7 @@ import SWBTestSupport @Suite(.requireXcode16()) fileprivate struct BuildOperationTests: CoreBasedTests { - @Test(.requireSDKs(.host), .requireThreadSafeWorkingDirectory, arguments: ["clang", "swiftc"]) + @Test(.requireSDKs(.host), arguments: ["clang", "swiftc"]) func commandLineTool(linkerDriver: String) async throws { try await withTemporaryDirectory { (tmpDir: Path) in let testProject = try await TestProject( @@ -164,7 +164,7 @@ fileprivate struct BuildOperationTests: CoreBasedTests { } } - @Test(.requireSDKs(.host), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host)) func commandLineToolAutolinkingFoundation() async throws { try await withTemporaryDirectory { (tmpDir: Path) in let testProject = try await TestProject( @@ -223,7 +223,7 @@ fileprivate struct BuildOperationTests: CoreBasedTests { } } - @Test(.requireSDKs(.host), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host)) func debuggableCommandLineTool() async throws { try await withTemporaryDirectory { (tmpDir: Path) in let testProject = try await TestProject( @@ -397,7 +397,7 @@ fileprivate struct BuildOperationTests: CoreBasedTests { } } - @Test(.requireSDKs(.host), .skipHostOS(.macOS), .skipHostOS(.windows, "cannot find testing library"), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host), .skipHostOS(.macOS), .skipHostOS(.windows, "cannot find testing library")) func unitTestWithGeneratedEntryPoint() async throws { try await withTemporaryDirectory { (tmpDir: Path) in let testProject = try await TestProject( @@ -842,7 +842,7 @@ That command depends on command in Target 'agg2' (project \'aProject\'): script } } - @Test(.requireSDKs(.host), .skipHostOS(.windows), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host), .skipHostOS(.windows)) func missingShellScriptInputs() async throws { // Test that shell scripts run, even if their inputs are missing. try await withTemporaryDirectory { tmpDirPath async throws -> Void in @@ -884,7 +884,7 @@ That command depends on command in Target 'agg2' (project \'aProject\'): script } /// Check that target dependencies are honored. - @Test(.requireSDKs(.host), .skipHostOS(.windows), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host), .skipHostOS(.windows)) func simulatedTargetDependencies() async throws { try await withTemporaryDirectory { tmpDirPath async throws -> Void in let testWorkspace = TestWorkspace( @@ -948,7 +948,7 @@ That command depends on command in Target 'agg2' (project \'aProject\'): script } /// Check that "build targets not in parallel" is honored. - @Test(.requireSDKs(.host), .skipHostOS(.windows), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host), .skipHostOS(.windows)) func simulatedNonParallelTargetBuild() async throws { try await withTemporaryDirectory { tmpDirPath async throws -> Void in let testWorkspace = TestWorkspace( @@ -990,7 +990,7 @@ That command depends on command in Target 'agg2' (project \'aProject\'): script } /// Check that build phase order is honored. - @Test(.requireSDKs(.host), .skipHostOS(.windows), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host), .skipHostOS(.windows)) func simulatedBuildPhaseOrder() async throws { try await withTemporaryDirectory { tmpDirPath async throws -> Void in let testWorkspace = TestWorkspace( @@ -2392,7 +2392,7 @@ That command depends on command in Target 'agg2' (project \'aProject\'): script } /// Check non-UTF8 encoded shell scripts don't cause any unexpected issues. - @Test(.requireSDKs(.host), .skipHostOS(.windows), .requireSystemPackages(apt: "xxd", yum: "vim-common"), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host), .skipHostOS(.windows), .requireSystemPackages(apt: "xxd", yum: "vim-common")) func nonUTF8ShellScript() async throws { try await withTemporaryDirectory { tmpDir in let testWorkspace = TestWorkspace( @@ -2461,7 +2461,7 @@ That command depends on command in Target 'agg2' (project \'aProject\'): script } /// Check special shell script dependency handling - @Test(.requireSDKs(.host), .skipHostOS(.windows), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host), .skipHostOS(.windows)) func shellScriptIncrementalBehaviors() async throws { try await withTemporaryDirectory { tmpDirPath async throws -> Void in // Test that an incremental rebuild of an empty project does nothing. @@ -2532,7 +2532,7 @@ That command depends on command in Target 'agg2' (project \'aProject\'): script } /// Check chown/chmod dependency handling. - @Test(.requireSDKs(.host), .skipHostOS(.windows), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host), .skipHostOS(.windows)) func setFileAttributesIncrementalBehaviors() async throws { try await withTemporaryDirectory { tmpDirPath async throws -> Void in // Test that an incremental rebuild of an empty project does nothing. @@ -2736,7 +2736,7 @@ That command depends on command in Target 'agg2' (project \'aProject\'): script } /// Check the handling of a minimal copied bundle. - @Test(.requireSDKs(.host), .skipHostOS(.windows), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host), .skipHostOS(.windows)) func minimalCopiedBundle() async throws { try await withTemporaryDirectory { tmpDirPath async throws -> Void in let testWorkspace = TestWorkspace( @@ -4201,7 +4201,7 @@ That command depends on command in Target 'agg2' (project \'aProject\'): script } /// Check that PCH file dependencies are respected. - @Test(.requireSDKs(.host), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host)) func prefixHeaderDependencies() async throws { try await withTemporaryDirectory { tmpDirPath async throws -> Void in let testWorkspace = TestWorkspace( diff --git a/Tests/SWBBuildSystemTests/BuildTaskBehaviorTests.swift b/Tests/SWBBuildSystemTests/BuildTaskBehaviorTests.swift index d8ce8799..17f091fa 100644 --- a/Tests/SWBBuildSystemTests/BuildTaskBehaviorTests.swift +++ b/Tests/SWBBuildSystemTests/BuildTaskBehaviorTests.swift @@ -82,7 +82,7 @@ fileprivate struct BuildTaskBehaviorTests: CoreBasedTests { // FIXME: We should migrate these tests to primarily only use internal execution nodes, and not end up running tools (except for tests which are explicitly trying to test that behavior). - @Test(.requireSDKs(.host), .requireThreadSafeWorkingDirectory, .skipHostOS(.windows, "no /bin/echo")) + @Test(.requireSDKs(.host), .skipHostOS(.windows, "no /bin/echo")) func simulatedSingleInputlessOutputlessCommand() async throws { let echoTask = createTask(ruleInfo: ["echo", "hi"], commandLine: ["/bin/echo", "hi"], inputs: [], outputs: [MakePlannedVirtualNode("")], action: nil) @@ -208,7 +208,7 @@ fileprivate struct BuildTaskBehaviorTests: CoreBasedTests { /// Stress concurrent access to the build system cache during rapid cancel /// then build scenarios. - @Test(.requireSDKs(.host), .skipHostOS(.windows, "no /usr/bin/true"), .requireThreadSafeWorkingDirectory, + @Test(.requireSDKs(.host), .skipHostOS(.windows, "no /usr/bin/true"), // To aid in establishing the subtle concurrent // timing required to trigger chaos, we disable early build operation // cancellation. @@ -308,7 +308,7 @@ fileprivate struct BuildTaskBehaviorTests: CoreBasedTests { } /// Check that we honor specs which are unsafe to interrupt. - @Test(.requireSDKs(.host), .skipHostOS(.windows, "no bash shell"), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host), .skipHostOS(.windows, "no bash shell")) func unsafeToInterrupt() async throws { let fs = localFS let output = MakePlannedVirtualNode("") @@ -383,7 +383,7 @@ fileprivate struct BuildTaskBehaviorTests: CoreBasedTests { } /// Check the behavior of gate tasks. - @Test(.requireSDKs(.host), .skipHostOS(.windows, "no /usr/bin/true"), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host), .skipHostOS(.windows, "no /usr/bin/true")) func simulatedTasksWithGate() async throws { let aNode = MakePlannedVirtualNode("A") let bNode = MakePlannedVirtualNode("B") @@ -407,7 +407,7 @@ fileprivate struct BuildTaskBehaviorTests: CoreBasedTests { } } - @Test(.requireSDKs(.host), .skipHostOS(.windows, "no /bin/echo"), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host), .skipHostOS(.windows, "no /bin/echo")) func simulatedDiamondGraph() async throws { let tasksToMake = [ ("START", inputs: ["/INPUT"]), @@ -443,7 +443,7 @@ fileprivate struct BuildTaskBehaviorTests: CoreBasedTests { } } - @Test(.requireSDKs(.host), .skipHostOS(.windows, "no /usr/bin/true"), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host), .skipHostOS(.windows, "no /usr/bin/true")) func simulatedMustPrecede() async throws { let tasksToMake = ["A", "B", "C", "D"] var tasks: [any PlannedTask] = [] @@ -903,7 +903,7 @@ fileprivate struct BuildTaskBehaviorTests: CoreBasedTests { } /// Check the handling of directory tree nodes. - @Test(.skipHostOS(.windows, "no /usr/bin/find"), .requireSDKs(.host), .requireThreadSafeWorkingDirectory) + @Test(.skipHostOS(.windows, "no /usr/bin/find"), .requireSDKs(.host)) func directoryTreeInputs() async throws { try await withTemporaryDirectory { tmpDir in let fs = localFS @@ -954,7 +954,7 @@ fileprivate struct BuildTaskBehaviorTests: CoreBasedTests { } } - @Test(.requireSDKs(.host), .skipHostOS(.windows, "no /bin/echo"), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host), .skipHostOS(.windows, "no /bin/echo")) func additionalInfoOutput() async throws { let echoTask = createTask(ruleInfo: ["echo", "additional-output"], commandLine: ["/bin/echo", "additional-output"], additionalOutput: ["just some extra output"], inputs: [], outputs: [MakePlannedVirtualNode("")], action: nil) diff --git a/Tests/SWBBuildSystemTests/CustomTaskBuildOperationTests.swift b/Tests/SWBBuildSystemTests/CustomTaskBuildOperationTests.swift index c8dfe2f0..f94d48ac 100644 --- a/Tests/SWBBuildSystemTests/CustomTaskBuildOperationTests.swift +++ b/Tests/SWBBuildSystemTests/CustomTaskBuildOperationTests.swift @@ -24,7 +24,7 @@ import class Foundation.ProcessInfo @Suite fileprivate struct CustomTaskBuildOperationTests: CoreBasedTests { - @Test(.requireSDKs(.host), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host)) func outputParsing() async throws { try await withTemporaryDirectory { tmpDir in let destination: RunDestinationInfo = .host diff --git a/Tests/SWBBuildSystemTests/LinkerTests.swift b/Tests/SWBBuildSystemTests/LinkerTests.swift index 184f51d2..36833664 100644 --- a/Tests/SWBBuildSystemTests/LinkerTests.swift +++ b/Tests/SWBBuildSystemTests/LinkerTests.swift @@ -181,7 +181,7 @@ fileprivate struct LinkerTests: CoreBasedTests { /// * The clang output on Windows has paths that have double slashes, not /// quite valid command quoted. i.e. "C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\VC" /// This needs to be taken into account. - @Test(.requireSDKs(.host), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host)) func alternateLinkerSelection() async throws { let runDestination: RunDestinationInfo = .host let swiftVersion = try await self.swiftVersion diff --git a/Tests/SWBBuildSystemTests/SwiftBuildTraceTests.swift b/Tests/SWBBuildSystemTests/SwiftBuildTraceTests.swift index 461cf787..563c0575 100644 --- a/Tests/SWBBuildSystemTests/SwiftBuildTraceTests.swift +++ b/Tests/SWBBuildSystemTests/SwiftBuildTraceTests.swift @@ -20,7 +20,7 @@ import SWBTaskExecution @Suite fileprivate struct SwiftBuildTraceTests: CoreBasedTests { - @Test(.requireSDKs(.host), .requireThreadSafeWorkingDirectory) + @Test(.requireSDKs(.host)) func swiftBuildTraceEmission() async throws { try await withTemporaryDirectory { tmpDirPath async throws -> Void in try await withEnvironment(["SWIFTBUILD_TRACE_FILE": tmpDirPath.join(".SWIFTBUILD_TRACE").str]) { @@ -71,7 +71,6 @@ fileprivate struct SwiftBuildTraceTests: CoreBasedTests { } let trace = try tester.fs.read(tmpDirPath.join(".SWIFTBUILD_TRACE")).asString - print(trace) #expect(try #/\{\"buildDescriptionSignature\":\".*\",\"isTargetParallelizationEnabled\":true,\"name\":\"Test\",\"path\":\".*\"\}\n\{\"buildDescriptionSignature\":\".*\",\"isTargetParallelizationEnabled\":true,\"name\":\"Test\",\"path\":\".*\"\}\n/#.wholeMatch(in: trace) != nil) } } diff --git a/Tests/SWBCorePerfTests/CommandLineSpecPerfTests.swift b/Tests/SWBCorePerfTests/CommandLineSpecPerfTests.swift index 6a2eae3b..d85ccb2c 100644 --- a/Tests/SWBCorePerfTests/CommandLineSpecPerfTests.swift +++ b/Tests/SWBCorePerfTests/CommandLineSpecPerfTests.swift @@ -236,7 +236,7 @@ extension CapturingTaskGenerationDelegate: TaskActionCreationDelegate { } extension CapturingTaskGenerationDelegate: CoreClientDelegate { - func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String: String]) async throws -> ExternalToolResult { + func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String: String]) async throws -> ExternalToolResult { return .emptyResult } } diff --git a/Tests/SWBCoreTests/ClangSerializedDiagnosticsTests.swift b/Tests/SWBCoreTests/ClangSerializedDiagnosticsTests.swift index 81070281..3f37e39e 100644 --- a/Tests/SWBCoreTests/ClangSerializedDiagnosticsTests.swift +++ b/Tests/SWBCoreTests/ClangSerializedDiagnosticsTests.swift @@ -28,11 +28,11 @@ fileprivate struct ClangSerializedDiagnosticsTests: CoreBasedTests { } /// Test that Clang serialized diagnostics are supported. - @Test + @Test(.requireThreadSafeWorkingDirectory) func clangSerializedDiagnosticSupported() async throws { try await withTemporaryDirectory { tmpDir in let diagnosticsPath = tmpDir.join("foo.diag") - _ = try? await runHostProcess(["clang", "-serialize-diagnostics", diagnosticsPath.str, "foo.c"], workingDirectory: tmpDir.str) + _ = try? await runHostProcess(["clang", "-serialize-diagnostics", diagnosticsPath.str, "foo.c"], workingDirectory: tmpDir) let libclang = try #require(await Libclang(path: libclangPath)) libclang.leak() @@ -53,7 +53,7 @@ fileprivate struct ClangSerializedDiagnosticsTests: CoreBasedTests { let cFilePath = tmpDir.join("dir/foo.c") try localFS.write(cFilePath, contents: "#include \"other/foo.h\"\nint main() { return 0; }") let taskWorkingDirectory = cFilePath.dirname - _ = try? await runHostProcess(["clang", "-I../", "-Wall", "-serialize-diagnostics", diagnosticsPath.str, "foo.c"], workingDirectory: taskWorkingDirectory.str) + _ = try? await runHostProcess(["clang", "-I../", "-Wall", "-serialize-diagnostics", diagnosticsPath.str, "foo.c"], workingDirectory: taskWorkingDirectory) let libclang = try #require(await Libclang(path: libclangPath)) libclang.leak() @@ -82,7 +82,7 @@ fileprivate struct ClangSerializedDiagnosticsTests: CoreBasedTests { let cFilePath = tmpDir.join("dir/foo.c") try localFS.write(cFilePath, contents: "#include \"other/foo.h\"\nint main() { return 0; }") let taskWorkingDirectory = cFilePath.dirname - _ = try? await runHostProcess(["clang", "-I../", "-Wall", "-serialize-diagnostics", diagnosticsPath.str, "foo.c"], workingDirectory: taskWorkingDirectory.str) + _ = try? await runHostProcess(["clang", "-I../", "-Wall", "-serialize-diagnostics", diagnosticsPath.str, "foo.c"], workingDirectory: taskWorkingDirectory) let libclang = try #require(await Libclang(path: libclangPath)) libclang.leak() @@ -106,7 +106,7 @@ fileprivate struct ClangSerializedDiagnosticsTests: CoreBasedTests { try localFS.write(swiftFilePath, contents: "#warning(\"custom warning\")") let taskWorkingDirectory = swiftFilePath.dirname let diagnosticsPath = taskWorkingDirectory.join("foo.dia") - _ = try? await runHostProcess(["swiftc", "-c", "-serialize-diagnostics", "foo.swift"], workingDirectory: taskWorkingDirectory.str) + _ = try? await runHostProcess(["swiftc", "-c", "-serialize-diagnostics", "foo.swift"], workingDirectory: taskWorkingDirectory) let libclang = try #require(await Libclang(path: libclangPath)) libclang.leak() diff --git a/Tests/SWBProtocolTests/MessageSerializationTests.swift b/Tests/SWBProtocolTests/MessageSerializationTests.swift index 53dcc2c4..c749db18 100644 --- a/Tests/SWBProtocolTests/MessageSerializationTests.swift +++ b/Tests/SWBProtocolTests/MessageSerializationTests.swift @@ -105,7 +105,7 @@ import Testing } @Test func clientExchangeMessagesRoundTrip() { - assertMsgPackMessageRoundTrip(ExternalToolExecutionRequest(sessionHandle: "theSession", exchangeHandle: "handle", commandLine: ["echo", "foo"], workingDirectory: "/foo", environment: ["FOO": "BAR"])) + assertMsgPackMessageRoundTrip(ExternalToolExecutionRequest(sessionHandle: "theSession", exchangeHandle: "handle", commandLine: ["echo", "foo"], workingDirectory: .root.join("foo"), environment: ["FOO": "BAR"])) assertMsgPackMessageRoundTrip(ExternalToolExecutionResponse(sessionHandle: "theSession", exchangeHandle: "handle", value: .success(.result(status: .exit(1), stdout: Data("foo".utf8), stderr: Data("bar".utf8))))) assertMsgPackMessageRoundTrip(ExternalToolExecutionResponse(sessionHandle: "theSession", exchangeHandle: "handle", value: .failure(StubError.error("broken!")))) diff --git a/Tests/SWBQNXPlatformTests/SWBQNXPlatformTests.swift b/Tests/SWBQNXPlatformTests/SWBQNXPlatformTests.swift index 3d9b9b92..bdce751c 100644 --- a/Tests/SWBQNXPlatformTests/SWBQNXPlatformTests.swift +++ b/Tests/SWBQNXPlatformTests/SWBQNXPlatformTests.swift @@ -19,7 +19,7 @@ import SWBCore @Suite fileprivate struct QNXBuildOperationTests: CoreBasedTests { - @Test(.requireSDKs(.qnx), .skipHostOS(.macOS), .requireThreadSafeWorkingDirectory, arguments: ["aarch64", "x86_64"]) + @Test(.requireSDKs(.qnx), .skipHostOS(.macOS), arguments: ["aarch64", "x86_64"]) func qnxCommandLineTool(arch: String) async throws { try await withTemporaryDirectory { (tmpDir: Path) in let testProject = TestProject( diff --git a/Tests/SWBTaskConstructionTests/BuildToolTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/BuildToolTaskConstructionTests.swift index 4f8f8ca7..5bcdaf39 100644 --- a/Tests/SWBTaskConstructionTests/BuildToolTaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/BuildToolTaskConstructionTests.swift @@ -245,7 +245,7 @@ fileprivate struct BuildToolTaskConstructionTests: CoreBasedTests { /// Client to generate files from the core data model. final class TestCoreDataCompilerTaskPlanningClientDelegate: MockTestTaskPlanningClientDelegate, @unchecked Sendable { - override func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + override func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { if commandLine.first.map(Path.init)?.basename != "momc" { return try await super.executeExternalTool(commandLine: commandLine, workingDirectory: workingDirectory, environment: environment) } @@ -469,7 +469,7 @@ fileprivate struct BuildToolTaskConstructionTests: CoreBasedTests { /// Client to generate files from the CoreML model. final class TestCoreMLCompilerTaskPlanningClientDelegate: MockTestTaskPlanningClientDelegate, @unchecked Sendable { - override func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + override func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { if commandLine.first.map(Path.init)?.basename == "coremlc", let outputDir = commandLine[safe: 3].map(Path.init), let input = commandLine.firstIndex(where: { $0.hasSuffix(".mlmodel") || $0.hasSuffix(".mlpackage") }).map({ Path(commandLine[$0]) }), @@ -1172,7 +1172,7 @@ fileprivate struct BuildToolTaskConstructionTests: CoreBasedTests { self.moduleName = moduleName } - override func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + override func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { if commandLine.first.map(Path.init)?.basename == "intentbuilderc", let outputDir = commandLine.elementAfterElements(["-output"]).map(Path.init), let classPrefix = commandLine.elementAfterElements(["-classPrefix"]), diff --git a/Tests/SWBTaskConstructionTests/ClangTests.swift b/Tests/SWBTaskConstructionTests/ClangTests.swift index 39cb40df..20010d33 100644 --- a/Tests/SWBTaskConstructionTests/ClangTests.swift +++ b/Tests/SWBTaskConstructionTests/ClangTests.swift @@ -196,7 +196,7 @@ fileprivate struct ClangTests: CoreBasedTests { super.init() } - override func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + override func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { if commandLine.first == mockClangPath { return .result(status: .exit(0), stdout: Data(), stderr: Data()) } diff --git a/Tests/SWBTaskConstructionTests/EagerCompilationTests.swift b/Tests/SWBTaskConstructionTests/EagerCompilationTests.swift index af9a6cc6..fe8f01bb 100644 --- a/Tests/SWBTaskConstructionTests/EagerCompilationTests.swift +++ b/Tests/SWBTaskConstructionTests/EagerCompilationTests.swift @@ -1256,7 +1256,7 @@ fileprivate struct EagerCompilationTests: CoreBasedTests { let buildRequest = BuildRequest(parameters: parameters, buildTargets: tester.workspace.projects[0].targets.map({ BuildRequest.BuildTargetInfo(parameters: parameters, target: $0) }), continueBuildingAfterErrors: true, useParallelTargets: true, useImplicitDependencies: false, useDryRun: false) final class Delegate: MockTestTaskPlanningClientDelegate, @unchecked Sendable { - override func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + override func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { switch commandLine.first.map(Path.init)?.basename { case "intentbuilderc"?: do { diff --git a/Tests/SWBTaskConstructionTests/InstallAPITaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/InstallAPITaskConstructionTests.swift index 8390c67d..10338052 100644 --- a/Tests/SWBTaskConstructionTests/InstallAPITaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/InstallAPITaskConstructionTests.swift @@ -1797,7 +1797,7 @@ fileprivate struct InstallAPITaskConstructionTests: CoreBasedTests { let tester = try await TaskConstructionTester(getCore(), testProject) final class Delegate: MockTestTaskPlanningClientDelegate, @unchecked Sendable { - override func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + override func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { if commandLine.first.map(Path.init)?.basename == "intentbuilderc", let outputDir = commandLine.elementAfterElements(["-output"]).map(Path.init), let classPrefix = commandLine.elementAfterElements(["-classPrefix"]), diff --git a/Tests/SWBTaskConstructionTests/ModuleVerifierTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/ModuleVerifierTaskConstructionTests.swift index 8d83632e..e01298a5 100644 --- a/Tests/SWBTaskConstructionTests/ModuleVerifierTaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/ModuleVerifierTaskConstructionTests.swift @@ -901,7 +901,7 @@ fileprivate struct ModuleVerifierTaskConstructionTests: CoreBasedTests { } fileprivate final class TestIntentsCompilerTaskPlanningClientDelegate: MockTestTaskPlanningClientDelegate, @unchecked Sendable { - override func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + override func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { let commandName = commandLine.first.map(Path.init)?.basename switch commandName { case "intentbuilderc": diff --git a/Tests/SWBTaskConstructionTests/OnDemandResourcesTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/OnDemandResourcesTaskConstructionTests.swift index 5695b3e5..feda1daf 100644 --- a/Tests/SWBTaskConstructionTests/OnDemandResourcesTaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/OnDemandResourcesTaskConstructionTests.swift @@ -67,7 +67,7 @@ fileprivate struct OnDemandResourcesTaskConstructionTests: CoreBasedTests { let SRCROOT = tester.workspace.projects[0].sourceRoot.str final class ClientDelegate: MockTestTaskPlanningClientDelegate, @unchecked Sendable { - override func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + override func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { if commandLine.first.map(Path.init)?.basename == "actool", commandLine.dropFirst().first != "--version" { return .result(status: .exit(0), stdout: Data("{}".utf8), stderr: Data()) } diff --git a/Tests/SWBTaskConstructionTests/PackageProductConstructionTests.swift b/Tests/SWBTaskConstructionTests/PackageProductConstructionTests.swift index e829ad94..ee33a453 100644 --- a/Tests/SWBTaskConstructionTests/PackageProductConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/PackageProductConstructionTests.swift @@ -1245,7 +1245,7 @@ fileprivate struct PackageProductConstructionTests: CoreBasedTests { /// Client to generate files from the core data model. final class TestCoreDataCompilerTaskPlanningClientDelegate: MockTestTaskPlanningClientDelegate, @unchecked Sendable { - override func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + override func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { if commandLine.first.map(Path.init)?.basename == "momc", let outputDir = commandLine.last.map(Path.init) { return .result(status: .exit(0), stdout: Data([ outputDir.join("EntityOne+CoreDataClass.swift"), diff --git a/Tests/SWBTaskConstructionTests/PreviewsTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/PreviewsTaskConstructionTests.swift index d829de9e..4e7e6eee 100644 --- a/Tests/SWBTaskConstructionTests/PreviewsTaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/PreviewsTaskConstructionTests.swift @@ -807,7 +807,7 @@ fileprivate struct PreviewsTaskConstructionTests: CoreBasedTests { ]) final class ClientDelegate: MockTestTaskPlanningClientDelegate, @unchecked Sendable { - override func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + override func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { if commandLine.first.map(Path.init)?.basename == "actool", commandLine.dropFirst().first != "--version" { return .result(status: .exit(0), stdout: Data("{}".utf8), stderr: Data()) } @@ -984,7 +984,7 @@ fileprivate struct PreviewsTaskConstructionTests: CoreBasedTests { ]) final class ClientDelegate: MockTestTaskPlanningClientDelegate, @unchecked Sendable { - override func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + override func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { if commandLine.first.map(Path.init)?.basename == "actool", commandLine.dropFirst().first != "--version" { return .result(status: .exit(0), stdout: Data("{}".utf8), stderr: Data()) } @@ -1074,7 +1074,7 @@ fileprivate struct PreviewsTaskConstructionTests: CoreBasedTests { ]) final class ClientDelegate: MockTestTaskPlanningClientDelegate, @unchecked Sendable { - override func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + override func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { if commandLine.first.map(Path.init)?.basename == "actool", commandLine.dropFirst().first != "--version" { return .result(status: .exit(0), stdout: Data("{}".utf8), stderr: Data()) } @@ -1175,7 +1175,7 @@ fileprivate struct PreviewsTaskConstructionTests: CoreBasedTests { ]) final class ClientDelegate: MockTestTaskPlanningClientDelegate, @unchecked Sendable { - override func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + override func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { if commandLine.first.map(Path.init)?.basename == "actool", commandLine.dropFirst().first != "--version" { return .result(status: .exit(0), stdout: Data("{}".utf8), stderr: Data()) } diff --git a/Tests/SWBTaskConstructionTests/TaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/TaskConstructionTests.swift index a40e9382..a3931342 100644 --- a/Tests/SWBTaskConstructionTests/TaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/TaskConstructionTests.swift @@ -7287,7 +7287,7 @@ fileprivate struct TaskConstructionTests: CoreBasedTests { /// Client to generate files from the core data model. class TestCoreDataCompilerTaskPlanningClientDelegate: TaskPlanningClientDelegate { - func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { return .emptyResult } } @@ -7393,7 +7393,7 @@ fileprivate struct TaskConstructionTests: CoreBasedTests { let tester = try await TaskConstructionTester(getCore(), testWorkspace) final class Delegate: MockTestTaskPlanningClientDelegate, @unchecked Sendable { - override func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + override func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { if commandLine.first.map(Path.init)?.basename == "intentbuilderc", let outputDir = commandLine.elementAfterElements(["-output"]).map(Path.init), let classPrefix = commandLine.elementAfterElements(["-classPrefix"]), diff --git a/Tests/SWBTaskConstructionTests/XCStringsTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/XCStringsTaskConstructionTests.swift index 9e5272fb..79ec9b21 100644 --- a/Tests/SWBTaskConstructionTests/XCStringsTaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/XCStringsTaskConstructionTests.swift @@ -32,7 +32,7 @@ private final class MockXCStringsTool: MockTestTaskPlanningClientDelegate, @unch self.requiredCommandLine = requiredCommandLine } - override func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + override func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { switch commandLine.first.map(Path.init)?.basename { case "xcstringstool": break diff --git a/Tests/SWBTaskExecutionTests/InProcessTaskTestSupport.swift b/Tests/SWBTaskExecutionTests/InProcessTaskTestSupport.swift index ac43422e..82e321c8 100644 --- a/Tests/SWBTaskExecutionTests/InProcessTaskTestSupport.swift +++ b/Tests/SWBTaskExecutionTests/InProcessTaskTestSupport.swift @@ -59,7 +59,7 @@ struct MockExecutionDelegate: TaskExecutionDelegate { } struct MockTaskExecutionClientDelegate: TaskExecutionClientDelegate { - func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult { + func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult { .deferred } } diff --git a/Tests/SWBTaskExecutionTests/TaskTestSupport.swift b/Tests/SWBTaskExecutionTests/TaskTestSupport.swift index 18c0fd93..ba202ff0 100644 --- a/Tests/SWBTaskExecutionTests/TaskTestSupport.swift +++ b/Tests/SWBTaskExecutionTests/TaskTestSupport.swift @@ -56,14 +56,14 @@ class MockDynamicTaskExecutionDelegate: DynamicTaskExecutionDelegate { func spawn( commandLine: [String], environment: [String: String], - workingDirectory: String, + workingDirectory: Path, processDelegate: any ProcessDelegate ) async throws -> Bool { if commandLine.isEmpty { throw StubError.error("Invalid number of arguments") } - let executionResult = try await Process.getOutput(url: URL(fileURLWithPath: commandLine[0]), arguments: Array(commandLine.dropFirst()), currentDirectoryURL: URL(fileURLWithPath: workingDirectory), environment: .init(environment)) + let executionResult = try await Process.getOutput(url: URL(fileURLWithPath: commandLine[0]), arguments: Array(commandLine.dropFirst()), currentDirectoryURL: URL(fileURLWithPath: workingDirectory.str), environment: .init(environment)) // FIXME: Pass the real PID let pid = llbuild_pid_t.invalid diff --git a/Tests/SWBUtilTests/MachOTests.swift b/Tests/SWBUtilTests/MachOTests.swift index c7709e1b..06f9554f 100644 --- a/Tests/SWBUtilTests/MachOTests.swift +++ b/Tests/SWBUtilTests/MachOTests.swift @@ -448,7 +448,7 @@ fileprivate struct MachOTests { try await withTemporaryDirectory { path in let expectedVersion = Version(11, 2, 3) try localFS.write(path.join("main.c"), contents: "int main() { return 0; }") - _ = try await InstalledXcode.currentlySelected().xcrun(["-sdk", "macosx", "clang", "-target", "\(#require(Architecture.host.stringValue))-apple-macos\(expectedVersion)", "main.c"], workingDirectory: path.str) + _ = try await InstalledXcode.currentlySelected().xcrun(["-sdk", "macosx", "clang", "-target", "\(#require(Architecture.host.stringValue))-apple-macos\(expectedVersion)", "main.c"], workingDirectory: path) let machOPath = path.join("a.out") let files: [BinaryReader] = try allReaders(machOPath) @@ -475,7 +475,7 @@ fileprivate struct MachOTests { func rPaths() async throws { try await withTemporaryDirectory { path in try localFS.write(path.join("main.c"), contents: "int main() { return 0; }") - _ = try await InstalledXcode.currentlySelected().xcrun(["-sdk", "macosx", "clang", "-target", "\(#require(Architecture.host.stringValue))-apple-macos11.0", "-rpath", "@loader_path/Frameworks", "-rpath", "@loader_path/../Frameworks", "main.c"], workingDirectory: path.str) + _ = try await InstalledXcode.currentlySelected().xcrun(["-sdk", "macosx", "clang", "-target", "\(#require(Architecture.host.stringValue))-apple-macos11.0", "-rpath", "@loader_path/Frameworks", "-rpath", "@loader_path/../Frameworks", "main.c"], workingDirectory: path) let machOPath = path.join("a.out") let files: [BinaryReader] = try allReaders(machOPath) @@ -501,7 +501,7 @@ fileprivate struct MachOTests { func atomInfo() async throws { try await withTemporaryDirectory { path in try localFS.write(path.join("file.c"), contents: "const int foo = 0;") - _ = try await InstalledXcode.currentlySelected().xcrun(["-sdk", "macosx", "clang", "-target", "\(#require(Architecture.host.stringValue))-apple-macos11.0", "-dynamiclib", "-Xlinker", "-make_mergeable", "file.c"], workingDirectory: path.str) + _ = try await InstalledXcode.currentlySelected().xcrun(["-sdk", "macosx", "clang", "-target", "\(#require(Architecture.host.stringValue))-apple-macos11.0", "-dynamiclib", "-Xlinker", "-make_mergeable", "file.c"], workingDirectory: path) let machOPath = path.join("a.out") let files: [BinaryReader] = try allReaders(machOPath) diff --git a/Tests/SWBWebAssemblyPlatformTests/SWBWebAssemblyPlatformTests.swift b/Tests/SWBWebAssemblyPlatformTests/SWBWebAssemblyPlatformTests.swift index fcef5cb9..3f1eddc1 100644 --- a/Tests/SWBWebAssemblyPlatformTests/SWBWebAssemblyPlatformTests.swift +++ b/Tests/SWBWebAssemblyPlatformTests/SWBWebAssemblyPlatformTests.swift @@ -21,7 +21,6 @@ import SWBUtil fileprivate struct SWBWebAssemblyPlatformTests: CoreBasedTests { @Test( .requireSDKs(.wasi), - .requireThreadSafeWorkingDirectory, .skipXcodeToolchain, arguments: ["wasm32"], [true, false] ) @@ -132,7 +131,7 @@ fileprivate struct SWBWebAssemblyPlatformTests: CoreBasedTests { } } - @Test(.requireSDKs(.wasi), .requireThreadSafeWorkingDirectory, .skipXcodeToolchain, arguments: ["wasm32"]) + @Test(.requireSDKs(.wasi), .skipXcodeToolchain, arguments: ["wasm32"]) func wasiCommandWithCAndCxx(arch: String) async throws { let sdkroot = try await #require(getCore().loadSDK(llvmTargetTripleSys: "wasi")).path.str diff --git a/Tests/SwiftBuildPerfTests/BuildOperationPerfTests.swift b/Tests/SwiftBuildPerfTests/BuildOperationPerfTests.swift index 327b2777..48654582 100644 --- a/Tests/SwiftBuildPerfTests/BuildOperationPerfTests.swift +++ b/Tests/SwiftBuildPerfTests/BuildOperationPerfTests.swift @@ -21,7 +21,7 @@ import SWBTestSupport import Testing -@Suite(.performance, .requireThreadSafeWorkingDirectory) +@Suite(.performance) fileprivate struct BuildOperationPerfTests: PerfTests { /// Run a test of a synthetic project with a given number of targets and files. /// diff --git a/Tests/SwiftBuildTests/BuildOperationTests.swift b/Tests/SwiftBuildTests/BuildOperationTests.swift index c6d217a0..e9d94dbf 100644 --- a/Tests/SwiftBuildTests/BuildOperationTests.swift +++ b/Tests/SwiftBuildTests/BuildOperationTests.swift @@ -288,7 +288,7 @@ fileprivate struct BuildOperationTests: CoreBasedTests { } } - @Test(.requireSDKs(.macOS), .skipHostOS(.windows), .requireThreadSafeWorkingDirectory) // version info discovery isn't working on Windows + @Test(.requireSDKs(.macOS), .skipHostOS(.windows)) // version info discovery isn't working on Windows func onlyCreateBuildDescription() async throws { try await withTemporaryDirectory { temporaryDirectory in try await withAsyncDeferrable { deferrable in @@ -445,8 +445,7 @@ fileprivate struct BuildOperationTests: CoreBasedTests { @Test( .requireSDKs(.host), - .skipHostOS(.windows), - .requireThreadSafeWorkingDirectory /* version info discovery isn't working on Windows */, + .skipHostOS(.windows), /* version info discovery isn't working on Windows */ .flaky("Test occasionally crashes in linux CI"), .bug("https://github.com/swiftlang/swift-build/issues/528") ) @@ -1777,7 +1776,7 @@ fileprivate struct BuildOperationTests: CoreBasedTests { } /// Check scraped build issues. - @Test(.requireSDKs(.macOS), .skipHostOS(.windows), .requireThreadSafeWorkingDirectory) // relies on UNIX shell, consider adding Windows command shell support for script phases? + @Test(.requireSDKs(.macOS), .skipHostOS(.windows)) // relies on UNIX shell, consider adding Windows command shell support for script phases? func buildScriptIssues() async throws { try await withTemporaryDirectory { temporaryDirectory in try await withAsyncDeferrable { deferrable in diff --git a/Tests/SwiftBuildTests/DocumentationTests.swift b/Tests/SwiftBuildTests/DocumentationTests.swift index c0534523..49409775 100644 --- a/Tests/SwiftBuildTests/DocumentationTests.swift +++ b/Tests/SwiftBuildTests/DocumentationTests.swift @@ -20,7 +20,7 @@ import Testing @Suite fileprivate struct DocumentationBuildTests: CoreBasedTests { // docc fails on Windows for some reason - @Test(.requireSDKs(.host), .requireThreadSafeWorkingDirectory, .skipHostOS(.windows)) + @Test(.requireSDKs(.host), .skipHostOS(.windows)) func documentationBuild() async throws { try await withTemporaryDirectory { tmpDir in let fs: any FSProxy = localFS diff --git a/Tests/SwiftBuildTests/ServiceTests.swift b/Tests/SwiftBuildTests/ServiceTests.swift index dcd8eef7..aa928a09 100644 --- a/Tests/SwiftBuildTests/ServiceTests.swift +++ b/Tests/SwiftBuildTests/ServiceTests.swift @@ -406,7 +406,7 @@ fileprivate struct ServiceTests { let osv = ProcessInfo.processInfo.operatingSystemVersion let osVersion = try Version(osv) let deploymentTarget = Version(Version.Component(osv.majorVersion), Version.Component(osv.minorVersion), UInt(osv.patchVersion) + 1) - _ = try await InstalledXcode.currentlySelected().xcrun(["-sdk", "macosx", "clang", "-target", "\(#require(Architecture.host.stringValue))-apple-macos\(deploymentTarget.canonicalDeploymentTargetForm)", "main.c"], workingDirectory: path.str) + _ = try await InstalledXcode.currentlySelected().xcrun(["-sdk", "macosx", "clang", "-target", "\(#require(Architecture.host.stringValue))-apple-macos\(deploymentTarget.canonicalDeploymentTargetForm)", "main.c"], workingDirectory: path) _ = await withEnvironment(["SWBBUILDSERVICE_PATH": path.join("a.out").str]) { await #expect(performing: { diff --git a/Tests/SwiftBuildTests/ValidationTests.swift b/Tests/SwiftBuildTests/ValidationTests.swift index 161ccacb..b58590e2 100644 --- a/Tests/SwiftBuildTests/ValidationTests.swift +++ b/Tests/SwiftBuildTests/ValidationTests.swift @@ -40,7 +40,7 @@ fileprivate struct ValidationTests: CoreBasedTests { // Run the subprocess, check the result, and return the output if we succeeded. let executionResult = try await Process.getOutput(url: url, arguments: args, currentDirectoryURL: URL(fileURLWithPath: workingDirectory.str), environment: environment) if !executionResult.exitStatus.isSuccess { - throw RunProcessNonZeroExitError(args: [url.path] + args, workingDirectory: workingDirectory.str, environment: environment, status: executionResult.exitStatus, stdout: ByteString(executionResult.stdout), stderr: ByteString(executionResult.stderr)) + throw RunProcessNonZeroExitError(args: [url.path] + args, workingDirectory: workingDirectory, environment: environment, status: executionResult.exitStatus, stdout: ByteString(executionResult.stdout), stderr: ByteString(executionResult.stderr)) } } From eca9ed51be23dc2b71a174e86680293588d19f8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20B=C3=BCgling?= Date: Tue, 3 Jun 2025 17:09:04 -0700 Subject: [PATCH 21/54] Initial support for tracking locations for assigned values in XCConfigs (#513) This can be used to emit fix-its for XCConfigs files during the build process. --- Sources/SWBCore/MacroConfigFileLoader.swift | 9 +- Sources/SWBMacro/MacroConfigFileParser.swift | 8 +- .../SWBMacro/MacroValueAssignmentTable.swift | 111 +++++++++++++++++- Tests/SWBCoreTests/SettingsTests.swift | 1 + Tests/SWBMacroTests/MacroParsingTests.swift | 41 ++++++- 5 files changed, 154 insertions(+), 16 deletions(-) diff --git a/Sources/SWBCore/MacroConfigFileLoader.swift b/Sources/SWBCore/MacroConfigFileLoader.swift index 2d5e4248..ba7a99b2 100644 --- a/Sources/SWBCore/MacroConfigFileLoader.swift +++ b/Sources/SWBCore/MacroConfigFileLoader.swift @@ -242,7 +242,7 @@ final class MacroConfigFileLoader: Sendable { return MacroConfigFileParser(byteString: data, path: path, delegate: delegate) } - mutating func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, parser: MacroConfigFileParser) { + mutating func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, path: Path, line: Int, startColumn: Int, endColumn: Int, parser: MacroConfigFileParser) { // Look up the macro name, creating it as a user-defined macro if it isn’t already known. let macro = table.namespace.lookupOrDeclareMacro(UserDefinedMacroDeclaration.self, macroName) @@ -253,7 +253,8 @@ final class MacroConfigFileLoader: Sendable { } // Parse the value in a manner consistent with the macro definition. - table.push(macro, table.namespace.parseForMacro(macro, value: value), conditions: conditionSet) + let location = MacroValueAssignmentLocation(path: path, line: line, startColumn: startColumn, endColumn: endColumn) + table.push(macro, table.namespace.parseForMacro(macro, value: value), conditions: conditionSet, location: location) } func handleDiagnostic(_ diagnostic: MacroConfigFileDiagnostic, parser: MacroConfigFileParser) { @@ -301,8 +302,8 @@ fileprivate final class MacroValueAssignmentTableRef { table.namespace } - func push(_ macro: MacroDeclaration, _ value: MacroExpression, conditions: MacroConditionSet? = nil) { - table.push(macro, value, conditions: conditions) + func push(_ macro: MacroDeclaration, _ value: MacroExpression, conditions: MacroConditionSet? = nil, location: MacroValueAssignmentLocation? = nil) { + table.push(macro, value, conditions: conditions, location: location) } } diff --git a/Sources/SWBMacro/MacroConfigFileParser.swift b/Sources/SWBMacro/MacroConfigFileParser.swift index a1fa7ebf..00453277 100644 --- a/Sources/SWBMacro/MacroConfigFileParser.swift +++ b/Sources/SWBMacro/MacroConfigFileParser.swift @@ -276,6 +276,7 @@ public final class MacroConfigFileParser { // MARK: Parsing of value assignment starts here. /// Parses a macro value assignment line of the form MACRONAME [ optional conditions ] ... = VALUE ';'? private func parseMacroValueAssignment() { + let startOfLine = currIdx - 1 // First skip over any whitespace and comments. skipWhitespaceAndComments() @@ -361,6 +362,7 @@ public final class MacroConfigFileParser { // Skip over the equals sign. assert(currChar == /* '=' */ 61) advance() + let startColumn = currIdx - startOfLine var chunks : [String] = [] while let chunk = parseNonListAssignmentRHS() { @@ -383,7 +385,7 @@ public final class MacroConfigFileParser { } // Finally, now that we have the name, conditions, and value, we tell the delegate about it. let value = chunks.joined(separator: " ") - delegate?.foundMacroValueAssignment(name, conditions: conditions, value: value, parser: self) + delegate?.foundMacroValueAssignment(name, conditions: conditions, value: value, path: path, line: currLine, startColumn: startColumn, endColumn: currIdx - startOfLine, parser: self) } public func parseNonListAssignmentRHS() -> String? { @@ -518,7 +520,7 @@ public final class MacroConfigFileParser { } func endPreprocessorInclusion() { } - func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, parser: MacroConfigFileParser) { + func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, path: Path, line: Int, startColumn: Int, endColumn: Int, parser: MacroConfigFileParser) { self.macroName = macroName self.conditions = conditions.isEmpty ? nil : conditions } @@ -565,7 +567,7 @@ public protocol MacroConfigFileParserDelegate { func endPreprocessorInclusion() /// Invoked once for each macro value assignment. The `macroName` is guaranteed to be non-empty, but `value` may be empty. Any macro conditions are passed as tuples in the `conditions`; parameters are guaranteed to be non-empty strings, but patterns may be empty. - mutating func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, parser: MacroConfigFileParser) + mutating func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, path: Path, line: Int, startColumn: Int, endColumn: Int, parser: MacroConfigFileParser) /// Invoked if an error, warning, or other diagnostic is detected. func handleDiagnostic(_ diagnostic: MacroConfigFileDiagnostic, parser: MacroConfigFileParser) diff --git a/Sources/SWBMacro/MacroValueAssignmentTable.swift b/Sources/SWBMacro/MacroValueAssignmentTable.swift index 84a6f962..6586fe43 100644 --- a/Sources/SWBMacro/MacroValueAssignmentTable.swift +++ b/Sources/SWBMacro/MacroValueAssignmentTable.swift @@ -20,18 +20,23 @@ public struct MacroValueAssignmentTable: Serializable, Sendable { /// Maps macro declarations to corresponding linked lists of assignments. public var valueAssignments: [MacroDeclaration: MacroValueAssignment] - private init(namespace: MacroNamespace, valueAssignments: [MacroDeclaration: MacroValueAssignment]) { + private var valueLocations: [String: InternedMacroValueAssignmentLocation] + private var macroConfigPaths: OrderedSet + + private init(namespace: MacroNamespace, valueAssignments: [MacroDeclaration: MacroValueAssignment], valueLocations: [String: InternedMacroValueAssignmentLocation], macroConfigPaths: OrderedSet) { self.namespace = namespace self.valueAssignments = valueAssignments + self.valueLocations = valueLocations + self.macroConfigPaths = macroConfigPaths } public init(namespace: MacroNamespace) { - self.init(namespace: namespace, valueAssignments: [:]) + self.init(namespace: namespace, valueAssignments: [:], valueLocations: [:], macroConfigPaths: OrderedSet()) } /// Convenience initializer to create a `MacroValueAssignmentTable` from another instance (i.e., to create a copy). public init(copying table: MacroValueAssignmentTable) { - self.init(namespace: table.namespace, valueAssignments: table.valueAssignments) + self.init(namespace: table.namespace, valueAssignments: table.valueAssignments, valueLocations: table.valueLocations, macroConfigPaths: table.macroConfigPaths) } /// Remove all assignments for the given macro. @@ -77,11 +82,24 @@ public struct MacroValueAssignmentTable: Serializable, Sendable { /// Adds a mapping from `macro` to `value`, inserting it ahead of any already existing assignment for the same macro. Unless the value refers to the lower-precedence expression (using `$(inherited)` notation), any existing assignments are shadowed but not removed. - public mutating func push(_ macro: MacroDeclaration, _ value: MacroExpression, conditions: MacroConditionSet? = nil) { + public mutating func push(_ macro: MacroDeclaration, _ value: MacroExpression, conditions: MacroConditionSet? = nil, location: MacroValueAssignmentLocation? = nil) { assert(namespace.lookupMacroDeclaration(macro.name) === macro) // Validate the type. assert(macro.type.matchesExpressionType(value)) valueAssignments[macro] = MacroValueAssignment(expression: value, conditions: conditions, next: valueAssignments[macro]) + + if let location { + let index = macroConfigPaths.append(location.path).index + valueLocations[macro.name] = InternedMacroValueAssignmentLocation(pathRef: index, line: location.line, startColumn: location.startColumn, endColumn: location.endColumn) + } + } + + private mutating func mergeLocations(from otherTable: MacroValueAssignmentTable) { + otherTable.valueLocations.forEach { + let path = otherTable.macroConfigPaths[$0.value.pathRef] + let index = macroConfigPaths.append(path).index + valueLocations[$0.key] = .init(pathRef: index, line: $0.value.line, startColumn: $0.value.startColumn, endColumn: $0.value.endColumn) + } } /// Adds a mapping from each of the macro-to-value mappings in `otherTable`, inserting them ahead of any already existing assignments in the receiving table. The other table isn’t affected in any way (in particular, no reference is kept from the receiver to the other table). @@ -89,6 +107,7 @@ public struct MacroValueAssignmentTable: Serializable, Sendable { for (macro, firstAssignment) in otherTable.valueAssignments { valueAssignments[macro] = insertCopiesOfMacroValueAssignmentNodes(firstAssignment, inFrontOf: valueAssignments[macro]) } + mergeLocations(from: otherTable) } /// Looks up and returns the first (highest-precedence) macro value assignment for `macro`, if there is one. @@ -106,6 +125,18 @@ public struct MacroValueAssignmentTable: Serializable, Sendable { return valueAssignments.isEmpty } + public func location(of macro: MacroDeclaration) -> MacroValueAssignmentLocation? { + guard let location = valueLocations[macro.name] else { + return nil + } + return MacroValueAssignmentLocation( + path: macroConfigPaths[location.pathRef], + line: location.line, + startColumn: location.startColumn, + endColumn: location.endColumn + ) + } + public func bindConditionParameter(_ parameter: MacroConditionParameter, _ conditionValues: [String]) -> MacroValueAssignmentTable { return bindConditionParameter(parameter, conditionValues.map { .string($0) }) } @@ -192,6 +223,7 @@ public struct MacroValueAssignmentTable: Serializable, Sendable { bindAndPushAssignment(firstAssignment) } + table.mergeLocations(from: self) return table } @@ -219,7 +251,7 @@ public struct MacroValueAssignmentTable: Serializable, Sendable { // MARK: Serialization public func serialize(to serializer: T) { - serializer.beginAggregate(1) + serializer.beginAggregate(3) // We don't directly serialize MacroDeclarations, but rather serialize their contents "by hand" so when we deserialize we can re-use existing declarations in our namespace. serializer.beginAggregate(valueAssignments.count) @@ -247,6 +279,17 @@ public struct MacroValueAssignmentTable: Serializable, Sendable { } serializer.endAggregate() // valueAssignments + serializer.beginAggregate(valueLocations.count) + for (decl, loc) in valueLocations.sorted(by: { $0.0 < $1.0 }) { + serializer.beginAggregate(2) + serializer.serialize(decl) + serializer.serialize(loc) + serializer.endAggregate() + } + serializer.endAggregate() + + serializer.serialize(macroConfigPaths) + serializer.endAggregate() // the whole table } @@ -255,9 +298,10 @@ public struct MacroValueAssignmentTable: Serializable, Sendable { guard let delegate = deserializer.delegate as? (any MacroValueAssignmentTableDeserializerDelegate) else { throw DeserializerError.invalidDelegate("delegate must be a MacroValueAssignmentTableDeserializerDelegate") } self.namespace = delegate.namespace self.valueAssignments = [:] + self.valueLocations = [:] // Deserialize the table. - try deserializer.beginAggregate(1) + try deserializer.beginAggregate(3) // Iterate over all the key-value pairs. let count: Int = try deserializer.beginAggregate() @@ -304,6 +348,16 @@ public struct MacroValueAssignmentTable: Serializable, Sendable { // Add it to the dictionary. self.valueAssignments[decl] = asgn } + + let count2 = try deserializer.beginAggregate() + for _ in 0...Index + let line: Int + let startColumn: Int + let endColumn: Int + + init(pathRef: OrderedSet.Index, line: Int, startColumn: Int, endColumn: Int) { + self.pathRef = pathRef + self.line = line + self.startColumn = startColumn + self.endColumn = endColumn + } + + public func serialize(to serializer: T) where T : SWBUtil.Serializer { + serializer.beginAggregate(4) + serializer.serialize(pathRef) + serializer.serialize(line) + serializer.serialize(startColumn) + serializer.serialize(endColumn) + serializer.endAggregate() + } + + public init(from deserializer: any SWBUtil.Deserializer) throws { + try deserializer.beginAggregate(4) + self.pathRef = try deserializer.deserialize() + self.line = try deserializer.deserialize() + self.startColumn = try deserializer.deserialize() + self.endColumn = try deserializer.deserialize() + } +} + /// Private function that inserts a copy of the given linked list of MacroValueAssignments (starting at `srcAsgn`) in front of `dstAsgn` (which is optional). The order of the copies is the same as the order of the originals, and the last one will have `dstAsgn` as its `next` property. This function returns the copy that corresponds to `srcAsgn` so the client can add a reference to it wherever it sees fit. private func insertCopiesOfMacroValueAssignmentNodes(_ srcAsgn: MacroValueAssignment, inFrontOf dstAsgn: MacroValueAssignment?) -> MacroValueAssignment { // If we aren't inserting in front of anything, we can preserve the input as is. diff --git a/Tests/SWBCoreTests/SettingsTests.swift b/Tests/SWBCoreTests/SettingsTests.swift index 47f6cf6a..f392d22b 100644 --- a/Tests/SWBCoreTests/SettingsTests.swift +++ b/Tests/SWBCoreTests/SettingsTests.swift @@ -134,6 +134,7 @@ import SWBMacro // Verify that the settings from the xcconfig were added. let XCCONFIG_USER_SETTING = try #require(settings.userNamespace.lookupMacroDeclaration("XCCONFIG_USER_SETTING")) #expect(settings.tableForTesting.lookupMacro(XCCONFIG_USER_SETTING)?.expression.stringRep == "from-xcconfig") + #expect(settings.tableForTesting.location(of: XCCONFIG_USER_SETTING) == MacroValueAssignmentLocation(path: .init("/tmp/xcconfigs/Base0.xcconfig"), line: 1, startColumn: 24, endColumn: 38)) // Verify the user project settings. let USER_PROJECT_SETTING = try #require(settings.userNamespace.lookupMacroDeclaration("USER_PROJECT_SETTING")) diff --git a/Tests/SWBMacroTests/MacroParsingTests.swift b/Tests/SWBMacroTests/MacroParsingTests.swift index 7e6778b7..e639ec4e 100644 --- a/Tests/SWBMacroTests/MacroParsingTests.swift +++ b/Tests/SWBMacroTests/MacroParsingTests.swift @@ -790,7 +790,7 @@ fileprivate let testFileData = [ } func endPreprocessorInclusion() { } - func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, parser: MacroConfigFileParser) { + func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, path: Path, line: Int, startColumn: Int, endColumn: Int, parser: MacroConfigFileParser) { } func handleDiagnostic(_ diagnostic: MacroConfigFileDiagnostic, parser: MacroConfigFileParser) { @@ -804,19 +804,41 @@ fileprivate let testFileData = [ MacroConfigFileParser(byteString: "// [-Wnullability-completeness-on-arrays] \t\t\t(on) Warns about missing nullability annotations on array parameters.", path: Path(""), delegate: delegate).parse() #expect(delegate.diagnosticMessages == [String]()) } + + @Test + func parserProvidesLocationInformation() throws { + TestMacroConfigFileParser("#include \"Multiline.xcconfig\"", + expectedAssignments: [ + (macro: "FEATURE_DEFINES_A", conditions: [], value: "$(A) $(B) $(C)"), + (macro: "FEATURE_DEFINES_B", conditions: [], value: "$(D) $(E) $(F)"), + (macro: "FEATURE_DEFINES_C", conditions: [], value: "$(G) $(H)"), + (macro: "FEATURE_DEFINES_D", conditions: [], value: "$(I)") + ], + expectedDiagnostics: [], + expectedLocations: [ + (macro: "FEATURE_DEFINES_A", path: .init("Multiline.xcconfig"), line: 2, startColumn: 20, endColumn: 37), + (macro: "FEATURE_DEFINES_B", path: .init("Multiline.xcconfig"), line: 5, startColumn: 20, endColumn: 87), + (macro: "FEATURE_DEFINES_C", path: .init("Multiline.xcconfig"), line: 9, startColumn: 20, endColumn: 61), + (macro: "FEATURE_DEFINES_D", path: .init("Multiline.xcconfig"), line: 11, startColumn: 20, endColumn: 45), + ], + expectedIncludeDirectivesCount: 1 + ) + } } // We used typealiased tuples for simplicity and readability. typealias ConditionInfo = (param: String, pattern: String) typealias AssignmentInfo = (macro: String, conditions: [ConditionInfo], value: String) typealias DiagnosticInfo = (level: MacroConfigFileDiagnostic.Level, kind: MacroConfigFileDiagnostic.Kind, line: Int) +typealias LocationInfo = (macro: String, path: Path, line: Int, startColumn: Int, endColumn: Int) -private func TestMacroConfigFileParser(_ string: String, expectedAssignments: [AssignmentInfo], expectedDiagnostics: [DiagnosticInfo], expectedIncludeDirectivesCount: Int, sourceLocation: SourceLocation = #_sourceLocation) { +private func TestMacroConfigFileParser(_ string: String, expectedAssignments: [AssignmentInfo], expectedDiagnostics: [DiagnosticInfo], expectedLocations: [LocationInfo]? = nil, expectedIncludeDirectivesCount: Int, sourceLocation: SourceLocation = #_sourceLocation) { /// We use a custom delegate to test that we’re getting the expected results, which for the sake of convenience are just kept in (name, conds:[(cond-param, cond-value)], value) tuples, i.e. conditions is an array of two-element tuples. class ConfigFileParserTestDelegate : MacroConfigFileParserDelegate { var assignments = Array() var diagnostics = Array() + var locations = Array() var includeDirectivesCount = 0 @@ -834,9 +856,10 @@ private func TestMacroConfigFileParser(_ string: String, expectedAssignments: [A func endPreprocessorInclusion() { self.includeDirectivesCount += 1 } - func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, parser: MacroConfigFileParser) { + func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, path: Path, line: Int, startColumn: Int, endColumn: Int, parser: MacroConfigFileParser) { // print("\(parser.lineNumber): \(macroName)\(conditions.map({ "[\($0.param)=\($0.pattern)]" }).joinWithSeparator(""))=\(value)") assignments.append((macro: macroName, conditions: conditions, value: value)) + locations.append((macro: macroName, path: path, line: line, startColumn: startColumn, endColumn: endColumn)) } func handleDiagnostic(_ diagnostic: MacroConfigFileDiagnostic, parser: MacroConfigFileParser) { // print("\(parser.lineNumber): \(diagnostic)") @@ -857,6 +880,10 @@ private func TestMacroConfigFileParser(_ string: String, expectedAssignments: [A // Check the diagnostics that the delegate saw against the expected ones. #expect(delegate.diagnostics == expectedDiagnostics, "expected parse diagnostics \(expectedDiagnostics), but instead got \(delegate.diagnostics)", sourceLocation: sourceLocation) + if let expectedLocations { + #expect(delegate.locations == expectedLocations, "expected parse locations \(expectedLocations), but instead ogt \(delegate.locations)", sourceLocation: sourceLocation) + } + #expect(delegate.includeDirectivesCount == expectedIncludeDirectivesCount, "expected number of configs parsed to be \(expectedIncludeDirectivesCount), but instead got \(delegate.includeDirectivesCount)", sourceLocation: sourceLocation) } @@ -885,6 +912,14 @@ func ==(lhs: [DiagnosticInfo], rhs: [DiagnosticInfo]) -> Bool { return lhs.count == rhs.count && zip(lhs, rhs).filter({ return !($0.0 == $0.1) }).isEmpty } +func ==(lhs: LocationInfo, rhs: LocationInfo) -> Bool { + return (lhs.macro == rhs.macro) && (lhs.path == rhs.path) && (lhs.line == rhs.line) && (lhs.startColumn == rhs.startColumn) && (lhs.endColumn == rhs.endColumn) +} + +func ==(lhs: [LocationInfo], rhs: [LocationInfo]) -> Bool { + return lhs.count == rhs.count && zip(lhs, rhs).filter({ return !($0.0 == $0.1) }).isEmpty +} + /// Private helper function that parses a string representation as either a string or a string list (depending on the parameter), and checks the resulting parser delegate method call sequence and diagnostics (if applicable) against what’s expected. This is a private function that’s called by the two internal test functions TestMacroStringParsing() and TestMacroStringListParsing(). The original file name and line number are passed in so that Xcode diagnostics will refer to the call site. Each diagnostic is provided by the unit test as a tuple containing the level, kind, and associated range (expressed as start and end “distances”, in the manner of Int.Distance, into the original string). private func TestMacroParsing(_ string: String, asList: Bool, expectedCallLogEntries: [ParseDelegateCallLogEntry], expectedDiagnosticInfos: [(level: MacroExpressionDiagnostic.Level, kind: MacroExpressionDiagnostic.Kind, start: Int, end: Int)], sourceLocation: SourceLocation = #_sourceLocation) { From 3047927443983ab9b547cb8aba674bf04d8d6bef Mon Sep 17 00:00:00 2001 From: Ian Anderson Date: Tue, 3 Jun 2025 17:54:22 -0700 Subject: [PATCH 22/54] Test the Isystem flag before using -Isystem swift-build needs to support older versions of swift-frontend that don't support -Isystem. Check the driver and front end before using it. Also set TEST_LIBRARY_SEARCH_PATHS in SWIFT_SYSTEM_INCLUDE_PATHS rather than SWIFT_INCLUDE_PATHS, to match the handling for TEST_FRAMEWORK_SEARCH_PATHS -> SYSTEM_FRAMEWORK_SEARCH_PATHS. rdar://152540600 --- Sources/SWBCore/Settings/Settings.swift | 2 +- .../SpecImplementations/Tools/SwiftCompiler.swift | 7 ++++++- .../SwiftTaskConstructionTests.swift | 12 ++++++++++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Sources/SWBCore/Settings/Settings.swift b/Sources/SWBCore/Settings/Settings.swift index e2180bb4..0bfabcf0 100644 --- a/Sources/SWBCore/Settings/Settings.swift +++ b/Sources/SWBCore/Settings/Settings.swift @@ -4210,7 +4210,7 @@ private class SettingsBuilder { if scope.evaluate(BuiltinMacros.ENABLE_TESTING_SEARCH_PATHS) { table.push(BuiltinMacros.SYSTEM_FRAMEWORK_SEARCH_PATHS, BuiltinMacros.namespace.parseStringList(["$(inherited)", "$(TEST_FRAMEWORK_SEARCH_PATHS$(TEST_BUILD_STYLE))"])) table.push(BuiltinMacros.LIBRARY_SEARCH_PATHS, BuiltinMacros.namespace.parseStringList(["$(inherited)", "$(TEST_LIBRARY_SEARCH_PATHS$(TEST_BUILD_STYLE))"])) - table.push(BuiltinMacros.SWIFT_INCLUDE_PATHS, BuiltinMacros.namespace.parseStringList(["$(inherited)", "$(TEST_LIBRARY_SEARCH_PATHS$(TEST_BUILD_STYLE))"])) + table.push(BuiltinMacros.SWIFT_SYSTEM_INCLUDE_PATHS, BuiltinMacros.namespace.parseStringList(["$(inherited)", "$(TEST_LIBRARY_SEARCH_PATHS$(TEST_BUILD_STYLE))"])) // If the toolchain contains a copy of Swift Testing, prefer it. let toolchainPath = Path(scope.evaluateAsString(BuiltinMacros.TOOLCHAIN_DIR)) diff --git a/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift b/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift index 6a495ad6..70e9ad3a 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift @@ -1058,6 +1058,7 @@ public struct DiscoveredSwiftCompilerToolSpecInfo: DiscoveredCommandLineToolSpec case constExtractCompleteMetadata = "const-extract-complete-metadata" case emitPackageModuleInterfacePath = "emit-package-module-interface-path" case compilationCaching = "compilation-caching" + case Isystem = "Isystem" } public var toolFeatures: ToolFeatures public func hasFeature(_ flag: String) -> Bool { @@ -1573,7 +1574,11 @@ public final class SwiftCompilerSpec : CompilerSpec, SpecIdentifierType, SwiftDi } for searchPath in cbc.producer.expandedSearchPaths(for: BuiltinMacros.SWIFT_SYSTEM_INCLUDE_PATHS, scope: cbc.scope) { - args.append(contentsOf: ["-Isystem", searchPath]) + if LibSwiftDriver.supportsDriverFlag(spelled: "-Isystem") && toolSpecInfo.hasFeature(DiscoveredSwiftCompilerToolSpecInfo.FeatureFlag.Isystem.rawValue) { + args.append(contentsOf: ["-Isystem", searchPath]) + } else { + args.append(contentsOf: ["-I", searchPath]) + } } // Add -F for the effective framework search paths. diff --git a/Tests/SWBTaskConstructionTests/SwiftTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/SwiftTaskConstructionTests.swift index 79189a64..8899422e 100644 --- a/Tests/SWBTaskConstructionTests/SwiftTaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/SwiftTaskConstructionTests.swift @@ -214,7 +214,11 @@ fileprivate struct SwiftTaskConstructionTests: CoreBasedTests { results.checkTask(.matchTarget(target), .matchRuleType("SwiftDriver Compilation")) { task in task.checkRuleInfo(["SwiftDriver Compilation", target.target.name, "normal", "x86_64", "com.apple.xcode.tools.swift.compiler"]) - task.checkCommandLineContains([swiftCompilerPath.str, "-module-name", "AppTarget", "-O", "@\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget.SwiftFileList", "-sdk", core.loadSDK(.macOS).path.str, "-target", "x86_64-apple-macos\(MACOSX_DEPLOYMENT_TARGET)", /* options from the xcspec which sometimes change appear here */ "-swift-version", swiftVersion, "-I", "\(SRCROOT)/build/Debug", "-I", "/tmp/include", "-Isystem", "/tmp/system/include", "-F", "\(SRCROOT)/build/Debug", "-c", "-j\(compilerParallelismLevel)", "-incremental", "-output-file-map", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget-OutputFileMap.json", "-serialize-diagnostics", "-emit-dependencies", "-emit-module", "-emit-module-path", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget.swiftmodule", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/swift-overrides.hmap", "-Xcc", "-iquote", "-Xcc", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-generated-files.hmap", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-own-target-headers.hmap", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-all-target-headers.hmap", "-Xcc", "-iquote", "-Xcc", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-project-headers.hmap", "-Xcc", "-I\(SRCROOT)/build/Debug/include", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/DerivedSources-normal/x86_64", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/DerivedSources", "-emit-objc-header", "-emit-objc-header-path", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget-Swift.h", "-working-directory", SRCROOT]) + if LibSwiftDriver.supportsDriverFlag(spelled: "-Isystem") && swiftFeatures.has(.Isystem) { + task.checkCommandLineContains([swiftCompilerPath.str, "-module-name", "AppTarget", "-O", "@\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget.SwiftFileList", "-sdk", core.loadSDK(.macOS).path.str, "-target", "x86_64-apple-macos\(MACOSX_DEPLOYMENT_TARGET)", /* options from the xcspec which sometimes change appear here */ "-swift-version", swiftVersion, "-I", "\(SRCROOT)/build/Debug", "-I", "/tmp/include", "-Isystem", "/tmp/system/include", "-F", "\(SRCROOT)/build/Debug", "-c", "-j\(compilerParallelismLevel)", "-incremental", "-output-file-map", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget-OutputFileMap.json", "-serialize-diagnostics", "-emit-dependencies", "-emit-module", "-emit-module-path", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget.swiftmodule", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/swift-overrides.hmap", "-Xcc", "-iquote", "-Xcc", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-generated-files.hmap", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-own-target-headers.hmap", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-all-target-headers.hmap", "-Xcc", "-iquote", "-Xcc", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-project-headers.hmap", "-Xcc", "-I\(SRCROOT)/build/Debug/include", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/DerivedSources-normal/x86_64", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/DerivedSources", "-emit-objc-header", "-emit-objc-header-path", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget-Swift.h", "-working-directory", SRCROOT]) + } else { + task.checkCommandLineContains([swiftCompilerPath.str, "-module-name", "AppTarget", "-O", "@\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget.SwiftFileList", "-sdk", core.loadSDK(.macOS).path.str, "-target", "x86_64-apple-macos\(MACOSX_DEPLOYMENT_TARGET)", /* options from the xcspec which sometimes change appear here */ "-swift-version", swiftVersion, "-I", "\(SRCROOT)/build/Debug", "-I", "/tmp/include", "-I", "/tmp/system/include", "-F", "\(SRCROOT)/build/Debug", "-c", "-j\(compilerParallelismLevel)", "-incremental", "-output-file-map", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget-OutputFileMap.json", "-serialize-diagnostics", "-emit-dependencies", "-emit-module", "-emit-module-path", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget.swiftmodule", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/swift-overrides.hmap", "-Xcc", "-iquote", "-Xcc", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-generated-files.hmap", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-own-target-headers.hmap", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-all-target-headers.hmap", "-Xcc", "-iquote", "-Xcc", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-project-headers.hmap", "-Xcc", "-I\(SRCROOT)/build/Debug/include", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/DerivedSources-normal/x86_64", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/DerivedSources", "-emit-objc-header", "-emit-objc-header-path", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget-Swift.h", "-working-directory", SRCROOT]) + } task.checkInputs([ .path("\(SRCROOT)/main.swift"), @@ -247,7 +251,11 @@ fileprivate struct SwiftTaskConstructionTests: CoreBasedTests { results.checkTask(.matchTarget(target), .matchRuleType("SwiftDriver Compilation Requirements")) { task in task.checkRuleInfo(["SwiftDriver Compilation Requirements", target.target.name, "normal", "x86_64", "com.apple.xcode.tools.swift.compiler"]) - task.checkCommandLineContains([swiftCompilerPath.str, "-module-name", "AppTarget", "-O", "@\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget.SwiftFileList", "-sdk", core.loadSDK(.macOS).path.str, "-target", "x86_64-apple-macos\(MACOSX_DEPLOYMENT_TARGET)", /* options from the xcspec which sometimes change appear here */ "-swift-version", swiftVersion, "-I", "\(SRCROOT)/build/Debug", "-I", "/tmp/include", "-Isystem", "/tmp/system/include", "-F", "\(SRCROOT)/build/Debug", "-c", "-j\(compilerParallelismLevel)", "-incremental", "-output-file-map", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget-OutputFileMap.json", "-serialize-diagnostics", "-emit-dependencies", "-emit-module", "-emit-module-path", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget.swiftmodule", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/swift-overrides.hmap", "-Xcc", "-iquote", "-Xcc", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-generated-files.hmap", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-own-target-headers.hmap", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-all-target-headers.hmap", "-Xcc", "-iquote", "-Xcc", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-project-headers.hmap", "-Xcc", "-I\(SRCROOT)/build/Debug/include", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/DerivedSources-normal/x86_64", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/DerivedSources", "-emit-objc-header", "-emit-objc-header-path", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget-Swift.h", "-working-directory", SRCROOT]) + if LibSwiftDriver.supportsDriverFlag(spelled: "-Isystem") && swiftFeatures.has(.Isystem) { + task.checkCommandLineContains([swiftCompilerPath.str, "-module-name", "AppTarget", "-O", "@\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget.SwiftFileList", "-sdk", core.loadSDK(.macOS).path.str, "-target", "x86_64-apple-macos\(MACOSX_DEPLOYMENT_TARGET)", /* options from the xcspec which sometimes change appear here */ "-swift-version", swiftVersion, "-I", "\(SRCROOT)/build/Debug", "-I", "/tmp/include", "-Isystem", "/tmp/system/include", "-F", "\(SRCROOT)/build/Debug", "-c", "-j\(compilerParallelismLevel)", "-incremental", "-output-file-map", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget-OutputFileMap.json", "-serialize-diagnostics", "-emit-dependencies", "-emit-module", "-emit-module-path", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget.swiftmodule", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/swift-overrides.hmap", "-Xcc", "-iquote", "-Xcc", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-generated-files.hmap", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-own-target-headers.hmap", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-all-target-headers.hmap", "-Xcc", "-iquote", "-Xcc", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-project-headers.hmap", "-Xcc", "-I\(SRCROOT)/build/Debug/include", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/DerivedSources-normal/x86_64", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/DerivedSources", "-emit-objc-header", "-emit-objc-header-path", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget-Swift.h", "-working-directory", SRCROOT]) + } else { + task.checkCommandLineContains([swiftCompilerPath.str, "-module-name", "AppTarget", "-O", "@\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget.SwiftFileList", "-sdk", core.loadSDK(.macOS).path.str, "-target", "x86_64-apple-macos\(MACOSX_DEPLOYMENT_TARGET)", /* options from the xcspec which sometimes change appear here */ "-swift-version", swiftVersion, "-I", "\(SRCROOT)/build/Debug", "-I", "/tmp/include", "-I", "/tmp/system/include", "-F", "\(SRCROOT)/build/Debug", "-c", "-j\(compilerParallelismLevel)", "-incremental", "-output-file-map", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget-OutputFileMap.json", "-serialize-diagnostics", "-emit-dependencies", "-emit-module", "-emit-module-path", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget.swiftmodule", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/swift-overrides.hmap", "-Xcc", "-iquote", "-Xcc", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-generated-files.hmap", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-own-target-headers.hmap", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-all-target-headers.hmap", "-Xcc", "-iquote", "-Xcc", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/AppTarget-project-headers.hmap", "-Xcc", "-I\(SRCROOT)/build/Debug/include", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/DerivedSources-normal/x86_64", "-Xcc", "-I\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/DerivedSources", "-emit-objc-header", "-emit-objc-header-path", "\(SRCROOT)/build/aProject.build/Debug/AppTarget.build/Objects-normal/x86_64/AppTarget-Swift.h", "-working-directory", SRCROOT]) + } task.checkInputs([ .path("\(SRCROOT)/main.swift"), From c1f1cf351b4770956e70fd7a051d7b897d37b275 Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Wed, 4 Jun 2025 13:17:38 -0700 Subject: [PATCH 23/54] Revert "XCTest discovery support for non-Darwin platforms" --- .../Specs/DarwinProductTypes.xcspec | 13 - Sources/SWBCSupport/IndexStore.h | 194 ------ Sources/SWBCSupport/SWBCSupport.h | 1 - Sources/SWBCore/Settings/BuiltinMacros.swift | 2 - .../SpecImplementations/ProductTypes.swift | 10 +- .../Tools/SwiftCompiler.swift | 3 - .../SWBGenericUnixPlatform/Specs/Unix.xcspec | 35 +- .../SWBProjectModel/PIFGenerationModel.swift | 2 - Sources/SWBQNXPlatform/Specs/QNX.xcspec | 33 +- .../ProductPlanning/ProductPlan.swift | 2 +- .../InfoPlistTaskProducer.swift | 2 +- .../SwiftStandardLibrariesTaskProducer.swift | 2 +- Sources/SWBTestSupport/TestWorkspaces.swift | 6 +- .../Specs/ProductTypes.xcspec | 15 - .../TestEntryPointGenerationTaskAction.swift | 603 +----------------- .../TestEntryPointGenerationTool.swift | 52 -- .../TestEntryPointTaskProducer.swift | 51 +- Sources/SWBUtil/CMakeLists.txt | 1 - Sources/SWBUtil/IndexStore.swift | 389 ----------- .../SWBWindowsPlatform/Specs/Windows.xcspec | 25 - .../ProjectModel/BuildSettings.swift | 2 - Sources/SwiftBuild/ProjectModel/Targets.swift | 1 - .../BuildOperationTests.swift | 52 +- .../UnitTestTaskConstructionTests.swift | 35 +- 24 files changed, 89 insertions(+), 1442 deletions(-) delete mode 100644 Sources/SWBCSupport/IndexStore.h delete mode 100644 Sources/SWBUtil/IndexStore.swift diff --git a/Sources/SWBApplePlatform/Specs/DarwinProductTypes.xcspec b/Sources/SWBApplePlatform/Specs/DarwinProductTypes.xcspec index d6e43e3f..e1bc5a17 100644 --- a/Sources/SWBApplePlatform/Specs/DarwinProductTypes.xcspec +++ b/Sources/SWBApplePlatform/Specs/DarwinProductTypes.xcspec @@ -461,17 +461,4 @@ ); Platforms = (driverkit); }, - { - _Domain = darwin; - Type = ProductType; - Identifier = com.apple.product-type.tool.swiftpm-test-runner; - Name = "SwiftPM Unit Test Runner"; - Description = "SwiftPM Unit Test Runner"; - DefaultBuildProperties = { - __SKIP_BUILD = YES; - }; - PackageTypes = ( - com.apple.package-type.mach-o-executable - ); - }, ) diff --git a/Sources/SWBCSupport/IndexStore.h b/Sources/SWBCSupport/IndexStore.h deleted file mode 100644 index 7d4b77b8..00000000 --- a/Sources/SWBCSupport/IndexStore.h +++ /dev/null @@ -1,194 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift open source project -// -// Copyright (c) 2025 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See http://swift.org/LICENSE.txt for license information -// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -#ifndef INDEXSTORE_H -#define INDEXSTORE_H - -#include -#include -#include -#include - -typedef void *indexstore_error_t; - -typedef struct { - const char *data; - size_t length; -} indexstore_string_ref_t; - -typedef void *indexstore_t; -typedef void *indexstore_symbol_t; - -typedef enum { - INDEXSTORE_SYMBOL_KIND_UNKNOWN = 0, - INDEXSTORE_SYMBOL_KIND_MODULE = 1, - INDEXSTORE_SYMBOL_KIND_NAMESPACE = 2, - INDEXSTORE_SYMBOL_KIND_NAMESPACEALIAS = 3, - INDEXSTORE_SYMBOL_KIND_MACRO = 4, - INDEXSTORE_SYMBOL_KIND_ENUM = 5, - INDEXSTORE_SYMBOL_KIND_STRUCT = 6, - INDEXSTORE_SYMBOL_KIND_CLASS = 7, - INDEXSTORE_SYMBOL_KIND_PROTOCOL = 8, - INDEXSTORE_SYMBOL_KIND_EXTENSION = 9, - INDEXSTORE_SYMBOL_KIND_UNION = 10, - INDEXSTORE_SYMBOL_KIND_TYPEALIAS = 11, - INDEXSTORE_SYMBOL_KIND_FUNCTION = 12, - INDEXSTORE_SYMBOL_KIND_VARIABLE = 13, - INDEXSTORE_SYMBOL_KIND_FIELD = 14, - INDEXSTORE_SYMBOL_KIND_ENUMCONSTANT = 15, - INDEXSTORE_SYMBOL_KIND_INSTANCEMETHOD = 16, - INDEXSTORE_SYMBOL_KIND_CLASSMETHOD = 17, - INDEXSTORE_SYMBOL_KIND_STATICMETHOD = 18, - INDEXSTORE_SYMBOL_KIND_INSTANCEPROPERTY = 19, - INDEXSTORE_SYMBOL_KIND_CLASSPROPERTY = 20, - INDEXSTORE_SYMBOL_KIND_STATICPROPERTY = 21, - INDEXSTORE_SYMBOL_KIND_CONSTRUCTOR = 22, - INDEXSTORE_SYMBOL_KIND_DESTRUCTOR = 23, - INDEXSTORE_SYMBOL_KIND_CONVERSIONFUNCTION = 24, - INDEXSTORE_SYMBOL_KIND_PARAMETER = 25, - INDEXSTORE_SYMBOL_KIND_USING = 26, - - INDEXSTORE_SYMBOL_KIND_COMMENTTAG = 1000, -} indexstore_symbol_kind_t; - -typedef enum { - INDEXSTORE_SYMBOL_PROPERTY_GENERIC = 1 << 0, - INDEXSTORE_SYMBOL_PROPERTY_TEMPLATE_PARTIAL_SPECIALIZATION = 1 << 1, - INDEXSTORE_SYMBOL_PROPERTY_TEMPLATE_SPECIALIZATION = 1 << 2, - INDEXSTORE_SYMBOL_PROPERTY_UNITTEST = 1 << 3, - INDEXSTORE_SYMBOL_PROPERTY_IBANNOTATED = 1 << 4, - INDEXSTORE_SYMBOL_PROPERTY_IBOUTLETCOLLECTION = 1 << 5, - INDEXSTORE_SYMBOL_PROPERTY_GKINSPECTABLE = 1 << 6, - INDEXSTORE_SYMBOL_PROPERTY_LOCAL = 1 << 7, - INDEXSTORE_SYMBOL_PROPERTY_PROTOCOL_INTERFACE = 1 << 8, - INDEXSTORE_SYMBOL_PROPERTY_SWIFT_ASYNC = 1 << 16, -} indexstore_symbol_property_t; - -typedef enum { - INDEXSTORE_SYMBOL_ROLE_DECLARATION = 1 << 0, - INDEXSTORE_SYMBOL_ROLE_DEFINITION = 1 << 1, - INDEXSTORE_SYMBOL_ROLE_REFERENCE = 1 << 2, - INDEXSTORE_SYMBOL_ROLE_READ = 1 << 3, - INDEXSTORE_SYMBOL_ROLE_WRITE = 1 << 4, - INDEXSTORE_SYMBOL_ROLE_CALL = 1 << 5, - INDEXSTORE_SYMBOL_ROLE_DYNAMIC = 1 << 6, - INDEXSTORE_SYMBOL_ROLE_ADDRESSOF = 1 << 7, - INDEXSTORE_SYMBOL_ROLE_IMPLICIT = 1 << 8, - INDEXSTORE_SYMBOL_ROLE_UNDEFINITION = 1 << 19, - - // Relation roles. - INDEXSTORE_SYMBOL_ROLE_REL_CHILDOF = 1 << 9, - INDEXSTORE_SYMBOL_ROLE_REL_BASEOF = 1 << 10, - INDEXSTORE_SYMBOL_ROLE_REL_OVERRIDEOF = 1 << 11, - INDEXSTORE_SYMBOL_ROLE_REL_RECEIVEDBY = 1 << 12, - INDEXSTORE_SYMBOL_ROLE_REL_CALLEDBY = 1 << 13, - INDEXSTORE_SYMBOL_ROLE_REL_EXTENDEDBY = 1 << 14, - INDEXSTORE_SYMBOL_ROLE_REL_ACCESSOROF = 1 << 15, - INDEXSTORE_SYMBOL_ROLE_REL_CONTAINEDBY = 1 << 16, - INDEXSTORE_SYMBOL_ROLE_REL_IBTYPEOF = 1 << 17, - INDEXSTORE_SYMBOL_ROLE_REL_SPECIALIZATIONOF = 1 << 18, -} indexstore_symbol_role_t; - -typedef void *indexstore_unit_dependency_t; - -typedef enum { - INDEXSTORE_UNIT_DEPENDENCY_UNIT = 1, - INDEXSTORE_UNIT_DEPENDENCY_RECORD = 2, - INDEXSTORE_UNIT_DEPENDENCY_FILE = 3, -} indexstore_unit_dependency_kind_t; - -typedef void *indexstore_symbol_relation_t; -typedef void *indexstore_occurrence_t; -typedef void *indexstore_record_reader_t; -typedef void *indexstore_unit_reader_t; - -typedef struct { - const char * - (*error_get_description)(indexstore_error_t); - - void - (*error_dispose)(indexstore_error_t); - - indexstore_t - (*store_create)(const char *store_path, indexstore_error_t *error); - - void - (*store_dispose)(indexstore_t); - - size_t - (*store_get_unit_name_from_output_path)(indexstore_t store, - const char *output_path, - char *name_buf, - size_t buf_size); - - indexstore_symbol_kind_t - (*symbol_get_kind)(indexstore_symbol_t); - - uint64_t - (*symbol_get_properties)(indexstore_symbol_t); - - indexstore_string_ref_t - (*symbol_get_name)(indexstore_symbol_t); - - uint64_t - (*symbol_relation_get_roles)(indexstore_symbol_relation_t); - - indexstore_symbol_t - (*symbol_relation_get_symbol)(indexstore_symbol_relation_t); - - indexstore_symbol_t - (*occurrence_get_symbol)(indexstore_occurrence_t); - - bool - (*occurrence_relations_apply_f)(indexstore_occurrence_t, - void *context, - bool(*applier)(void *context, indexstore_symbol_relation_t symbol_rel)); - - indexstore_record_reader_t - (*record_reader_create)(indexstore_t store, const char *record_name, - indexstore_error_t *error); - - void - (*record_reader_dispose)(indexstore_record_reader_t); - - bool - (*record_reader_occurrences_apply_f)(indexstore_record_reader_t, - void *context, - bool(*applier)(void *context, indexstore_occurrence_t occur)); - - indexstore_unit_reader_t - (*unit_reader_create)(indexstore_t store, const char *unit_name, - indexstore_error_t *error); - - void - (*unit_reader_dispose)(indexstore_unit_reader_t); - - indexstore_string_ref_t - (*unit_reader_get_module_name)(indexstore_unit_reader_t); - - indexstore_unit_dependency_kind_t - (*unit_dependency_get_kind)(indexstore_unit_dependency_t); - - indexstore_string_ref_t - (*unit_dependency_get_name)(indexstore_unit_dependency_t); - - bool - (*unit_reader_dependencies_apply)(indexstore_unit_reader_t, - bool(^applier)(indexstore_unit_dependency_t)); - - bool - (*unit_reader_dependencies_apply_f)(indexstore_unit_reader_t, - void *context, - bool(*applier)(void *context, indexstore_unit_dependency_t)); -} indexstore_functions_t; - -#endif diff --git a/Sources/SWBCSupport/SWBCSupport.h b/Sources/SWBCSupport/SWBCSupport.h index 18591c94..c020472c 100644 --- a/Sources/SWBCSupport/SWBCSupport.h +++ b/Sources/SWBCSupport/SWBCSupport.h @@ -21,7 +21,6 @@ #include "CLibclang.h" #include "CLibRemarksHelper.h" -#include "IndexStore.h" #include "PluginAPI.h" #include "PluginAPI_functions.h" #include "PluginAPI_types.h" diff --git a/Sources/SWBCore/Settings/BuiltinMacros.swift b/Sources/SWBCore/Settings/BuiltinMacros.swift index 52beb989..0ac934ff 100644 --- a/Sources/SWBCore/Settings/BuiltinMacros.swift +++ b/Sources/SWBCore/Settings/BuiltinMacros.swift @@ -750,7 +750,6 @@ public final class BuiltinMacros { public static let INDEX_PREPARED_TARGET_MARKER_PATH = BuiltinMacros.declareStringMacro("INDEX_PREPARED_TARGET_MARKER_PATH") public static let INDEX_REGULAR_BUILD_PRODUCTS_DIR = BuiltinMacros.declareStringMacro("INDEX_REGULAR_BUILD_PRODUCTS_DIR") public static let INDEX_REGULAR_BUILD_INTERMEDIATES_DIR = BuiltinMacros.declareStringMacro("INDEX_REGULAR_BUILD_INTERMEDIATES_DIR") - public static let INDEX_STORE_LIBRARY_PATH = BuiltinMacros.declarePathMacro("INDEX_STORE_LIBRARY_PATH") public static let INFOPLIST_ENFORCE_MINIMUM_OS = BuiltinMacros.declareBooleanMacro("INFOPLIST_ENFORCE_MINIMUM_OS") public static let INFOPLIST_EXPAND_BUILD_SETTINGS = BuiltinMacros.declareBooleanMacro("INFOPLIST_EXPAND_BUILD_SETTINGS") public static let INFOPLIST_FILE = BuiltinMacros.declarePathMacro("INFOPLIST_FILE") @@ -1797,7 +1796,6 @@ public final class BuiltinMacros { INDEX_PREPARED_TARGET_MARKER_PATH, INDEX_REGULAR_BUILD_PRODUCTS_DIR, INDEX_REGULAR_BUILD_INTERMEDIATES_DIR, - INDEX_STORE_LIBRARY_PATH, INDEX_ENABLE_DATA_STORE, INDEX_PRECOMPS_DIR, INFOPLIST_ENFORCE_MINIMUM_OS, diff --git a/Sources/SWBCore/SpecImplementations/ProductTypes.swift b/Sources/SWBCore/SpecImplementations/ProductTypes.swift index 28415e4b..7cb6a1d5 100644 --- a/Sources/SWBCore/SpecImplementations/ProductTypes.swift +++ b/Sources/SWBCore/SpecImplementations/ProductTypes.swift @@ -321,7 +321,7 @@ public class ProductTypeSpec : Spec, SpecType, @unchecked Sendable { } /// Returns whether the product type supports embedding Swift standard libraries inside it. - public func supportsEmbeddingSwiftStandardLibraries(producer: CommandProducer) -> Bool { + public var supportsEmbeddingSwiftStandardLibraries: Bool { // Most product types don't support having the Swift libraries embedded in them. return false } @@ -381,7 +381,7 @@ public final class ApplicationProductTypeSpec : BundleProductTypeSpec, @unchecke return "PBXApplicationProductType" } - public override func supportsEmbeddingSwiftStandardLibraries(producer: CommandProducer) -> Bool { + public override var supportsEmbeddingSwiftStandardLibraries: Bool { return true } @@ -602,8 +602,8 @@ public final class XCTestBundleProductTypeSpec : BundleProductTypeSpec, @uncheck super.init(parser, basedOnSpec) } - public override func supportsEmbeddingSwiftStandardLibraries(producer: CommandProducer) -> Bool { - return producer.isApplePlatform + public override var supportsEmbeddingSwiftStandardLibraries: Bool { + return true } public class func usesXCTRunner(_ scope: MacroEvaluationScope) -> Bool { @@ -649,7 +649,7 @@ public final class XCTestBundleProductTypeSpec : BundleProductTypeSpec, @uncheck var (tableOpt, warnings, errors) = super.overridingBuildSettings(scope, platform: platform) var table = tableOpt ?? MacroValueAssignmentTable(namespace: scope.namespace) - let isDeviceBuild = platform?.isDeploymentPlatform == true && platform?.name != scope.evaluate(BuiltinMacros.HOST_PLATFORM) + let isDeviceBuild = platform?.isDeploymentPlatform == true && platform?.identifier != "com.apple.platform.macosx" if isDeviceBuild { // For tests running on devices (not simulators) we always want to generate dSYMs so that symbolication can give file and line information about test failures. table.push(BuiltinMacros.DEBUG_INFORMATION_FORMAT, literal: "dwarf-with-dsym") diff --git a/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift b/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift index 687820ec..70e9ad3a 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift @@ -3763,9 +3763,6 @@ public extension BuildPhaseWithBuildFiles { /// - Returns: If the build phase contains any Swift source files that are not filtered out via the platform filter or excluded source file name patterns. func containsSwiftSources(_ referenceLookupContext: any ReferenceLookupContext, _ specLookupContext: any SpecLookupContext, _ scope: MacroEvaluationScope, _ filePathResolver: FilePathResolver) -> Bool { guard let swiftFileType = specLookupContext.lookupFileType(identifier: "sourcecode.swift") else { return false } - if scope.evaluate(BuiltinMacros.GENERATE_TEST_ENTRY_POINT) { - return true - } return containsFiles(ofType: swiftFileType, referenceLookupContext, specLookupContext, scope, filePathResolver) } } diff --git a/Sources/SWBGenericUnixPlatform/Specs/Unix.xcspec b/Sources/SWBGenericUnixPlatform/Specs/Unix.xcspec index 00bc75d5..eb72ba9c 100644 --- a/Sources/SWBGenericUnixPlatform/Specs/Unix.xcspec +++ b/Sources/SWBGenericUnixPlatform/Specs/Unix.xcspec @@ -24,18 +24,41 @@ SortNumber = 0; }, + // Test type bundle (bodged to be a tool) { Domain = generic-unix; Type = ProductType; Identifier = com.apple.product-type.bundle.unit-test; - BasedOn = com.apple.product-type.library.dynamic; + Class = PBXToolProductType; + Name = "Command-line Tool"; + Description = "Standalone command-line tool"; + DefaultTargetName = "Command-line Tool"; DefaultBuildProperties = { - // Index store data is required to discover XCTest tests - COMPILER_INDEX_STORE_ENABLE = YES; - SWIFT_INDEX_STORE_ENABLE = YES; - // Testability is needed to generate code to invoke discovered XCTest tests - SWIFT_ENABLE_TESTABILITY = YES; + FULL_PRODUCT_NAME = "$(EXECUTABLE_NAME)"; + EXECUTABLE_PREFIX = ""; + EXECUTABLE_SUFFIX = ".xctest"; + REZ_EXECUTABLE = YES; + INSTALL_PATH = "/usr/local/bin"; + FRAMEWORK_FLAG_PREFIX = "-framework"; + LIBRARY_FLAG_PREFIX = "-l"; + LIBRARY_FLAG_NOSPACE = YES; + GCC_DYNAMIC_NO_PIC = NO; + LD_NO_PIE = NO; + GCC_SYMBOLS_PRIVATE_EXTERN = YES; + GCC_INLINES_ARE_PRIVATE_EXTERN = YES; + STRIP_STYLE = "all"; + CODE_SIGNING_ALLOWED = NO; + IsUnitTest = YES; + SWIFT_FORCE_DYNAMIC_LINK_STDLIB = YES; + SWIFT_FORCE_STATIC_LINK_STDLIB = NO; + // Avoid warning for executable types + ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = NO; + GENERATE_TEST_ENTRY_POINT = YES; + GENERATED_TEST_ENTRY_POINT_PATH = "$(DERIVED_SOURCES_DIR)/test_entry_point.swift"; }; + PackageTypes = ( + com.apple.package-type.mach-o-executable // default + ); }, // Dynamic library (masquerading as a framework to placate Swift's project structure) diff --git a/Sources/SWBProjectModel/PIFGenerationModel.swift b/Sources/SWBProjectModel/PIFGenerationModel.swift index f3db2bce..698a7343 100644 --- a/Sources/SWBProjectModel/PIFGenerationModel.swift +++ b/Sources/SWBProjectModel/PIFGenerationModel.swift @@ -295,7 +295,6 @@ public enum PIF { case executable = "com.apple.product-type.tool" case hostBuildTool = "com.apple.product-type.tool.host-build" case unitTest = "com.apple.product-type.bundle.unit-test" - case swiftpmTestRunner = "com.apple.product-type.tool.swiftpm-test-runner" case bundle = "com.apple.product-type.bundle" case packageProduct = "packageProduct" public var asString: String { return rawValue } @@ -1023,7 +1022,6 @@ public enum PIF { public var SWIFT_ADD_TOOLCHAIN_SWIFTSYNTAX_SEARCH_PATHS: String? public var SWIFT_FORCE_STATIC_LINK_STDLIB: String? public var SWIFT_FORCE_DYNAMIC_LINK_STDLIB: String? - public var SWIFT_INDEX_STORE_ENABLE: String? public var SWIFT_INSTALL_OBJC_HEADER: String? public var SWIFT_LOAD_BINARY_MACROS: [String]? public var SWIFT_MODULE_ALIASES: [String]? diff --git a/Sources/SWBQNXPlatform/Specs/QNX.xcspec b/Sources/SWBQNXPlatform/Specs/QNX.xcspec index aea12e3b..3c72c620 100644 --- a/Sources/SWBQNXPlatform/Specs/QNX.xcspec +++ b/Sources/SWBQNXPlatform/Specs/QNX.xcspec @@ -24,18 +24,39 @@ SortNumber = 0; }, + // Test type bundle (bodged to be a tool) { Domain = qnx; Type = ProductType; Identifier = com.apple.product-type.bundle.unit-test; - BasedOn = com.apple.product-type.library.dynamic; + Class = PBXToolProductType; + Name = "Command-line Tool"; + Description = "Standalone command-line tool"; + DefaultTargetName = "Command-line Tool"; DefaultBuildProperties = { - // Index store data is required to discover XCTest tests - COMPILER_INDEX_STORE_ENABLE = YES; - SWIFT_INDEX_STORE_ENABLE = YES; - // Testability is needed to generate code to invoke discovered XCTest tests - SWIFT_ENABLE_TESTABILITY = YES; + FULL_PRODUCT_NAME = "$(EXECUTABLE_NAME)"; + EXECUTABLE_PREFIX = ""; + EXECUTABLE_SUFFIX = ".xctest"; + REZ_EXECUTABLE = YES; + INSTALL_PATH = "/usr/local/bin"; + FRAMEWORK_FLAG_PREFIX = "-framework"; + LIBRARY_FLAG_PREFIX = "-l"; + LIBRARY_FLAG_NOSPACE = YES; + GCC_DYNAMIC_NO_PIC = NO; + LD_NO_PIE = NO; + GCC_SYMBOLS_PRIVATE_EXTERN = YES; + GCC_INLINES_ARE_PRIVATE_EXTERN = YES; + STRIP_STYLE = "all"; + CODE_SIGNING_ALLOWED = NO; + IsUnitTest = YES; + SWIFT_FORCE_DYNAMIC_LINK_STDLIB = YES; + SWIFT_FORCE_STATIC_LINK_STDLIB = NO; + // Avoid warning for executable types + ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = NO; }; + PackageTypes = ( + com.apple.package-type.mach-o-executable // default + ); }, // Dynamic library (masquerading as a framework to placate Swift's project structure) diff --git a/Sources/SWBTaskConstruction/ProductPlanning/ProductPlan.swift b/Sources/SWBTaskConstruction/ProductPlanning/ProductPlan.swift index 883073b0..88d2edfe 100644 --- a/Sources/SWBTaskConstruction/ProductPlanning/ProductPlan.swift +++ b/Sources/SWBTaskConstruction/ProductPlanning/ProductPlan.swift @@ -31,7 +31,7 @@ package protocol GlobalProductPlanDelegate: CoreClientTargetDiagnosticProducingD package final class GlobalProductPlan: GlobalTargetInfoProvider { /// The build plan request. - package let planRequest: BuildPlanRequest + let planRequest: BuildPlanRequest /// The target task info for each configured target. private(set) var targetTaskInfos: [ConfiguredTarget: TargetTaskInfo] diff --git a/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/InfoPlistTaskProducer.swift b/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/InfoPlistTaskProducer.swift index 820a55f5..1f740497 100644 --- a/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/InfoPlistTaskProducer.swift +++ b/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/InfoPlistTaskProducer.swift @@ -57,7 +57,7 @@ private extension ProductTypeSpec break } - fatalError("unknown product type \(type(of: self))") + fatalError("unknown product type") } } diff --git a/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/SwiftStandardLibrariesTaskProducer.swift b/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/SwiftStandardLibrariesTaskProducer.swift index c74e1843..bba8b051 100644 --- a/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/SwiftStandardLibrariesTaskProducer.swift +++ b/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/SwiftStandardLibrariesTaskProducer.swift @@ -41,7 +41,7 @@ final class SwiftStandardLibrariesTaskProducer: PhasedTaskProducer, TaskProducer let buildingAnySwiftSourceFiles = (context.configuredTarget?.target as? BuildPhaseTarget)?.sourcesBuildPhase?.containsSwiftSources(context.workspaceContext.workspace, context, scope, context.filePathResolver) ?? false // Determine whether we want to embed swift libraries. - var shouldEmbedSwiftLibraries = (buildingAnySwiftSourceFiles && productType.supportsEmbeddingSwiftStandardLibraries(producer: context)) + var shouldEmbedSwiftLibraries = (buildingAnySwiftSourceFiles && productType.supportsEmbeddingSwiftStandardLibraries) // If ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES then we will override our earlier reasoning if the product is a wrapper. if !shouldEmbedSwiftLibraries && scope.evaluate(BuiltinMacros.ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES) { diff --git a/Sources/SWBTestSupport/TestWorkspaces.swift b/Sources/SWBTestSupport/TestWorkspaces.swift index 6837bcf7..c225eb2c 100644 --- a/Sources/SWBTestSupport/TestWorkspaces.swift +++ b/Sources/SWBTestSupport/TestWorkspaces.swift @@ -924,7 +924,6 @@ package final class TestStandardTarget: TestInternalTarget, Sendable { case extensionKitExtension case xcodeExtension case unitTest - case swiftpmTestRunner case uiTest case multiDeviceUITest case systemExtension @@ -973,8 +972,6 @@ package final class TestStandardTarget: TestInternalTarget, Sendable { return "com.apple.product-type.xcode-extension" case .unitTest: return "com.apple.product-type.bundle.unit-test" - case .swiftpmTestRunner: - return "com.apple.product-type.tool.swiftpm-test-runner" case .uiTest: return "com.apple.product-type.bundle.ui-testing" case .multiDeviceUITest: @@ -1018,8 +1015,7 @@ package final class TestStandardTarget: TestInternalTarget, Sendable { .appClip: return "\(name).app" case .commandLineTool, - .hostBuildTool, - .swiftpmTestRunner: + .hostBuildTool: return "\(name)" case .framework, .staticFramework: diff --git a/Sources/SWBUniversalPlatform/Specs/ProductTypes.xcspec b/Sources/SWBUniversalPlatform/Specs/ProductTypes.xcspec index bd6cf9c8..0067050f 100644 --- a/Sources/SWBUniversalPlatform/Specs/ProductTypes.xcspec +++ b/Sources/SWBUniversalPlatform/Specs/ProductTypes.xcspec @@ -312,19 +312,4 @@ IsUnitTest = YES; WantsBundleIdentifierEditing = NO; }, - // SwiftPM test runner - { Type = ProductType; - Identifier = com.apple.product-type.tool.swiftpm-test-runner; - BasedOn = com.apple.product-type.tool; - Name = "SwiftPM Unit Test Runner"; - Description = "SwiftPM Unit Test Runner"; - DefaultBuildProperties = { - ENABLE_TESTING_SEARCH_PATHS = YES; - GENERATE_TEST_ENTRY_POINT = YES; - GENERATED_TEST_ENTRY_POINT_PATH = "$(DERIVED_SOURCES_DIR)/test_entry_point.swift"; - }; - PackageTypes = ( - com.apple.package-type.mach-o-executable - ); - }, ) diff --git a/Sources/SWBUniversalPlatform/TestEntryPointGenerationTaskAction.swift b/Sources/SWBUniversalPlatform/TestEntryPointGenerationTaskAction.swift index 9765719a..57dff473 100644 --- a/Sources/SWBUniversalPlatform/TestEntryPointGenerationTaskAction.swift +++ b/Sources/SWBUniversalPlatform/TestEntryPointGenerationTaskAction.swift @@ -23,37 +23,10 @@ class TestEntryPointGenerationTaskAction: TaskAction { override func performTaskAction(_ task: any ExecutableTask, dynamicExecutionDelegate: any DynamicTaskExecutionDelegate, executionDelegate: any TaskExecutionDelegate, clientDelegate: any TaskExecutionClientDelegate, outputDelegate: any TaskOutputDelegate) async -> CommandResult { do { let options = try Options.parse(Array(task.commandLineAsStrings.dropFirst())) - - var tests: [IndexStore.TestCaseClass] = [] - var objects: [Path] = [] - for linkerFilelist in options.linkerFilelist { - let filelistContents = String(String(decoding: try executionDelegate.fs.read(linkerFilelist), as: UTF8.self)) - let entries = filelistContents.split(separator: "\n", omittingEmptySubsequences: true).map { Path($0) }.map { - for indexUnitBasePath in options.indexUnitBasePath { - if let remappedPath = generateIndexOutputPath(from: $0, basePath: indexUnitBasePath) { - return remappedPath - } - } - return $0 - } - objects.append(contentsOf: entries) - } - let indexStoreAPI = try IndexStoreAPI(dylib: options.indexStoreLibraryPath) - for indexStore in options.indexStore { - let store = try IndexStore.open(store: indexStore, api: indexStoreAPI) - let testInfo = try store.listTests(in: objects) - tests.append(contentsOf: testInfo) - } - - try executionDelegate.fs.write(options.output, contents: ByteString(encodingAsUTF8: """ + try executionDelegate.fs.write(options.output, contents: #""" #if canImport(Testing) import Testing #endif - - \(testObservationFragment) - - import XCTest - \(discoveredTestsFragment(tests: tests)) @main @available(macOS 10.15, iOS 11, watchOS 4, tvOS 11, visionOS 1, *) @@ -71,16 +44,6 @@ class TestEntryPointGenerationTaskAction: TaskAction { return "xctest" } - private static func testOutputPath() -> String? { - var iterator = CommandLine.arguments.makeIterator() - while let argument = iterator.next() { - if argument == "--testing-output-path", let outputPath = iterator.next() { - return outputPath - } - } - return nil - } - #if os(Linux) @_silgen_name("$ss13_runAsyncMainyyyyYaKcF") private static func _runAsyncMain(_ asyncFun: @Sendable @escaping () async throws -> ()) @@ -94,16 +57,6 @@ class TestEntryPointGenerationTaskAction: TaskAction { } } #endif - if testingLibrary == "xctest" { - #if !os(Windows) && \(options.enableExperimentalTestOutput) - _ = Self.testOutputPath().map { SwiftPMXCTestObserver(testOutputPath: testOutputPath) } - #endif - #if os(WASI) - await XCTMain(__allDiscoveredTests()) as Never - #else - XCTMain(__allDiscoveredTests()) as Never - #endif - } } #else static func main() async { @@ -113,564 +66,18 @@ class TestEntryPointGenerationTaskAction: TaskAction { await Testing.__swiftPMEntryPoint() as Never } #endif - if testingLibrary == "xctest" { - #if !os(Windows) && \(options.enableExperimentalTestOutput) - _ = Self.testOutputPath().map { SwiftPMXCTestObserver(testOutputPath: testOutputPath) } - #endif - #if os(WASI) - await XCTMain(__allDiscoveredTests()) as Never - #else - XCTMain(__allDiscoveredTests()) as Never - #endif - } } #endif } - """)) - + """#) return .succeeded } catch { outputDelegate.emitError("\(error)") return .failed } } +} - private struct Options: ParsableArguments { - @Option var output: Path - @Option var indexStoreLibraryPath: Path - @Option var linkerFilelist: [Path] - @Option var indexStore: [Path] - @Option var indexUnitBasePath: [Path] - @Flag var enableExperimentalTestOutput: Bool = false - } - - private func discoveredTestsFragment(tests: [IndexStore.TestCaseClass]) -> String { - var fragment = "" - for moduleName in Set(tests.map { $0.module }).sorted() { - fragment += "@testable import \(moduleName)\n" - } - fragment += """ - @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings") - public func __allDiscoveredTests() -> [XCTestCaseEntry] { - return [ - - """ - for testClass in tests { - - let testTuples = testClass.testMethods.map { method in - let basename = method.name.hasSuffix("()") ? String(method.name.dropLast(2)) : method.name - if method.isAsync { - return " (\"\(basename)\", asyncTest(\(testClass.name).\(basename)))" - } else { - return " (\"\(basename)\", \(testClass.name).\(basename))" - } - } - fragment += " testCase([\(testTuples.joined(separator: ",\n"))]),\n" - } - fragment += """ - ] - } - """ - return fragment - } - - private var testObservationFragment: String = - """ - #if !os(Windows) // Test observation is not supported on Windows - import Foundation - import XCTest - - public final class SwiftPMXCTestObserver: NSObject { - let testOutputPath: String - - public init(testOutputPath: String) { - self.testOutputPath = testOutputPath - super.init() - XCTestObservationCenter.shared.addTestObserver(self) - } - } - - extension SwiftPMXCTestObserver: XCTestObservation { - private func write(record: any Encodable) { - let lock = FileLock(at: URL(fileURLWithPath: self.testOutputPath + ".lock")) - _ = try? lock.withLock { - self._write(record: record) - } - } - - private func _write(record: any Encodable) { - if let data = try? JSONEncoder().encode(record) { - if let fileHandle = FileHandle(forWritingAtPath: self.testOutputPath) { - defer { fileHandle.closeFile() } - fileHandle.seekToEndOfFile() - fileHandle.write("\\n".data(using: .utf8)!) - fileHandle.write(data) - } else { - _ = try? data.write(to: URL(fileURLWithPath: self.testOutputPath)) - } - } - } - - public func testBundleWillStart(_ testBundle: Bundle) { - let record = TestBundleEventRecord(bundle: .init(testBundle), event: .start) - write(record: TestEventRecord(bundleEvent: record)) - } - - public func testSuiteWillStart(_ testSuite: XCTestSuite) { - let record = TestSuiteEventRecord(suite: .init(testSuite), event: .start) - write(record: TestEventRecord(suiteEvent: record)) - } - - public func testCaseWillStart(_ testCase: XCTestCase) { - let record = TestCaseEventRecord(testCase: .init(testCase), event: .start) - write(record: TestEventRecord(caseEvent: record)) - } - - #if canImport(Darwin) - public func testCase(_ testCase: XCTestCase, didRecord issue: XCTIssue) { - let record = TestCaseFailureRecord(testCase: .init(testCase), issue: .init(issue), failureKind: .unexpected) - write(record: TestEventRecord(caseFailure: record)) - } - - public func testCase(_ testCase: XCTestCase, didRecord expectedFailure: XCTExpectedFailure) { - let record = TestCaseFailureRecord(testCase: .init(testCase), issue: .init(expectedFailure.issue), failureKind: .expected(failureReason: expectedFailure.failureReason)) - write(record: TestEventRecord(caseFailure: record)) - } - #else - public func testCase(_ testCase: XCTestCase, didFailWithDescription description: String, inFile filePath: String?, atLine lineNumber: Int) { - let issue = TestIssue(description: description, inFile: filePath, atLine: lineNumber) - let record = TestCaseFailureRecord(testCase: .init(testCase), issue: issue, failureKind: .unexpected) - write(record: TestEventRecord(caseFailure: record)) - } - #endif - - public func testCaseDidFinish(_ testCase: XCTestCase) { - let record = TestCaseEventRecord(testCase: .init(testCase), event: .finish) - write(record: TestEventRecord(caseEvent: record)) - } - - #if canImport(Darwin) - public func testSuite(_ testSuite: XCTestSuite, didRecord issue: XCTIssue) { - let record = TestSuiteFailureRecord(suite: .init(testSuite), issue: .init(issue), failureKind: .unexpected) - write(record: TestEventRecord(suiteFailure: record)) - } - - public func testSuite(_ testSuite: XCTestSuite, didRecord expectedFailure: XCTExpectedFailure) { - let record = TestSuiteFailureRecord(suite: .init(testSuite), issue: .init(expectedFailure.issue), failureKind: .expected(failureReason: expectedFailure.failureReason)) - write(record: TestEventRecord(suiteFailure: record)) - } - #else - public func testSuite(_ testSuite: XCTestSuite, didFailWithDescription description: String, inFile filePath: String?, atLine lineNumber: Int) { - let issue = TestIssue(description: description, inFile: filePath, atLine: lineNumber) - let record = TestSuiteFailureRecord(suite: .init(testSuite), issue: issue, failureKind: .unexpected) - write(record: TestEventRecord(suiteFailure: record)) - } - #endif - - public func testSuiteDidFinish(_ testSuite: XCTestSuite) { - let record = TestSuiteEventRecord(suite: .init(testSuite), event: .finish) - write(record: TestEventRecord(suiteEvent: record)) - } - - public func testBundleDidFinish(_ testBundle: Bundle) { - let record = TestBundleEventRecord(bundle: .init(testBundle), event: .finish) - write(record: TestEventRecord(bundleEvent: record)) - } - } - - // FIXME: Copied from `Lock.swift` in TSCBasic, would be nice if we had a better way - - #if canImport(Glibc) - @_exported import Glibc - #elseif canImport(Musl) - @_exported import Musl - #elseif os(Windows) - @_exported import CRT - @_exported import WinSDK - #elseif os(WASI) - @_exported import WASILibc - #elseif canImport(Android) - @_exported import Android - #else - @_exported import Darwin.C - #endif - - import Foundation - - public final class FileLock { - #if os(Windows) - private var handle: HANDLE? - #else - private var fileDescriptor: CInt? - #endif - - private let lockFile: URL - - public init(at lockFile: URL) { - self.lockFile = lockFile - } - - public func lock() throws { - #if os(Windows) - if handle == nil { - let h: HANDLE = lockFile.path.withCString(encodedAs: UTF16.self, { - CreateFileW( - $0, - UInt32(GENERIC_READ) | UInt32(GENERIC_WRITE), - UInt32(FILE_SHARE_READ) | UInt32(FILE_SHARE_WRITE), - nil, - DWORD(OPEN_ALWAYS), - DWORD(FILE_ATTRIBUTE_NORMAL), - nil - ) - }) - if h == INVALID_HANDLE_VALUE { - throw FileSystemError(errno: Int32(GetLastError()), lockFile) - } - self.handle = h - } - var overlapped = OVERLAPPED() - overlapped.Offset = 0 - overlapped.OffsetHigh = 0 - overlapped.hEvent = nil - if !LockFileEx(handle, DWORD(LOCKFILE_EXCLUSIVE_LOCK), 0, - UInt32.max, UInt32.max, &overlapped) { - throw ProcessLockError.unableToAquireLock(errno: Int32(GetLastError())) - } - #elseif os(WASI) - // WASI doesn't support flock - #else - if fileDescriptor == nil { - let fd = open(lockFile.path, O_WRONLY | O_CREAT | O_CLOEXEC, 0o666) - if fd == -1 { - fatalError("errno: \\(errno), lockFile: \\(lockFile)") - } - self.fileDescriptor = fd - } - while true { - if flock(fileDescriptor!, LOCK_EX) == 0 { - break - } - if errno == EINTR { continue } - fatalError("unable to acquire lock, errno: \\(errno)") - } - #endif - } - - public func unlock() { - #if os(Windows) - var overlapped = OVERLAPPED() - overlapped.Offset = 0 - overlapped.OffsetHigh = 0 - overlapped.hEvent = nil - UnlockFileEx(handle, 0, UInt32.max, UInt32.max, &overlapped) - #elseif os(WASI) - // WASI doesn't support flock - #else - guard let fd = fileDescriptor else { return } - flock(fd, LOCK_UN) - #endif - } - - deinit { - #if os(Windows) - guard let handle = handle else { return } - CloseHandle(handle) - #elseif os(WASI) - // WASI doesn't support flock - #else - guard let fd = fileDescriptor else { return } - close(fd) - #endif - } - - public func withLock(_ body: () throws -> T) throws -> T { - try lock() - defer { unlock() } - return try body() - } - - public func withLock(_ body: () async throws -> T) async throws -> T { - try lock() - defer { unlock() } - return try await body() - } - } - - // FIXME: Copied from `XCTEvents.swift`, would be nice if we had a better way - - struct TestEventRecord: Codable { - let caseFailure: TestCaseFailureRecord? - let suiteFailure: TestSuiteFailureRecord? - - let bundleEvent: TestBundleEventRecord? - let suiteEvent: TestSuiteEventRecord? - let caseEvent: TestCaseEventRecord? - - init( - caseFailure: TestCaseFailureRecord? = nil, - suiteFailure: TestSuiteFailureRecord? = nil, - bundleEvent: TestBundleEventRecord? = nil, - suiteEvent: TestSuiteEventRecord? = nil, - caseEvent: TestCaseEventRecord? = nil - ) { - self.caseFailure = caseFailure - self.suiteFailure = suiteFailure - self.bundleEvent = bundleEvent - self.suiteEvent = suiteEvent - self.caseEvent = caseEvent - } - } - - // MARK: - Records - - struct TestAttachment: Codable { - let name: String? - // TODO: Handle `userInfo: [AnyHashable : Any]?` - let uniformTypeIdentifier: String - let payload: Data? - } - - struct TestBundleEventRecord: Codable { - let bundle: TestBundle - let event: TestEvent - } - - struct TestCaseEventRecord: Codable { - let testCase: TestCase - let event: TestEvent - } - - struct TestCaseFailureRecord: Codable, CustomStringConvertible { - let testCase: TestCase - let issue: TestIssue - let failureKind: TestFailureKind - - var description: String { - return "\\(issue.sourceCodeContext.description)\\(testCase) \\(issue.compactDescription)" - } - } - - struct TestSuiteEventRecord: Codable { - let suite: TestSuiteRecord - let event: TestEvent - } - - struct TestSuiteFailureRecord: Codable { - let suite: TestSuiteRecord - let issue: TestIssue - let failureKind: TestFailureKind - } - - // MARK: Primitives - - struct TestBundle: Codable { - let bundleIdentifier: String? - let bundlePath: String - } - - struct TestCase: Codable { - let name: String - } - - struct TestErrorInfo: Codable { - let description: String - let type: String - } - - enum TestEvent: Codable { - case start - case finish - } - - enum TestFailureKind: Codable, Equatable { - case unexpected - case expected(failureReason: String?) - - var isExpected: Bool { - switch self { - case .expected: return true - case .unexpected: return false - } - } - } - - struct TestIssue: Codable { - let type: TestIssueType - let compactDescription: String - let detailedDescription: String? - let associatedError: TestErrorInfo? - let sourceCodeContext: TestSourceCodeContext - let attachments: [TestAttachment] - } - - enum TestIssueType: Codable { - case assertionFailure - case performanceRegression - case system - case thrownError - case uncaughtException - case unmatchedExpectedFailure - case unknown - } - - struct TestLocation: Codable, CustomStringConvertible { - let file: String - let line: Int - - var description: String { - return "\\(file):\\(line) " - } - } - - struct TestSourceCodeContext: Codable, CustomStringConvertible { - let callStack: [TestSourceCodeFrame] - let location: TestLocation? - - var description: String { - return location?.description ?? "" - } - } - - struct TestSourceCodeFrame: Codable { - let address: UInt64 - let symbolInfo: TestSourceCodeSymbolInfo? - let symbolicationError: TestErrorInfo? - } - - struct TestSourceCodeSymbolInfo: Codable { - let imageName: String - let symbolName: String - let location: TestLocation? - } - - struct TestSuiteRecord: Codable { - let name: String - } - - // MARK: XCTest compatibility - - extension TestIssue { - init(description: String, inFile filePath: String?, atLine lineNumber: Int) { - let location: TestLocation? - if let filePath = filePath { - location = .init(file: filePath, line: lineNumber) - } else { - location = nil - } - self.init(type: .assertionFailure, compactDescription: description, detailedDescription: description, associatedError: nil, sourceCodeContext: .init(callStack: [], location: location), attachments: []) - } - } - - import XCTest - - #if canImport(Darwin) // XCTAttachment is unavailable in swift-corelibs-xctest. - extension TestAttachment { - init(_ attachment: XCTAttachment) { - self.init( - name: attachment.name, - uniformTypeIdentifier: attachment.uniformTypeIdentifier, - payload: attachment.value(forKey: "payload") as? Data - ) - } - } - #endif - - extension TestBundle { - init(_ testBundle: Bundle) { - self.init( - bundleIdentifier: testBundle.bundleIdentifier, - bundlePath: testBundle.bundlePath - ) - } - } - - extension TestCase { - init(_ testCase: XCTestCase) { - self.init(name: testCase.name) - } - } - - extension TestErrorInfo { - init(_ error: any Swift.Error) { - self.init(description: "\\(error)", type: "\\(Swift.type(of: error))") - } - } - - #if canImport(Darwin) // XCTIssue is unavailable in swift-corelibs-xctest. - extension TestIssue { - init(_ issue: XCTIssue) { - self.init( - type: .init(issue.type), - compactDescription: issue.compactDescription, - detailedDescription: issue.detailedDescription, - associatedError: issue.associatedError.map { .init($0) }, - sourceCodeContext: .init(issue.sourceCodeContext), - attachments: issue.attachments.map { .init($0) } - ) - } - } - - extension TestIssueType { - init(_ type: XCTIssue.IssueType) { - switch type { - case .assertionFailure: self = .assertionFailure - case .thrownError: self = .thrownError - case .uncaughtException: self = .uncaughtException - case .performanceRegression: self = .performanceRegression - case .system: self = .system - case .unmatchedExpectedFailure: self = .unmatchedExpectedFailure - @unknown default: self = .unknown - } - } - } - #endif - - #if canImport(Darwin) // XCTSourceCodeLocation/XCTSourceCodeContext/XCTSourceCodeFrame/XCTSourceCodeSymbolInfo is unavailable in swift-corelibs-xctest. - extension TestLocation { - init(_ location: XCTSourceCodeLocation) { - self.init( - file: location.fileURL.absoluteString, - line: location.lineNumber - ) - } - } - - extension TestSourceCodeContext { - init(_ context: XCTSourceCodeContext) { - self.init( - callStack: context.callStack.map { .init($0) }, - location: context.location.map { .init($0) } - ) - } - } - - extension TestSourceCodeFrame { - init(_ frame: XCTSourceCodeFrame) { - self.init( - address: frame.address, - symbolInfo: (try? frame.symbolInfo()).map { .init($0) }, - symbolicationError: frame.symbolicationError.map { .init($0) } - ) - } - } - - extension TestSourceCodeSymbolInfo { - init(_ symbolInfo: XCTSourceCodeSymbolInfo) { - self.init( - imageName: symbolInfo.imageName, - symbolName: symbolInfo.symbolName, - location: symbolInfo.location.map { .init($0) } - ) - } - } - #endif - - extension TestSuiteRecord { - init(_ testSuite: XCTestSuite) { - self.init(name: testSuite.name) - } - } - #endif - """ +private struct Options: ParsableArguments { + @Option var output: Path } diff --git a/Sources/SWBUniversalPlatform/TestEntryPointGenerationTool.swift b/Sources/SWBUniversalPlatform/TestEntryPointGenerationTool.swift index 5ee78857..007611ad 100644 --- a/Sources/SWBUniversalPlatform/TestEntryPointGenerationTool.swift +++ b/Sources/SWBUniversalPlatform/TestEntryPointGenerationTool.swift @@ -17,59 +17,7 @@ import SWBCore final class TestEntryPointGenerationToolSpec: GenericCommandLineToolSpec, SpecIdentifierType, @unchecked Sendable { static let identifier = "org.swift.test-entry-point-generator" - override func commandLineFromTemplate(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, optionContext: (any DiscoveredCommandLineToolSpecInfo)?, specialArgs: [String] = [], lookup: ((MacroDeclaration) -> MacroExpression?)? = nil) -> [CommandLineArgument] { - var args = super.commandLineFromTemplate(cbc, delegate, optionContext: optionContext, specialArgs: specialArgs, lookup: lookup) - for (toolchainPath, toolchainLibrarySearchPath) in cbc.producer.toolchains.map({ ($0.path, $0.librarySearchPaths) }) { - if let path = toolchainLibrarySearchPath.findLibrary(operatingSystem: cbc.producer.hostOperatingSystem, basename: "IndexStore") { - args.append(contentsOf: ["--index-store-library-path", .path(path)]) - } - for input in cbc.inputs { - if input.fileType.conformsTo(identifier: "text") { - args.append(contentsOf: ["--linker-filelist", .path(input.absolutePath)]) - } else if input.fileType.conformsTo(identifier: "compiled.mach-o") { - // Do nothing - } else { - delegate.error("Unexpected input of type '\(input.fileType)' to test entry point generation") - } - } - } - return args - } - override func createTaskAction(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate) -> (any PlannedTaskAction)? { TestEntryPointGenerationTaskAction() } - - public func constructTasks(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, indexStorePaths: [Path], indexUnitBasePaths: [Path]) async { - var commandLine = commandLineFromTemplate(cbc, delegate, optionContext: nil) - - for indexStorePath in indexStorePaths { - commandLine.append(contentsOf: ["--index-store", .path(indexStorePath)]) - } - - for basePath in indexUnitBasePaths { - commandLine.append(contentsOf: ["--index-unit-base-path", .path(basePath)]) - } - - delegate.createTask( - type: self, - dependencyData: nil, - payload: nil, - ruleInfo: defaultRuleInfo(cbc, delegate), - additionalSignatureData: "", - commandLine: commandLine, - additionalOutput: [], - environment: environmentFromSpec(cbc, delegate), - workingDirectory: cbc.producer.defaultWorkingDirectory, - inputs: cbc.inputs.map { delegate.createNode($0.absolutePath) }, - outputs: cbc.outputs.map { delegate.createNode($0) }, - mustPrecede: [], - action: createTaskAction(cbc, delegate), - execDescription: resolveExecutionDescription(cbc, delegate), - preparesForIndexing: true, - enableSandboxing: enableSandboxing, - llbuildControlDisabled: true, - additionalTaskOrderingOptions: [] - ) - } } diff --git a/Sources/SWBUniversalPlatform/TestEntryPointTaskProducer.swift b/Sources/SWBUniversalPlatform/TestEntryPointTaskProducer.swift index 23ec56f0..fe4b56ef 100644 --- a/Sources/SWBUniversalPlatform/TestEntryPointTaskProducer.swift +++ b/Sources/SWBUniversalPlatform/TestEntryPointTaskProducer.swift @@ -13,7 +13,6 @@ import SWBCore import SWBTaskConstruction import SWBMacro -import SWBUtil class TestEntryPointTaskProducer: PhasedTaskProducer, TaskProducer { func generateTasks() async -> [any PlannedTask] { @@ -22,54 +21,8 @@ class TestEntryPointTaskProducer: PhasedTaskProducer, TaskProducer { await self.appendGeneratedTasks(&tasks) { delegate in let scope = context.settings.globalScope let outputPath = scope.evaluate(BuiltinMacros.GENERATED_TEST_ENTRY_POINT_PATH) - - guard let configuredTarget = context.configuredTarget else { - context.error("Cannot generate a test entry point without a target") - return - } - var indexStoreDirectories: OrderedSet = [] - var linkerFileLists: OrderedSet = [] - var indexUnitBasePaths: OrderedSet = [] - var binaryPaths: OrderedSet = [] - for directDependency in context.globalProductPlan.dependencies(of: configuredTarget) { - let settings = context.globalProductPlan.planRequest.buildRequestContext.getCachedSettings(directDependency.parameters, target: directDependency.target) - guard settings.productType?.conformsTo(identifier: "com.apple.product-type.bundle.unit-test") == true else { - continue - } - guard settings.globalScope.evaluate(BuiltinMacros.SWIFT_INDEX_STORE_ENABLE) else { - context.error("Cannot perform test discovery for '\(directDependency.target.name)' because index while building is disabled") - continue - } - let path = settings.globalScope.evaluate(BuiltinMacros.SWIFT_INDEX_STORE_PATH) - guard !path.isEmpty else { - continue - } - indexStoreDirectories.append(path) - - for arch in settings.globalScope.evaluate(BuiltinMacros.ARCHS) { - for variant in settings.globalScope.evaluate(BuiltinMacros.BUILD_VARIANTS) { - let innerScope = settings.globalScope - .subscope(binding: BuiltinMacros.archCondition, to: arch) - .subscope(binding: BuiltinMacros.variantCondition, to: variant) - let linkerFileListPath = innerScope.evaluate(BuiltinMacros.__INPUT_FILE_LIST_PATH__) - if !linkerFileListPath.isEmpty { - linkerFileLists.append(linkerFileListPath) - } - let objroot = innerScope.evaluate(BuiltinMacros.OBJROOT) - if !objroot.isEmpty { - indexUnitBasePaths.append(objroot) - } - - let binaryPath = innerScope.evaluate(BuiltinMacros.TARGET_BUILD_DIR).join(innerScope.evaluate(BuiltinMacros.EXECUTABLE_PATH)).normalize() - binaryPaths.append(binaryPath) - } - } - } - - let inputs: [FileToBuild] = linkerFileLists.map { FileToBuild(absolutePath: $0, fileType: self.context.workspaceContext.core.specRegistry.getSpec("text") as! FileTypeSpec) } + binaryPaths.map { FileToBuild(absolutePath: $0, fileType: self.context.workspaceContext.core.specRegistry.getSpec("compiled.mach-o") as! FileTypeSpec) } - - let cbc = CommandBuildContext(producer: context, scope: scope, inputs: inputs, outputs: [outputPath]) - await context.testEntryPointGenerationToolSpec.constructTasks(cbc, delegate, indexStorePaths: indexStoreDirectories.elements, indexUnitBasePaths: indexUnitBasePaths.elements) + let cbc = CommandBuildContext(producer: context, scope: scope, inputs: [], outputs: [outputPath]) + await context.testEntryPointGenerationToolSpec.constructTasks(cbc, delegate) } } return tasks diff --git a/Sources/SWBUtil/CMakeLists.txt b/Sources/SWBUtil/CMakeLists.txt index 9d2d611d..91348fd9 100644 --- a/Sources/SWBUtil/CMakeLists.txt +++ b/Sources/SWBUtil/CMakeLists.txt @@ -46,7 +46,6 @@ add_library(SWBUtil HashContext.swift Headermap.swift HeavyCache.swift - IndexStore.swift Int.swift InterningArena.swift IO.swift diff --git a/Sources/SWBUtil/IndexStore.swift b/Sources/SWBUtil/IndexStore.swift deleted file mode 100644 index 46cbd8bc..00000000 --- a/Sources/SWBUtil/IndexStore.swift +++ /dev/null @@ -1,389 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift open source project -// -// Copyright (c) 2025 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See http://swift.org/LICENSE.txt for license information -// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import SWBCSupport -import Foundation - -public final class IndexStore { - - public struct TestCaseClass { - public struct TestMethod: Hashable, Comparable { - public let name: String - public let isAsync: Bool - - public static func < (lhs: IndexStore.TestCaseClass.TestMethod, rhs: IndexStore.TestCaseClass.TestMethod) -> Bool { - return (lhs.name, (lhs.isAsync ? 1 : 0)) < (rhs.name, (rhs.isAsync ? 1 : 0)) - } - } - - public var name: String - public var module: String - public var testMethods: [TestMethod] - @available(*, deprecated, message: "use testMethods instead") public var methods: [String] - } - - fileprivate var impl: IndexStoreImpl { _impl as! IndexStoreImpl } - private let _impl: Any - - fileprivate init(_ impl: IndexStoreImpl) { - self._impl = impl - } - - static public func open(store path: Path, api: IndexStoreAPI) throws -> IndexStore { - let impl = try IndexStoreImpl.open(store: path, api: api.impl) - return IndexStore(impl) - } - - public func listTests(in objectFiles: [Path]) throws -> [TestCaseClass] { - return try impl.listTests(in: objectFiles) - } - - @available(*, deprecated, message: "use listTests(in:) instead") - public func listTests(inObjectFile object: Path) throws -> [TestCaseClass] { - return try impl.listTests(inObjectFile: object) - } -} - -public final class IndexStoreAPI { - fileprivate var impl: IndexStoreAPIImpl { - _impl as! IndexStoreAPIImpl - } - private let _impl: Any - - public init(dylib path: Path) throws { - self._impl = try IndexStoreAPIImpl(dylib: path) - } -} - -private final class IndexStoreImpl { - typealias TestCaseClass = IndexStore.TestCaseClass - - let api: IndexStoreAPIImpl - - let store: indexstore_t - - private init(store: indexstore_t, api: IndexStoreAPIImpl) { - self.store = store - self.api = api - } - - static public func open(store path: Path, api: IndexStoreAPIImpl) throws -> IndexStoreImpl { - if let store = try api.call({ api.fn.store_create(path.str, &$0) }) { - return IndexStoreImpl(store: store, api: api) - } - throw StubError.error("Unable to open store at \(path.str)") - } - - public func listTests(in objectFiles: [Path]) throws -> [TestCaseClass] { - var inheritance = [String: [String: String]]() - var testMethods = [String: [String: [(name: String, async: Bool)]]]() - - for objectFile in objectFiles { - // Get the records of this object file. - guard let unitReader = try? self.api.call ({ self.api.fn.unit_reader_create(store, unitName(object: objectFile), &$0) }) else { - continue - } - let records = try getRecords(unitReader: unitReader) - let moduleName = self.api.fn.unit_reader_get_module_name(unitReader).str - for record in records { - // get tests info - let testsInfo = try self.getTestsInfo(record: record) - // merge results across module - for (className, parentClassName) in testsInfo.inheritance { - inheritance[moduleName, default: [:]][className] = parentClassName - } - for (className, classTestMethods) in testsInfo.testMethods { - testMethods[moduleName, default: [:]][className, default: []].append(contentsOf: classTestMethods) - } - } - } - - // merge across inheritance in module boundries - func flatten(moduleName: String, className: String) -> [String: (name: String, async: Bool)] { - var allMethods = [String: (name: String, async: Bool)]() - - if let parentClassName = inheritance[moduleName]?[className] { - let parentMethods = flatten(moduleName: moduleName, className: parentClassName) - allMethods.merge(parentMethods, uniquingKeysWith: { (lhs, _) in lhs }) - } - - for method in testMethods[moduleName]?[className] ?? [] { - allMethods[method.name] = (name: method.name, async: method.async) - } - - return allMethods - } - - var testCaseClasses = [TestCaseClass]() - for (moduleName, classMethods) in testMethods { - for className in classMethods.keys { - let methods = flatten(moduleName: moduleName, className: className) - .map { (name, info) in TestCaseClass.TestMethod(name: name, isAsync: info.async) } - .sorted() - testCaseClasses.append(TestCaseClass(name: className, module: moduleName, testMethods: methods, methods: methods.map(\.name))) - } - } - - return testCaseClasses - } - - - @available(*, deprecated, message: "use listTests(in:) instead") - public func listTests(inObjectFile object: Path) throws -> [TestCaseClass] { - // Get the records of this object file. - let unitReader = try api.call{ self.api.fn.unit_reader_create(store, unitName(object: object), &$0) } - let records = try getRecords(unitReader: unitReader) - - // Get the test classes. - var inheritance = [String: String]() - var testMethods = [String: [(name: String, async: Bool)]]() - - for record in records { - let testsInfo = try self.getTestsInfo(record: record) - inheritance.merge(testsInfo.inheritance, uniquingKeysWith: { (lhs, _) in lhs }) - testMethods.merge(testsInfo.testMethods, uniquingKeysWith: { (lhs, _) in lhs }) - } - - func flatten(className: String) -> [(method: String, async: Bool)] { - var results = [(String, Bool)]() - if let parentClassName = inheritance[className] { - let parentMethods = flatten(className: parentClassName) - results.append(contentsOf: parentMethods) - } - if let methods = testMethods[className] { - results.append(contentsOf: methods) - } - return results - } - - let moduleName = self.api.fn.unit_reader_get_module_name(unitReader).str - - var testCaseClasses = [TestCaseClass]() - for className in testMethods.keys { - let methods = flatten(className: className) - .map { TestCaseClass.TestMethod(name: $0.method, isAsync: $0.async) } - .sorted() - testCaseClasses.append(TestCaseClass(name: className, module: moduleName, testMethods: methods, methods: methods.map(\.name))) - } - - return testCaseClasses - } - - private func getTestsInfo(record: String) throws -> (inheritance: [String: String], testMethods: [String: [(name: String, async: Bool)]] ) { - let recordReader = try api.call{ self.api.fn.record_reader_create(store, record, &$0) } - - // scan for inheritance - - let inheritanceStoreRef = StoreRef([String: String](), api: self.api) - let inheritancePointer = unsafeBitCast(Unmanaged.passUnretained(inheritanceStoreRef), to: UnsafeMutableRawPointer.self) - - _ = self.api.fn.record_reader_occurrences_apply_f(recordReader, inheritancePointer) { inheritancePointer , occ -> Bool in - let inheritanceStoreRef = Unmanaged>.fromOpaque(inheritancePointer!).takeUnretainedValue() - let fn = inheritanceStoreRef.api.fn - - // Get the symbol. - let sym = fn.occurrence_get_symbol(occ) - let symbolProperties = fn.symbol_get_properties(sym) - // We only care about symbols that are marked unit tests and are instance methods. - if symbolProperties & UInt64(INDEXSTORE_SYMBOL_PROPERTY_UNITTEST.rawValue) == 0 { - return true - } - if fn.symbol_get_kind(sym) != INDEXSTORE_SYMBOL_KIND_CLASS{ - return true - } - - let parentClassName = fn.symbol_get_name(sym).str - - let childClassNameStoreRef = StoreRef("", api: inheritanceStoreRef.api) - let childClassNamePointer = unsafeBitCast(Unmanaged.passUnretained(childClassNameStoreRef), to: UnsafeMutableRawPointer.self) - _ = fn.occurrence_relations_apply_f(occ!, childClassNamePointer) { childClassNamePointer, relation in - guard let relation = relation else { return true } - let childClassNameStoreRef = Unmanaged>.fromOpaque(childClassNamePointer!).takeUnretainedValue() - let fn = childClassNameStoreRef.api.fn - - // Look for the base class. - if fn.symbol_relation_get_roles(relation) != UInt64(INDEXSTORE_SYMBOL_ROLE_REL_BASEOF.rawValue) { - return true - } - - let childClassNameSym = fn.symbol_relation_get_symbol(relation) - childClassNameStoreRef.instance = fn.symbol_get_name(childClassNameSym).str - return true - } - - if !childClassNameStoreRef.instance.isEmpty { - inheritanceStoreRef.instance[childClassNameStoreRef.instance] = parentClassName - } - - return true - } - - // scan for methods - - let testMethodsStoreRef = StoreRef([String: [(name: String, async: Bool)]](), api: api) - let testMethodsPointer = unsafeBitCast(Unmanaged.passUnretained(testMethodsStoreRef), to: UnsafeMutableRawPointer.self) - - _ = self.api.fn.record_reader_occurrences_apply_f(recordReader, testMethodsPointer) { testMethodsPointer , occ -> Bool in - let testMethodsStoreRef = Unmanaged>.fromOpaque(testMethodsPointer!).takeUnretainedValue() - let fn = testMethodsStoreRef.api.fn - - // Get the symbol. - let sym = fn.occurrence_get_symbol(occ) - let symbolProperties = fn.symbol_get_properties(sym) - // We only care about symbols that are marked unit tests and are instance methods. - if symbolProperties & UInt64(INDEXSTORE_SYMBOL_PROPERTY_UNITTEST.rawValue) == 0 { - return true - } - if fn.symbol_get_kind(sym) != INDEXSTORE_SYMBOL_KIND_INSTANCEMETHOD { - return true - } - - let classNameStoreRef = StoreRef("", api: testMethodsStoreRef.api) - let classNamePointer = unsafeBitCast(Unmanaged.passUnretained(classNameStoreRef), to: UnsafeMutableRawPointer.self) - - _ = fn.occurrence_relations_apply_f(occ!, classNamePointer) { classNamePointer, relation in - guard let relation = relation else { return true } - let classNameStoreRef = Unmanaged>.fromOpaque(classNamePointer!).takeUnretainedValue() - let fn = classNameStoreRef.api.fn - - // Look for the class. - if fn.symbol_relation_get_roles(relation) != UInt64(INDEXSTORE_SYMBOL_ROLE_REL_CHILDOF.rawValue) { - return true - } - - let classNameSym = fn.symbol_relation_get_symbol(relation) - classNameStoreRef.instance = fn.symbol_get_name(classNameSym).str - return true - } - - if !classNameStoreRef.instance.isEmpty { - let methodName = fn.symbol_get_name(sym).str - let isAsync = symbolProperties & UInt64(INDEXSTORE_SYMBOL_PROPERTY_SWIFT_ASYNC.rawValue) != 0 - testMethodsStoreRef.instance[classNameStoreRef.instance, default: []].append((name: methodName, async: isAsync)) - } - - return true - } - - return ( - inheritance: inheritanceStoreRef.instance, - testMethods: testMethodsStoreRef.instance - ) - - } - - private func getRecords(unitReader: indexstore_unit_reader_t?) throws -> [String] { - let builder = StoreRef([String](), api: api) - - let ctx = unsafeBitCast(Unmanaged.passUnretained(builder), to: UnsafeMutableRawPointer.self) - _ = self.api.fn.unit_reader_dependencies_apply_f(unitReader, ctx) { ctx , unit -> Bool in - let store = Unmanaged>.fromOpaque(ctx!).takeUnretainedValue() - let fn = store.api.fn - if fn.unit_dependency_get_kind(unit) == INDEXSTORE_UNIT_DEPENDENCY_RECORD { - store.instance.append(fn.unit_dependency_get_name(unit).str) - } - return true - } - - return builder.instance - } - - private func unitName(object: Path) -> String { - let initialSize = 64 - var buf = UnsafeMutablePointer.allocate(capacity: initialSize) - let len = self.api.fn.store_get_unit_name_from_output_path(store, object.str, buf, initialSize) - - if len + 1 > initialSize { - buf.deallocate() - buf = UnsafeMutablePointer.allocate(capacity: len + 1) - _ = self.api.fn.store_get_unit_name_from_output_path(store, object.str, buf, len + 1) - } - - defer { - buf.deallocate() - } - - return String(cString: buf) - } -} - -private class StoreRef { - let api: IndexStoreAPIImpl - var instance: T - init(_ instance: T, api: IndexStoreAPIImpl) { - self.instance = instance - self.api = api - } -} - -private final class IndexStoreAPIImpl { - - /// The path of the index store dylib. - private let path: Path - - /// Handle of the dynamic library. - private let dylib: LibraryHandle - - /// The index store API functions. - fileprivate let fn: indexstore_functions_t - - fileprivate func call(_ fn: (inout indexstore_error_t?) -> T) throws -> T { - var error: indexstore_error_t? = nil - let ret = fn(&error) - - if let error = error { - if let desc = self.fn.error_get_description(error) { - throw StubError.error(String(cString: desc)) - } - throw StubError.error("Unable to get description for error: \(error)") - } - - return ret - } - - public init(dylib path: Path) throws { - self.path = path - self.dylib = try Library.open(path) - - var api = indexstore_functions_t() - api.store_create = Library.lookup(dylib, "indexstore_store_create") - api.store_get_unit_name_from_output_path = Library.lookup(dylib, "indexstore_store_get_unit_name_from_output_path") - api.unit_reader_create = Library.lookup(dylib, "indexstore_unit_reader_create") - api.error_get_description = Library.lookup(dylib, "indexstore_error_get_description") - api.unit_reader_dependencies_apply_f = Library.lookup(dylib, "indexstore_unit_reader_dependencies_apply_f") - api.unit_reader_get_module_name = Library.lookup(dylib, "indexstore_unit_reader_get_module_name") - api.unit_dependency_get_kind = Library.lookup(dylib, "indexstore_unit_dependency_get_kind") - api.unit_dependency_get_name = Library.lookup(dylib, "indexstore_unit_dependency_get_name") - api.record_reader_create = Library.lookup(dylib, "indexstore_record_reader_create") - api.symbol_get_name = Library.lookup(dylib, "indexstore_symbol_get_name") - api.symbol_get_properties = Library.lookup(dylib, "indexstore_symbol_get_properties") - api.symbol_get_kind = Library.lookup(dylib, "indexstore_symbol_get_kind") - api.record_reader_occurrences_apply_f = Library.lookup(dylib, "indexstore_record_reader_occurrences_apply_f") - api.occurrence_get_symbol = Library.lookup(dylib, "indexstore_occurrence_get_symbol") - api.occurrence_relations_apply_f = Library.lookup(dylib, "indexstore_occurrence_relations_apply_f") - api.symbol_relation_get_symbol = Library.lookup(dylib, "indexstore_symbol_relation_get_symbol") - api.symbol_relation_get_roles = Library.lookup(dylib, "indexstore_symbol_relation_get_roles") - - self.fn = api - } -} - -extension indexstore_string_ref_t { - fileprivate var str: String { - return String( - bytesNoCopy: UnsafeMutableRawPointer(mutating: data), - length: length, - encoding: .utf8, - freeWhenDone: false - )! - } -} diff --git a/Sources/SWBWindowsPlatform/Specs/Windows.xcspec b/Sources/SWBWindowsPlatform/Specs/Windows.xcspec index 9c58f4f4..df09990f 100644 --- a/Sources/SWBWindowsPlatform/Specs/Windows.xcspec +++ b/Sources/SWBWindowsPlatform/Specs/Windows.xcspec @@ -42,31 +42,6 @@ BasedOn = com.apple.product-type.tool; }, - { - Domain = windows; - Type = ProductType; - Identifier = com.apple.product-type.bundle.unit-test; - BasedOn = com.apple.product-type.library.dynamic; - DefaultBuildProperties = { - // Index store data is required to discover XCTest tests - COMPILER_INDEX_STORE_ENABLE = YES; - SWIFT_INDEX_STORE_ENABLE = YES; - // Testability is needed to generate code to invoke discovered XCTest tests - SWIFT_ENABLE_TESTABILITY = YES; - }; - }, - - { - Domain = windows; - Type = ProductType; - Identifier = com.apple.product-type.tool.swiftpm-test-runner; - BasedOn = default:com.apple.product-type.tool.swiftpm-test-runner; - DefaultBuildProperties = { - EXECUTABLE_SUFFIX = ".$(EXECUTABLE_EXTENSION)"; - EXECUTABLE_EXTENSION = "exe"; - }; - }, - { Domain = windows; Type = ProductType; diff --git a/Sources/SwiftBuild/ProjectModel/BuildSettings.swift b/Sources/SwiftBuild/ProjectModel/BuildSettings.swift index ae10d9de..a8909c03 100644 --- a/Sources/SwiftBuild/ProjectModel/BuildSettings.swift +++ b/Sources/SwiftBuild/ProjectModel/BuildSettings.swift @@ -99,7 +99,6 @@ extension ProjectModel { case SUPPORTS_TEXT_BASED_API case SUPPRESS_WARNINGS case SWIFT_ENABLE_BARE_SLASH_REGEX - case SWIFT_INDEX_STORE_ENABLE case SWIFT_INSTALL_MODULE case SWIFT_PACKAGE_NAME case SWIFT_USER_MODULE_VERSION @@ -147,7 +146,6 @@ extension ProjectModel { case SPECIALIZATION_SDK_OPTIONS case SWIFT_VERSION case SWIFT_ACTIVE_COMPILATION_CONDITIONS - case DYLIB_INSTALL_NAME_BASE } public enum Platform: Hashable, CaseIterable, Sendable { diff --git a/Sources/SwiftBuild/ProjectModel/Targets.swift b/Sources/SwiftBuild/ProjectModel/Targets.swift index efd4749c..0f8a201a 100644 --- a/Sources/SwiftBuild/ProjectModel/Targets.swift +++ b/Sources/SwiftBuild/ProjectModel/Targets.swift @@ -316,7 +316,6 @@ extension ProjectModel { case executable = "com.apple.product-type.tool" case hostBuildTool = "com.apple.product-type.tool.host-build" case unitTest = "com.apple.product-type.bundle.unit-test" - case swiftpmTestRunner = "com.apple.product-type.tool.swiftpm-test-runner" case bundle = "com.apple.product-type.bundle" case packageProduct = "packageProduct" } diff --git a/Tests/SWBBuildSystemTests/BuildOperationTests.swift b/Tests/SWBBuildSystemTests/BuildOperationTests.swift index f3a0c446..e6c0caaf 100644 --- a/Tests/SWBBuildSystemTests/BuildOperationTests.swift +++ b/Tests/SWBBuildSystemTests/BuildOperationTests.swift @@ -399,7 +399,7 @@ fileprivate struct BuildOperationTests: CoreBasedTests { @Test(.requireSDKs(.host), .skipHostOS(.macOS), .skipHostOS(.windows, "cannot find testing library")) func unitTestWithGeneratedEntryPoint() async throws { - try await withTemporaryDirectory(removeTreeOnDeinit: false) { (tmpDir: Path) in + try await withTemporaryDirectory { (tmpDir: Path) in let testProject = try await TestProject( "TestProject", sourceRoot: tmpDir, @@ -417,32 +417,14 @@ fileprivate struct BuildOperationTests: CoreBasedTests { "SDKROOT": "$(HOST_PLATFORM)", "SUPPORTED_PLATFORMS": "$(HOST_PLATFORM)", "SWIFT_VERSION": swiftVersion, - "INDEX_DATA_STORE_DIR": "\(tmpDir.join("index").str)", - "LINKER_DRIVER": "swiftc" ]) ], targets: [ TestStandardTarget( - "UnitTestRunner", - type: .swiftpmTestRunner, - buildConfigurations: [ - TestBuildConfiguration("Debug", - buildSettings: [:]), - ], - buildPhases: [ - TestSourcesBuildPhase(), - TestFrameworksBuildPhase([ - "MyTests.so" - ]) - ], - dependencies: ["MyTests"] - ), - TestStandardTarget( - "MyTests", + "test", type: .unitTest, buildConfigurations: [ TestBuildConfiguration("Debug", buildSettings: [ - "DYLIB_INSTALL_NAME_BASE": "$ORIGIN", "LD_RUNPATH_SEARCH_PATHS": "@loader_path/", ]) ], @@ -451,10 +433,10 @@ fileprivate struct BuildOperationTests: CoreBasedTests { TestFrameworksBuildPhase([ TestBuildFile(.target("library")), ]) - ], dependencies: [ - "library" ], - productReferenceName: "MyTests.so" + dependencies: [ + "library" + ] ), TestStandardTarget( "library", @@ -462,7 +444,6 @@ fileprivate struct BuildOperationTests: CoreBasedTests { buildConfigurations: [ TestBuildConfiguration("Debug", buildSettings: [ "DYLIB_INSTALL_NAME_BASE": "$ORIGIN", - "LD_RUNPATH_SEARCH_PATHS": "@loader_path/", // FIXME: Find a way to make these default "EXECUTABLE_PREFIX": "lib", @@ -476,7 +457,7 @@ fileprivate struct BuildOperationTests: CoreBasedTests { ]) let core = try await getCore() let tester = try await BuildOperationTester(core, testProject, simulated: false) - try localFS.createDirectory(tmpDir.join("index")) + let projectDir = tester.workspace.projects[0].sourceRoot try await tester.fs.writeFileContents(projectDir.join("library.swift")) { stream in @@ -486,19 +467,12 @@ fileprivate struct BuildOperationTests: CoreBasedTests { try await tester.fs.writeFileContents(projectDir.join("test.swift")) { stream in stream <<< """ import Testing - import XCTest import library @Suite struct MySuite { - @Test func myTest() { + @Test func myTest() async throws { #expect(foo() == 42) } } - - final class MYXCTests: XCTestCase { - func testFoo() { - XCTAssertTrue(true) - } - } """ } @@ -509,19 +483,13 @@ fileprivate struct BuildOperationTests: CoreBasedTests { let toolchain = try #require(try await getCore().toolchainRegistry.defaultToolchain) let environment: Environment if destination.platform == "linux" { - environment = ["LD_LIBRARY_PATH": "\(toolchain.path.join("usr/lib/swift/linux").str):\(projectDir.join("build").join("Debug\(destination.builtProductsDirSuffix)"))"] + environment = ["LD_LIBRARY_PATH": toolchain.path.join("usr/lib/swift/linux").str] } else { environment = .init() } - do { - let executionResult = try await Process.getOutput(url: URL(fileURLWithPath: projectDir.join("build").join("Debug\(destination.builtProductsDirSuffix)").join(core.hostOperatingSystem.imageFormat.executableName(basename: "UnitTestRunner")).str), arguments: [], environment: environment) - #expect(String(decoding: executionResult.stdout, as: UTF8.self).contains("Executed 1 test, with 0 failures")) - } - do { - let executionResult = try await Process.getOutput(url: URL(fileURLWithPath: projectDir.join("build").join("Debug\(destination.builtProductsDirSuffix)").join(core.hostOperatingSystem.imageFormat.executableName(basename: "UnitTestRunner")).str), arguments: ["--testing-library", "swift-testing"], environment: environment) - #expect(String(decoding: executionResult.stderr, as: UTF8.self).contains("Test run with 1 test in 1 suite passed")) - } + let executionResult = try await Process.getOutput(url: URL(fileURLWithPath: projectDir.join("build").join("Debug\(destination.builtProductsDirSuffix)").join(core.hostOperatingSystem.imageFormat.executableName(basename: "test.xctest")).str), arguments: ["--testing-library", "swift-testing"], environment: environment) + #expect(String(decoding: executionResult.stderr, as: UTF8.self).contains("Test run started")) } } } diff --git a/Tests/SWBTaskConstructionTests/UnitTestTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/UnitTestTaskConstructionTests.swift index ce4515df..d40bc363 100644 --- a/Tests/SWBTaskConstructionTests/UnitTestTaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/UnitTestTaskConstructionTests.swift @@ -300,7 +300,7 @@ fileprivate struct UnitTestTaskConstructionTests: CoreBasedTests { } @Test(.requireSDKs(.linux)) - func unitTestRunnerTarget_linux() async throws { + func unitTestTarget_linux() async throws { let swiftCompilerPath = try await self.swiftCompilerPath let swiftVersion = try await self.swiftVersion let testProject = TestProject( @@ -319,26 +319,9 @@ fileprivate struct UnitTestTaskConstructionTests: CoreBasedTests { "PRODUCT_NAME": "$(TARGET_NAME)", "SDKROOT": "linux", "SWIFT_VERSION": swiftVersion, - "INDEX_DATA_STORE_DIR": "/index", - "LINKER_DRIVER": "swiftc" ]), ], targets: [ - TestStandardTarget( - "UnitTestRunner", - type: .swiftpmTestRunner, - buildConfigurations: [ - TestBuildConfiguration("Debug", - buildSettings: [:]), - ], - buildPhases: [ - TestSourcesBuildPhase(), - TestFrameworksBuildPhase([ - "UnitTestTarget.so" - ]) - ], - dependencies: ["UnitTestTarget"], - ), TestStandardTarget( "UnitTestTarget", type: .unitTest, @@ -352,8 +335,7 @@ fileprivate struct UnitTestTaskConstructionTests: CoreBasedTests { "TestTwo.swift", ]), ], - dependencies: [], - productReferenceName: "UnitTestTarget.so" + dependencies: [] ), ]) let core = try await getCore() @@ -364,17 +346,14 @@ fileprivate struct UnitTestTaskConstructionTests: CoreBasedTests { try await fs.writeFileContents(swiftCompilerPath) { $0 <<< "binary" } await tester.checkBuild(runDestination: .linux, fs: fs) { results in - results.checkTarget("UnitTestRunner") { target in + results.checkTarget("UnitTestTarget") { target in results.checkTask(.matchTarget(target), .matchRuleType("GenerateTestEntryPoint")) { task in - task.checkCommandLineMatches([.suffix("builtin-generateTestEntryPoint"), "--output", .suffix("test_entry_point.swift"), "--index-store-library-path", .suffix("libIndexStore.so"), "--linker-filelist", .suffix("UnitTestTarget.LinkFileList"), "--index-store", "/index", "--index-unit-base-path", "/tmp/Test/aProject/build"]) - task.checkInputs([ - .pathPattern(.suffix("UnitTestTarget.LinkFileList")), - .pathPattern(.suffix("UnitTestTarget.so")), - .namePattern(.any), - .namePattern(.any) - ]) + task.checkCommandLineMatches([.suffix("builtin-generateTestEntryPoint"), "--output", .suffix("test_entry_point.swift")]) task.checkOutputs([.pathPattern(.suffix("test_entry_point.swift"))]) } + results.checkTask(.matchTarget(target), .matchRuleType("SwiftDriver Compilation")) { task in + task.checkInputs(contain: [.pathPattern(.suffix("test_entry_point.swift"))]) + } } results.checkNoDiagnostics() From db41e13184aa9f7b0de9914a7f7d7109c1314e0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20B=C3=BCgling?= Date: Wed, 4 Jun 2025 14:53:20 -0700 Subject: [PATCH 24/54] Track value locations in XCConfigs per assignment (#552) In #513, initial support for this was added, this PR moves the location to be per assignment which allows emitting fix its not just for the last seen assignments. This will also allow looking at any conditions of assignments when choosing the location for emitting fix its. --- Sources/SWBMacro/MacroEvaluationScope.swift | 2 +- .../SWBMacro/MacroValueAssignmentTable.swift | 104 +++++++----------- 2 files changed, 43 insertions(+), 63 deletions(-) diff --git a/Sources/SWBMacro/MacroEvaluationScope.swift b/Sources/SWBMacro/MacroEvaluationScope.swift index 98d6e692..f8c739e2 100644 --- a/Sources/SWBMacro/MacroEvaluationScope.swift +++ b/Sources/SWBMacro/MacroEvaluationScope.swift @@ -17,7 +17,7 @@ private extension MacroValueAssignmentTable { func lookupMacro(_ macro: MacroDeclaration, overrideLookup: ((MacroDeclaration) -> MacroExpression?)? = nil) -> MacroValueAssignment? { // See if we have an overriding binding. if let override = overrideLookup?(macro) { - return MacroValueAssignment(expression: override, conditions: nil, next: lookupMacro(macro)) + return MacroValueAssignment(expression: override, conditions: nil, next: lookupMacro(macro), location: nil) } // Otherwise, return the normal lookup. diff --git a/Sources/SWBMacro/MacroValueAssignmentTable.swift b/Sources/SWBMacro/MacroValueAssignmentTable.swift index 6586fe43..e5213f07 100644 --- a/Sources/SWBMacro/MacroValueAssignmentTable.swift +++ b/Sources/SWBMacro/MacroValueAssignmentTable.swift @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// public import SWBUtil +import Synchronization /// A mapping from macro declarations to corresponding macro value assignments, each of which is a linked list of macro expressions in precedence order. At the moment it doesn’t support conditional assignments, but that functionality will be implemented soon. public struct MacroValueAssignmentTable: Serializable, Sendable { @@ -20,23 +21,18 @@ public struct MacroValueAssignmentTable: Serializable, Sendable { /// Maps macro declarations to corresponding linked lists of assignments. public var valueAssignments: [MacroDeclaration: MacroValueAssignment] - private var valueLocations: [String: InternedMacroValueAssignmentLocation] - private var macroConfigPaths: OrderedSet - - private init(namespace: MacroNamespace, valueAssignments: [MacroDeclaration: MacroValueAssignment], valueLocations: [String: InternedMacroValueAssignmentLocation], macroConfigPaths: OrderedSet) { + private init(namespace: MacroNamespace, valueAssignments: [MacroDeclaration: MacroValueAssignment]) { self.namespace = namespace self.valueAssignments = valueAssignments - self.valueLocations = valueLocations - self.macroConfigPaths = macroConfigPaths } public init(namespace: MacroNamespace) { - self.init(namespace: namespace, valueAssignments: [:], valueLocations: [:], macroConfigPaths: OrderedSet()) + self.init(namespace: namespace, valueAssignments: [:]) } /// Convenience initializer to create a `MacroValueAssignmentTable` from another instance (i.e., to create a copy). public init(copying table: MacroValueAssignmentTable) { - self.init(namespace: table.namespace, valueAssignments: table.valueAssignments, valueLocations: table.valueLocations, macroConfigPaths: table.macroConfigPaths) + self.init(namespace: table.namespace, valueAssignments: table.valueAssignments) } /// Remove all assignments for the given macro. @@ -86,20 +82,7 @@ public struct MacroValueAssignmentTable: Serializable, Sendable { assert(namespace.lookupMacroDeclaration(macro.name) === macro) // Validate the type. assert(macro.type.matchesExpressionType(value)) - valueAssignments[macro] = MacroValueAssignment(expression: value, conditions: conditions, next: valueAssignments[macro]) - - if let location { - let index = macroConfigPaths.append(location.path).index - valueLocations[macro.name] = InternedMacroValueAssignmentLocation(pathRef: index, line: location.line, startColumn: location.startColumn, endColumn: location.endColumn) - } - } - - private mutating func mergeLocations(from otherTable: MacroValueAssignmentTable) { - otherTable.valueLocations.forEach { - let path = otherTable.macroConfigPaths[$0.value.pathRef] - let index = macroConfigPaths.append(path).index - valueLocations[$0.key] = .init(pathRef: index, line: $0.value.line, startColumn: $0.value.startColumn, endColumn: $0.value.endColumn) - } + valueAssignments[macro] = MacroValueAssignment(expression: value, conditions: conditions, next: valueAssignments[macro], location: location) } /// Adds a mapping from each of the macro-to-value mappings in `otherTable`, inserting them ahead of any already existing assignments in the receiving table. The other table isn’t affected in any way (in particular, no reference is kept from the receiver to the other table). @@ -107,7 +90,6 @@ public struct MacroValueAssignmentTable: Serializable, Sendable { for (macro, firstAssignment) in otherTable.valueAssignments { valueAssignments[macro] = insertCopiesOfMacroValueAssignmentNodes(firstAssignment, inFrontOf: valueAssignments[macro]) } - mergeLocations(from: otherTable) } /// Looks up and returns the first (highest-precedence) macro value assignment for `macro`, if there is one. @@ -126,15 +108,7 @@ public struct MacroValueAssignmentTable: Serializable, Sendable { } public func location(of macro: MacroDeclaration) -> MacroValueAssignmentLocation? { - guard let location = valueLocations[macro.name] else { - return nil - } - return MacroValueAssignmentLocation( - path: macroConfigPaths[location.pathRef], - line: location.line, - startColumn: location.startColumn, - endColumn: location.endColumn - ) + return lookupMacro(macro)?.location } public func bindConditionParameter(_ parameter: MacroConditionParameter, _ conditionValues: [String]) -> MacroValueAssignmentTable { @@ -223,7 +197,6 @@ public struct MacroValueAssignmentTable: Serializable, Sendable { bindAndPushAssignment(firstAssignment) } - table.mergeLocations(from: self) return table } @@ -251,7 +224,7 @@ public struct MacroValueAssignmentTable: Serializable, Sendable { // MARK: Serialization public func serialize(to serializer: T) { - serializer.beginAggregate(3) + serializer.beginAggregate(1) // We don't directly serialize MacroDeclarations, but rather serialize their contents "by hand" so when we deserialize we can re-use existing declarations in our namespace. serializer.beginAggregate(valueAssignments.count) @@ -279,17 +252,6 @@ public struct MacroValueAssignmentTable: Serializable, Sendable { } serializer.endAggregate() // valueAssignments - serializer.beginAggregate(valueLocations.count) - for (decl, loc) in valueLocations.sorted(by: { $0.0 < $1.0 }) { - serializer.beginAggregate(2) - serializer.serialize(decl) - serializer.serialize(loc) - serializer.endAggregate() - } - serializer.endAggregate() - - serializer.serialize(macroConfigPaths) - serializer.endAggregate() // the whole table } @@ -298,10 +260,9 @@ public struct MacroValueAssignmentTable: Serializable, Sendable { guard let delegate = deserializer.delegate as? (any MacroValueAssignmentTableDeserializerDelegate) else { throw DeserializerError.invalidDelegate("delegate must be a MacroValueAssignmentTableDeserializerDelegate") } self.namespace = delegate.namespace self.valueAssignments = [:] - self.valueLocations = [:] // Deserialize the table. - try deserializer.beginAggregate(3) + try deserializer.beginAggregate(1) // Iterate over all the key-value pairs. let count: Int = try deserializer.beginAggregate() @@ -348,16 +309,6 @@ public struct MacroValueAssignmentTable: Serializable, Sendable { // Add it to the dictionary. self.valueAssignments[decl] = asgn } - - let count2 = try deserializer.beginAggregate() - for _ in 0..>(OrderedSet()) + + var location: MacroValueAssignmentLocation? { + if let _location { + return .init( + path: Self.macroConfigPaths.withLock { $0[_location.pathRef] }, + line: _location.line, + startColumn: _location.startColumn, + endColumn: _location.endColumn + ) + } else { + return nil + } + } + /// Initializes the macro value assignment to represent `expression`, with the next existing macro value assignment (if any). - init(expression: MacroExpression, conditions: MacroConditionSet? = nil, next: MacroValueAssignment?) { + init(expression: MacroExpression, conditions: MacroConditionSet? = nil, next: MacroValueAssignment?, location: MacroValueAssignmentLocation?) { self.expression = expression self.conditions = conditions self.next = next + + if let location { + self._location = InternedMacroValueAssignmentLocation( + pathRef: Self.macroConfigPaths.withLock({ $0.append(location.path).index }), + line: location.line, + startColumn: location.startColumn, + endColumn: location.endColumn + ) + } else { + self._location = nil + } } /// Returns the first macro value assignment that is reachable from the receiver and whose conditions match the given set of parameter values, or nil if there is no such assignment value. The returned assignment may be the receiver itself, or it may be any assignment that’s downstream in the linked list of macro value assignments, or it may be nil if there is none. Unconditional macro value assignments are considered to match any conditions. Conditions that reference parameters that don’t have a value in `paramValues` are only considered to match if the match pattern is `*`, i.e. the “match-anything” pattern (which is effectively a no-op). @@ -435,18 +413,20 @@ public final class MacroValueAssignment: Serializable, CustomStringConvertible, // MARK: Serialization public func serialize(to serializer: T) { - serializer.beginAggregate(3) + serializer.beginAggregate(4) serializer.serialize(expression) serializer.serialize(conditions) serializer.serialize(next) + serializer.serialize(_location) serializer.endAggregate() } public init(from deserializer: any Deserializer) throws { - try deserializer.beginAggregate(3) + try deserializer.beginAggregate(4) self.expression = try deserializer.deserialize() self.conditions = try deserializer.deserialize() self.next = try deserializer.deserialize() + self._location = try deserializer.deserialize() } } @@ -510,10 +490,10 @@ private func insertCopiesOfMacroValueAssignmentNodes(_ srcAsgn: MacroValueAssign } if let srcNext = srcAsgn.next { - return MacroValueAssignment(expression: srcAsgn.expression, conditions:srcAsgn.conditions, next: insertCopiesOfMacroValueAssignmentNodes(srcNext, inFrontOf: dstAsgn)) + return MacroValueAssignment(expression: srcAsgn.expression, conditions:srcAsgn.conditions, next: insertCopiesOfMacroValueAssignmentNodes(srcNext, inFrontOf: dstAsgn), location: srcAsgn.location) } else { - return MacroValueAssignment(expression: srcAsgn.expression, conditions:srcAsgn.conditions, next: dstAsgn) + return MacroValueAssignment(expression: srcAsgn.expression, conditions:srcAsgn.conditions, next: dstAsgn, location: srcAsgn.location) } } From b78e04ad7c35e16045ee68f4078cc6064be7fc48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20B=C3=BCgling?= Date: Wed, 4 Jun 2025 18:31:57 -0700 Subject: [PATCH 25/54] Fix handling of multi-line XCConfig assignments (#559) Originally only one line was tracked here (the end line effectively), but we also have to track the start line separately for correct multi-line ranges. --- Sources/SWBCore/MacroConfigFileLoader.swift | 4 +-- Sources/SWBMacro/MacroConfigFileParser.swift | 7 ++-- .../SWBMacro/MacroValueAssignmentTable.swift | 32 ++++++++++++------- Tests/SWBCoreTests/SettingsTests.swift | 2 +- Tests/SWBMacroTests/MacroParsingTests.swift | 18 +++++------ 5 files changed, 36 insertions(+), 27 deletions(-) diff --git a/Sources/SWBCore/MacroConfigFileLoader.swift b/Sources/SWBCore/MacroConfigFileLoader.swift index ba7a99b2..cbb184b4 100644 --- a/Sources/SWBCore/MacroConfigFileLoader.swift +++ b/Sources/SWBCore/MacroConfigFileLoader.swift @@ -242,7 +242,7 @@ final class MacroConfigFileLoader: Sendable { return MacroConfigFileParser(byteString: data, path: path, delegate: delegate) } - mutating func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, path: Path, line: Int, startColumn: Int, endColumn: Int, parser: MacroConfigFileParser) { + mutating func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, path: Path, startLine: Int, endLine: Int, startColumn: Int, endColumn: Int, parser: MacroConfigFileParser) { // Look up the macro name, creating it as a user-defined macro if it isn’t already known. let macro = table.namespace.lookupOrDeclareMacro(UserDefinedMacroDeclaration.self, macroName) @@ -253,7 +253,7 @@ final class MacroConfigFileLoader: Sendable { } // Parse the value in a manner consistent with the macro definition. - let location = MacroValueAssignmentLocation(path: path, line: line, startColumn: startColumn, endColumn: endColumn) + let location = MacroValueAssignmentLocation(path: path, startLine: startLine, endLine: endLine, startColumn: startColumn, endColumn: endColumn) table.push(macro, table.namespace.parseForMacro(macro, value: value), conditions: conditionSet, location: location) } diff --git a/Sources/SWBMacro/MacroConfigFileParser.swift b/Sources/SWBMacro/MacroConfigFileParser.swift index 00453277..7497f867 100644 --- a/Sources/SWBMacro/MacroConfigFileParser.swift +++ b/Sources/SWBMacro/MacroConfigFileParser.swift @@ -362,6 +362,7 @@ public final class MacroConfigFileParser { // Skip over the equals sign. assert(currChar == /* '=' */ 61) advance() + let startLine = currLine let startColumn = currIdx - startOfLine var chunks : [String] = [] @@ -385,7 +386,7 @@ public final class MacroConfigFileParser { } // Finally, now that we have the name, conditions, and value, we tell the delegate about it. let value = chunks.joined(separator: " ") - delegate?.foundMacroValueAssignment(name, conditions: conditions, value: value, path: path, line: currLine, startColumn: startColumn, endColumn: currIdx - startOfLine, parser: self) + delegate?.foundMacroValueAssignment(name, conditions: conditions, value: value, path: path, startLine: startLine, endLine: currLine, startColumn: startColumn, endColumn: currIdx - startOfLine, parser: self) } public func parseNonListAssignmentRHS() -> String? { @@ -520,7 +521,7 @@ public final class MacroConfigFileParser { } func endPreprocessorInclusion() { } - func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, path: Path, line: Int, startColumn: Int, endColumn: Int, parser: MacroConfigFileParser) { + func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, path: Path, startLine: Int, endLine: Int, startColumn: Int, endColumn: Int, parser: MacroConfigFileParser) { self.macroName = macroName self.conditions = conditions.isEmpty ? nil : conditions } @@ -567,7 +568,7 @@ public protocol MacroConfigFileParserDelegate { func endPreprocessorInclusion() /// Invoked once for each macro value assignment. The `macroName` is guaranteed to be non-empty, but `value` may be empty. Any macro conditions are passed as tuples in the `conditions`; parameters are guaranteed to be non-empty strings, but patterns may be empty. - mutating func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, path: Path, line: Int, startColumn: Int, endColumn: Int, parser: MacroConfigFileParser) + mutating func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, path: Path, startLine: Int, endLine: Int, startColumn: Int, endColumn: Int, parser: MacroConfigFileParser) /// Invoked if an error, warning, or other diagnostic is detected. func handleDiagnostic(_ diagnostic: MacroConfigFileDiagnostic, parser: MacroConfigFileParser) diff --git a/Sources/SWBMacro/MacroValueAssignmentTable.swift b/Sources/SWBMacro/MacroValueAssignmentTable.swift index e5213f07..7eb83402 100644 --- a/Sources/SWBMacro/MacroValueAssignmentTable.swift +++ b/Sources/SWBMacro/MacroValueAssignmentTable.swift @@ -337,7 +337,8 @@ public final class MacroValueAssignment: Serializable, CustomStringConvertible, if let _location { return .init( path: Self.macroConfigPaths.withLock { $0[_location.pathRef] }, - line: _location.line, + startLine: _location.startLine, + endLine: _location.endLine, startColumn: _location.startColumn, endColumn: _location.endColumn ) @@ -355,7 +356,8 @@ public final class MacroValueAssignment: Serializable, CustomStringConvertible, if let location { self._location = InternedMacroValueAssignmentLocation( pathRef: Self.macroConfigPaths.withLock({ $0.append(location.path).index }), - line: location.line, + startLine: location.startLine, + endLine: location.endLine, startColumn: location.startColumn, endColumn: location.endColumn ) @@ -432,13 +434,15 @@ public final class MacroValueAssignment: Serializable, CustomStringConvertible, public struct MacroValueAssignmentLocation: Sendable, Equatable { public let path: Path - public let line: Int + public let startLine: Int + public let endLine: Int public let startColumn: Int public let endColumn: Int - public init(path: Path, line: Int, startColumn: Int, endColumn: Int) { + public init(path: Path, startLine: Int, endLine: Int, startColumn: Int, endColumn: Int) { self.path = path - self.line = line + self.startLine = startLine + self.endLine = endLine self.startColumn = startColumn self.endColumn = endColumn } @@ -446,30 +450,34 @@ public struct MacroValueAssignmentLocation: Sendable, Equatable { private struct InternedMacroValueAssignmentLocation: Serializable, Sendable { let pathRef: OrderedSet.Index - let line: Int + public let startLine: Int + public let endLine: Int let startColumn: Int let endColumn: Int - init(pathRef: OrderedSet.Index, line: Int, startColumn: Int, endColumn: Int) { + init(pathRef: OrderedSet.Index, startLine: Int, endLine: Int, startColumn: Int, endColumn: Int) { self.pathRef = pathRef - self.line = line + self.startLine = startLine + self.endLine = endLine self.startColumn = startColumn self.endColumn = endColumn } public func serialize(to serializer: T) where T : SWBUtil.Serializer { - serializer.beginAggregate(4) + serializer.beginAggregate(5) serializer.serialize(pathRef) - serializer.serialize(line) + serializer.serialize(startLine) + serializer.serialize(endLine) serializer.serialize(startColumn) serializer.serialize(endColumn) serializer.endAggregate() } public init(from deserializer: any SWBUtil.Deserializer) throws { - try deserializer.beginAggregate(4) + try deserializer.beginAggregate(5) self.pathRef = try deserializer.deserialize() - self.line = try deserializer.deserialize() + self.startLine = try deserializer.deserialize() + self.endLine = try deserializer.deserialize() self.startColumn = try deserializer.deserialize() self.endColumn = try deserializer.deserialize() } diff --git a/Tests/SWBCoreTests/SettingsTests.swift b/Tests/SWBCoreTests/SettingsTests.swift index f392d22b..435100e8 100644 --- a/Tests/SWBCoreTests/SettingsTests.swift +++ b/Tests/SWBCoreTests/SettingsTests.swift @@ -134,7 +134,7 @@ import SWBMacro // Verify that the settings from the xcconfig were added. let XCCONFIG_USER_SETTING = try #require(settings.userNamespace.lookupMacroDeclaration("XCCONFIG_USER_SETTING")) #expect(settings.tableForTesting.lookupMacro(XCCONFIG_USER_SETTING)?.expression.stringRep == "from-xcconfig") - #expect(settings.tableForTesting.location(of: XCCONFIG_USER_SETTING) == MacroValueAssignmentLocation(path: .init("/tmp/xcconfigs/Base0.xcconfig"), line: 1, startColumn: 24, endColumn: 38)) + #expect(settings.tableForTesting.location(of: XCCONFIG_USER_SETTING) == MacroValueAssignmentLocation(path: .init("/tmp/xcconfigs/Base0.xcconfig"), startLine: 1, endLine: 1, startColumn: 24, endColumn: 38)) // Verify the user project settings. let USER_PROJECT_SETTING = try #require(settings.userNamespace.lookupMacroDeclaration("USER_PROJECT_SETTING")) diff --git a/Tests/SWBMacroTests/MacroParsingTests.swift b/Tests/SWBMacroTests/MacroParsingTests.swift index e639ec4e..4c4c71cf 100644 --- a/Tests/SWBMacroTests/MacroParsingTests.swift +++ b/Tests/SWBMacroTests/MacroParsingTests.swift @@ -790,7 +790,7 @@ fileprivate let testFileData = [ } func endPreprocessorInclusion() { } - func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, path: Path, line: Int, startColumn: Int, endColumn: Int, parser: MacroConfigFileParser) { + func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, path: Path, startLine: Int, endLine: Int, startColumn: Int, endColumn: Int, parser: MacroConfigFileParser) { } func handleDiagnostic(_ diagnostic: MacroConfigFileDiagnostic, parser: MacroConfigFileParser) { @@ -816,10 +816,10 @@ fileprivate let testFileData = [ ], expectedDiagnostics: [], expectedLocations: [ - (macro: "FEATURE_DEFINES_A", path: .init("Multiline.xcconfig"), line: 2, startColumn: 20, endColumn: 37), - (macro: "FEATURE_DEFINES_B", path: .init("Multiline.xcconfig"), line: 5, startColumn: 20, endColumn: 87), - (macro: "FEATURE_DEFINES_C", path: .init("Multiline.xcconfig"), line: 9, startColumn: 20, endColumn: 61), - (macro: "FEATURE_DEFINES_D", path: .init("Multiline.xcconfig"), line: 11, startColumn: 20, endColumn: 45), + (macro: "FEATURE_DEFINES_A", path: .init("Multiline.xcconfig"), startLine: 1, endLine: 2, startColumn: 20, endColumn: 37), + (macro: "FEATURE_DEFINES_B", path: .init("Multiline.xcconfig"), startLine: 3, endLine: 5, startColumn: 20, endColumn: 87), + (macro: "FEATURE_DEFINES_C", path: .init("Multiline.xcconfig"), startLine: 6, endLine: 9, startColumn: 20, endColumn: 61), + (macro: "FEATURE_DEFINES_D", path: .init("Multiline.xcconfig"), startLine: 10, endLine: 11, startColumn: 20, endColumn: 45), ], expectedIncludeDirectivesCount: 1 ) @@ -830,7 +830,7 @@ fileprivate let testFileData = [ typealias ConditionInfo = (param: String, pattern: String) typealias AssignmentInfo = (macro: String, conditions: [ConditionInfo], value: String) typealias DiagnosticInfo = (level: MacroConfigFileDiagnostic.Level, kind: MacroConfigFileDiagnostic.Kind, line: Int) -typealias LocationInfo = (macro: String, path: Path, line: Int, startColumn: Int, endColumn: Int) +typealias LocationInfo = (macro: String, path: Path, startLine: Int, endLine: Int, startColumn: Int, endColumn: Int) private func TestMacroConfigFileParser(_ string: String, expectedAssignments: [AssignmentInfo], expectedDiagnostics: [DiagnosticInfo], expectedLocations: [LocationInfo]? = nil, expectedIncludeDirectivesCount: Int, sourceLocation: SourceLocation = #_sourceLocation) { @@ -856,10 +856,10 @@ private func TestMacroConfigFileParser(_ string: String, expectedAssignments: [A func endPreprocessorInclusion() { self.includeDirectivesCount += 1 } - func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, path: Path, line: Int, startColumn: Int, endColumn: Int, parser: MacroConfigFileParser) { + func foundMacroValueAssignment(_ macroName: String, conditions: [(param: String, pattern: String)], value: String, path: Path, startLine: Int, endLine: Int, startColumn: Int, endColumn: Int, parser: MacroConfigFileParser) { // print("\(parser.lineNumber): \(macroName)\(conditions.map({ "[\($0.param)=\($0.pattern)]" }).joinWithSeparator(""))=\(value)") assignments.append((macro: macroName, conditions: conditions, value: value)) - locations.append((macro: macroName, path: path, line: line, startColumn: startColumn, endColumn: endColumn)) + locations.append((macro: macroName, path: path, startLine: startLine, endLine: endLine, startColumn: startColumn, endColumn: endColumn)) } func handleDiagnostic(_ diagnostic: MacroConfigFileDiagnostic, parser: MacroConfigFileParser) { // print("\(parser.lineNumber): \(diagnostic)") @@ -913,7 +913,7 @@ func ==(lhs: [DiagnosticInfo], rhs: [DiagnosticInfo]) -> Bool { } func ==(lhs: LocationInfo, rhs: LocationInfo) -> Bool { - return (lhs.macro == rhs.macro) && (lhs.path == rhs.path) && (lhs.line == rhs.line) && (lhs.startColumn == rhs.startColumn) && (lhs.endColumn == rhs.endColumn) + return (lhs.macro == rhs.macro) && (lhs.path == rhs.path) && (lhs.startLine == rhs.startLine) && (lhs.endLine == rhs.endLine) && (lhs.startColumn == rhs.startColumn) && (lhs.endColumn == rhs.endColumn) } func ==(lhs: [LocationInfo], rhs: [LocationInfo]) -> Bool { From b1337725e2807dd9ccb069a73058be1db5485455 Mon Sep 17 00:00:00 2001 From: Jonathan Penn Date: Thu, 5 Jun 2025 12:30:48 -0400 Subject: [PATCH 26/54] Ignore `-no_exported_symbols` when the debug dylib is used (rdar://152244838) (#554) The stub executor needs to trampoline to the entry point in the debug dylib. But linking with `-no_exported_symbols` strips it and launching the app fails to find the symbol. Handles both the `LD_EXPORT_SYMBOLS` build setting and manual flags added to `OTHER_LDFLAGS`. --- Sources/SWBCore/Settings/BuiltinMacros.swift | 2 ++ .../Tools/LinkerTools.swift | 30 ++++++++++++++++++- .../PreviewsBuildOperationTests.swift | 4 +-- .../PreviewsTaskConstructionTests.swift | 24 ++++++++++----- .../GeneratePreviewInfoTests.swift | 2 +- 5 files changed, 51 insertions(+), 11 deletions(-) diff --git a/Sources/SWBCore/Settings/BuiltinMacros.swift b/Sources/SWBCore/Settings/BuiltinMacros.swift index 0ac934ff..7fbab73d 100644 --- a/Sources/SWBCore/Settings/BuiltinMacros.swift +++ b/Sources/SWBCore/Settings/BuiltinMacros.swift @@ -805,6 +805,7 @@ public final class BuiltinMacros { public static let LD_DEPENDENCY_INFO_FILE = BuiltinMacros.declarePathMacro("LD_DEPENDENCY_INFO_FILE") public static let LD_DYLIB_INSTALL_NAME = BuiltinMacros.declareStringMacro("LD_DYLIB_INSTALL_NAME") public static let LD_ENTRY_POINT = BuiltinMacros.declareStringMacro("LD_ENTRY_POINT") + public static let LD_EXPORT_SYMBOLS = BuiltinMacros.declareBooleanMacro("LD_EXPORT_SYMBOLS") public static let LD_EXPORT_GLOBAL_SYMBOLS = BuiltinMacros.declareBooleanMacro("LD_EXPORT_GLOBAL_SYMBOLS") public static let LD_LTO_OBJECT_FILE = BuiltinMacros.declarePathMacro("LD_LTO_OBJECT_FILE") public static let LD_NO_PIE = BuiltinMacros.declareBooleanMacro("LD_NO_PIE") @@ -1871,6 +1872,7 @@ public final class BuiltinMacros { LD_ENTRY_POINT, LD_ENTITLEMENTS_SECTION, LD_ENTITLEMENTS_SECTION_DER, + LD_EXPORT_SYMBOLS, LD_EXPORT_GLOBAL_SYMBOLS, LD_LTO_OBJECT_FILE, LD_NO_PIE, diff --git a/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift b/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift index 8dd603b1..44c1681d 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift @@ -314,6 +314,8 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec delegate.emit(Diagnostic(behavior: dyldEnvDiagnosticBehavior, location: .buildSetting(BuiltinMacros.OTHER_LDFLAGS), data: DiagnosticData("The \(BuiltinMacros.OTHER_LDFLAGS.name) build setting is not allowed to contain \(arg), use the dedicated LD_ENVIRONMENT build setting instead."))) case "-client_name": delegate.emit(Diagnostic(behavior: dyldEnvDiagnosticBehavior, location: .buildSetting(BuiltinMacros.OTHER_LDFLAGS), data: DiagnosticData("The \(BuiltinMacros.OTHER_LDFLAGS.name) build setting is not allowed to contain \(arg), use the dedicated LD_CLIENT_NAME build setting instead."))) + case "-no_exported_symbols": + delegate.emit(Diagnostic(behavior: dyldEnvDiagnosticBehavior, location: .buildSetting(BuiltinMacros.OTHER_LDFLAGS), data: DiagnosticData("The \(BuiltinMacros.OTHER_LDFLAGS.name) build setting is not allowed to contain \(arg), use the dedicated LD_EXPORT_SYMBOLS build setting instead."))) default: break } @@ -504,6 +506,15 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec return nil } return cbc.scope.namespace.parseLiteralString(name) + case BuiltinMacros.DEAD_CODE_STRIPPING where isPreviewDylib: + // We need to keep otherwise unused stub executor library symbols present so + // PreviewsInjection can call them when doing the XOJIT handshake. + return cbc.scope.namespace.parseLiteralString("NO") + case BuiltinMacros.LD_EXPORT_SYMBOLS where isPreviewDylib, + BuiltinMacros.LD_EXPORT_GLOBAL_SYMBOLS where isPreviewDylib: + // We need to keep otherwise unused stub executor library symbols present so + // PreviewsInjection can call them when doing the XOJIT handshake. + return cbc.scope.namespace.parseLiteralString("YES") case BuiltinMacros.OTHER_LDFLAGS where isPreviewDylib: let ldFlagsToEvaluate: [String] if dyldEnvDiagnosticBehavior == .warning { @@ -774,7 +785,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec switch macro { case BuiltinMacros.LD_ENTRY_POINT where cbc.scope.previewStyle == .xojit: return cbc.scope.namespace.parseLiteralString("___debug_blank_executor_main") - case BuiltinMacros.LD_EXPORT_GLOBAL_SYMBOLS: + case BuiltinMacros.LD_EXPORT_SYMBOLS, BuiltinMacros.LD_EXPORT_GLOBAL_SYMBOLS: // We need to keep otherwise unused stub executor library symbols present so // PreviewsInjection can call them when doing the XOJIT handshake. return cbc.scope.namespace.parseLiteralString("YES") @@ -1821,6 +1832,23 @@ fileprivate func filterLinkerFlagsWhenUnderPreviewsDylib(_ flags: [String]) -> [ } continue } + else if flag == "-Wl,-no_exported_symbols" { + continue + } + else if flag == "-no_exported_symbols" && newFlags.last == "-Xlinker" { + // Filter out `-no_exported_symbols` when using the previews dylib, since this + // strips important symbols that are needed for the stub executor trampoline.. + // Transition from `OTHER_LD_FLAGS` to the dedicated `LD_EXPORT_SYMBOLS` (by + // defining both at once) in order to remain compatible with Xcode versions both + // before and after this change. + newFlags.removeLast() + while let next = it.next() { + if next != "-Xlinker" { + break + } + } + continue + } newFlags.append(flag) } return newFlags diff --git a/Tests/SWBBuildSystemTests/PreviewsBuildOperationTests.swift b/Tests/SWBBuildSystemTests/PreviewsBuildOperationTests.swift index 7df199bf..f27b8a59 100644 --- a/Tests/SWBBuildSystemTests/PreviewsBuildOperationTests.swift +++ b/Tests/SWBBuildSystemTests/PreviewsBuildOperationTests.swift @@ -308,7 +308,7 @@ fileprivate struct PreviewsBuildOperationTests: CoreBasedTests { linkerCommandLine.remove(at: idx) } } - XCTAssertEqualSequences(linkerCommandLine, ["\(core.developerPath.path.str)/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang", "-Xlinker", "-reproducible", "-target", "\(results.runDestinationTargetArchitecture)-apple-ios\(core.loadSDK(.iOSSimulator).defaultDeploymentTarget)-simulator", "-dynamiclib", "-isysroot", core.loadSDK(.iOSSimulator).path.str, "-Os", "-Xlinker", "-warn_unused_dylibs", "-L\(srcRoot.str)/build/EagerLinkingTBDs/Debug-iphonesimulator", "-L\(srcRoot.str)/build/Debug-iphonesimulator", "-F\(srcRoot.str)/build/EagerLinkingTBDs/Debug-iphonesimulator", "-F\(srcRoot.str)/build/Debug-iphonesimulator", "-filelist", "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/AppTarget.LinkFileList", "-install_name", "@rpath/AppTarget.debug.dylib", "-dead_strip", "-Xlinker", "-object_path_lto", "-Xlinker", "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/AppTarget_lto.o", "-Xlinker", "-objc_abi_version", "-Xlinker", "2", "-Xlinker", "-dependency_info", "-Xlinker", "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/AppTarget_dependency_info.dat", "-L\(core.developerPath.path.str)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/iphonesimulator", "-L/usr/lib/swift", "-Xlinker", "-add_ast_path", "-Xlinker", "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/AppTarget.swiftmodule", "-Xlinker", "-alias", "-Xlinker", "_main", "-Xlinker", "___debug_main_executable_dylib_entry_point", "-Xlinker", "-no_adhoc_codesign", "-o", "\(srcRoot.str)/build/Debug-iphonesimulator/AppTarget.app/AppTarget.debug.dylib"]) + XCTAssertEqualSequences(linkerCommandLine, ["\(core.developerPath.path.str)/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang", "-Xlinker", "-reproducible", "-target", "\(results.runDestinationTargetArchitecture)-apple-ios\(core.loadSDK(.iOSSimulator).defaultDeploymentTarget)-simulator", "-dynamiclib", "-isysroot", core.loadSDK(.iOSSimulator).path.str, "-Os", "-Xlinker", "-warn_unused_dylibs", "-L\(srcRoot.str)/build/EagerLinkingTBDs/Debug-iphonesimulator", "-L\(srcRoot.str)/build/Debug-iphonesimulator", "-F\(srcRoot.str)/build/EagerLinkingTBDs/Debug-iphonesimulator", "-F\(srcRoot.str)/build/Debug-iphonesimulator", "-filelist", "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/AppTarget.LinkFileList", "-install_name", "@rpath/AppTarget.debug.dylib", "-Xlinker", "-object_path_lto", "-Xlinker", "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/AppTarget_lto.o", "-rdynamic", "-Xlinker", "-objc_abi_version", "-Xlinker", "2", "-Xlinker", "-dependency_info", "-Xlinker", "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/AppTarget_dependency_info.dat", "-L\(core.developerPath.path.str)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/iphonesimulator", "-L/usr/lib/swift", "-Xlinker", "-add_ast_path", "-Xlinker", "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/AppTarget.swiftmodule", "-Xlinker", "-alias", "-Xlinker", "_main", "-Xlinker", "___debug_main_executable_dylib_entry_point", "-Xlinker", "-no_adhoc_codesign", "-o", "\(srcRoot.str)/build/Debug-iphonesimulator/AppTarget.app/AppTarget.debug.dylib"]) } } @@ -615,7 +615,7 @@ fileprivate struct PreviewsBuildOperationTests: CoreBasedTests { linkerCommandLine.remove(at: idx) } } - XCTAssertEqualSequences(linkerCommandLine, ["\(core.developerPath.path.str)/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang", "-Xlinker", "-reproducible", "-target", "\(results.runDestinationTargetArchitecture)-apple-ios\(core.loadSDK(.iOSSimulator).defaultDeploymentTarget)-simulator", "-dynamiclib", "-isysroot", core.loadSDK(.iOSSimulator).path.str, "-Os", "-L\(srcRoot.str)/build/EagerLinkingTBDs/Debug-iphonesimulator", "-L\(srcRoot.str)/build/Debug-iphonesimulator", "-F\(srcRoot.str)/build/EagerLinkingTBDs/Debug-iphonesimulator", "-F\(srcRoot.str)/build/Debug-iphonesimulator", "-filelist", "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/AppTarget.LinkFileList", "-install_name", "@rpath/AppTarget.debug.dylib", "-dead_strip", "-Xlinker", "-object_path_lto", "-Xlinker", "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/AppTarget_lto.o", "-Xlinker", "-objc_abi_version", "-Xlinker", "2", "-Xlinker", "-dependency_info", "-Xlinker", "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/AppTarget_dependency_info.dat", "-fobjc-link-runtime", "-L\(core.developerPath.path.str)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/iphonesimulator", "-L/usr/lib/swift", "-Xlinker", "-add_ast_path", "-Xlinker", "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/AppTarget.swiftmodule", "-Xlinker", "-alias", "-Xlinker", "_main", "-Xlinker", "___debug_main_executable_dylib_entry_point", "-Xlinker", "-no_adhoc_codesign", "-o", "\(srcRoot.str)/build/Debug-iphonesimulator/AppTarget.app/AppTarget.debug.dylib"]) + XCTAssertEqualSequences(linkerCommandLine, ["\(core.developerPath.path.str)/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang", "-Xlinker", "-reproducible", "-target", "\(results.runDestinationTargetArchitecture)-apple-ios\(core.loadSDK(.iOSSimulator).defaultDeploymentTarget)-simulator", "-dynamiclib", "-isysroot", core.loadSDK(.iOSSimulator).path.str, "-Os", "-L\(srcRoot.str)/build/EagerLinkingTBDs/Debug-iphonesimulator", "-L\(srcRoot.str)/build/Debug-iphonesimulator", "-F\(srcRoot.str)/build/EagerLinkingTBDs/Debug-iphonesimulator", "-F\(srcRoot.str)/build/Debug-iphonesimulator", "-filelist", "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/AppTarget.LinkFileList", "-install_name", "@rpath/AppTarget.debug.dylib", "-Xlinker", "-object_path_lto", "-Xlinker", "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/AppTarget_lto.o", "-rdynamic", "-Xlinker", "-objc_abi_version", "-Xlinker", "2", "-Xlinker", "-dependency_info", "-Xlinker", "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/AppTarget_dependency_info.dat", "-fobjc-link-runtime", "-L\(core.developerPath.path.str)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/iphonesimulator", "-L/usr/lib/swift", "-Xlinker", "-add_ast_path", "-Xlinker", "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/AppTarget.swiftmodule", "-Xlinker", "-alias", "-Xlinker", "_main", "-Xlinker", "___debug_main_executable_dylib_entry_point", "-Xlinker", "-no_adhoc_codesign", "-o", "\(srcRoot.str)/build/Debug-iphonesimulator/AppTarget.app/AppTarget.debug.dylib"]) } } } diff --git a/Tests/SWBTaskConstructionTests/PreviewsTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/PreviewsTaskConstructionTests.swift index 4e7e6eee..2092b03f 100644 --- a/Tests/SWBTaskConstructionTests/PreviewsTaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/PreviewsTaskConstructionTests.swift @@ -932,7 +932,7 @@ fileprivate struct PreviewsTaskConstructionTests: CoreBasedTests { /// Test that the `__info_plist` section ends up in the stub executor instead of the preview dylib when `CREATE_INFOPLIST_SECTION_IN_BINARY` is enabled. @Test(.requireSDKs(.iOS)) - func xOJITEmbeddedInfoPlist() async throws { + func XOJITEmbeddedInfoPlist() async throws { try await withTemporaryDirectory { tmpDirPath in let srcRoot = tmpDirPath.join("srcroot") @@ -1019,9 +1019,9 @@ fileprivate struct PreviewsTaskConstructionTests: CoreBasedTests { } } - /// Test that any `-dyld_env` arguments end up in the stub executor instead of the preview dylib when is enabled. + /// Test that any `OTHER_LDFLAGS` arguments end up in the stub executor instead of the debug dylib when is enabled. @Test(.requireSDKs(.iOS)) - func xOJITDyldEnv() async throws { + func XOJITOtherLDFlags() async throws { try await withTemporaryDirectory { tmpDirPath in let srcRoot = tmpDirPath.join("srcroot") @@ -1037,7 +1037,11 @@ fileprivate struct PreviewsTaskConstructionTests: CoreBasedTests { buildConfigurations: [ TestBuildConfiguration("Debug", buildSettings: [ "LD_ENVIRONMENT": "DYLD_X_PATH=/foo", - "OTHER_LDFLAGS": "-Wl,-dyld_env,NOT=allowed_here -Xlinker -dyld_env -Xlinker NOR=this", + "OTHER_LDFLAGS": """ + -Wl,-dyld_env,NOT=allowed_here -Xlinker -dyld_env -Xlinker NOR=this \ + -Wl,-no_exported_symbols -Xlinker -no_exported_symbols + """, + "LD_EXPORT_SYMBOLS": "NO", "GENERATE_INFOPLIST_FILE": "YES", "PRODUCT_NAME": "$(TARGET_NAME)", "PRODUCT_BUNDLE_IDENTIFIER": "com.test.ProjectName", @@ -1100,20 +1104,26 @@ fileprivate struct PreviewsTaskConstructionTests: CoreBasedTests { // from OTHER_LDFLAGS, which is overridden to a custom set of flags _without_ $(inherited), so the stub executor doesn't get them task.checkCommandLineDoesNotContain("-Wl,-dyld_env,NOT=allowed_here") task.checkCommandLineDoesNotContain("NOR=this") + task.checkCommandLineDoesNotContain("-Wl,-no_exported_symbols") + task.checkCommandLineDoesNotContain("-no_exported_symbols") } results.checkTask(.matchRule(["Ld", "\(srcRoot.str)/build/Debug-iphonesimulator/Tool.debug.dylib", "normal"])) { task in - // from LD_ENVIRONMENT, which is conditional on MACH_O_TYPE=mh_execute, so the previews dylib (which overrides MACH_O_TYPE=mh_dylib) doesn't get it + // from LD_ENVIRONMENT, which is conditional on MACH_O_TYPE=mh_execute, so the debug dylib (which overrides MACH_O_TYPE=mh_dylib) doesn't get it task.checkCommandLineDoesNotContain("-dyld_env") task.checkCommandLineDoesNotContain("DYLD_X_PATH=/foo") - // from OTHER_LDFLAGS, which is passed through unchanged to the previews dylib + // from OTHER_LDFLAGS, which is passed through unchanged to the debug dylib task.checkCommandLineDoesNotContain("-Wl,-dyld_env,NOT=allowed_here") task.checkCommandLineDoesNotContain("NOR=this") + task.checkCommandLineDoesNotContain("-Wl,-no_exported_symbols") + task.checkCommandLineDoesNotContain("-no_exported_symbols") } results.checkWarning(.equal("The OTHER_LDFLAGS build setting is not allowed to contain -dyld_env, use the dedicated LD_ENVIRONMENT build setting instead. (in target 'Tool' from project 'ProjectName')")) results.checkWarning(.equal("The OTHER_LDFLAGS build setting is not allowed to contain -dyld_env, use the dedicated LD_ENVIRONMENT build setting instead. (in target 'Tool' from project 'ProjectName')")) + results.checkWarning(.equal("The OTHER_LDFLAGS build setting is not allowed to contain -no_exported_symbols, use the dedicated LD_EXPORT_SYMBOLS build setting instead. (in target 'Tool' from project 'ProjectName')")) + results.checkWarning(.equal("The OTHER_LDFLAGS build setting is not allowed to contain -no_exported_symbols, use the dedicated LD_EXPORT_SYMBOLS build setting instead. (in target 'Tool' from project 'ProjectName')")) results.checkNoTask() results.checkNoDiagnostics() @@ -1123,7 +1133,7 @@ fileprivate struct PreviewsTaskConstructionTests: CoreBasedTests { } @Test(.requireSDKs(.iOS)) - func xOJITPropagatingRpaths() async throws { + func XOJITPropagatingRpaths() async throws { try await withTemporaryDirectory { tmpDirPath in let srcRoot = tmpDirPath.join("srcroot") diff --git a/Tests/SwiftBuildTests/GeneratePreviewInfoTests.swift b/Tests/SwiftBuildTests/GeneratePreviewInfoTests.swift index 71a699b9..aa2d667c 100644 --- a/Tests/SwiftBuildTests/GeneratePreviewInfoTests.swift +++ b/Tests/SwiftBuildTests/GeneratePreviewInfoTests.swift @@ -229,11 +229,11 @@ fileprivate struct GeneratePreviewInfoTests: CoreBasedTests { "\(tmpDir.str)/Test/build/Test.build/Debug-iphoneos/App.build/Objects-normal/\(activeRunDestination.targetArchitecture)/App.LinkFileList", "-install_name", "@rpath/App.debug.dylib", - "-dead_strip", "-Xlinker", "-object_path_lto", "-Xlinker", "\(tmpDir.str)/Test/build/Test.build/Debug-iphoneos/App.build/Objects-normal/\(activeRunDestination.targetArchitecture)/App_lto.o", + "-rdynamic", "-Xlinker", "-dependency_info", "-Xlinker", From 8502138054f430e3da9318614d9ca030957fe993 Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Thu, 5 Jun 2025 17:08:58 -0700 Subject: [PATCH 27/54] Improve error message when build description attachment saving failswq --- Sources/SWBTaskExecution/BuildDescriptionManager.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/SWBTaskExecution/BuildDescriptionManager.swift b/Sources/SWBTaskExecution/BuildDescriptionManager.swift index bc1da89a..fc351510 100644 --- a/Sources/SWBTaskExecution/BuildDescriptionManager.swift +++ b/Sources/SWBTaskExecution/BuildDescriptionManager.swift @@ -753,7 +753,7 @@ private final class BuildSystemTaskPlanningDelegate: TaskPlanningDelegate { try fileSystem.createDirectory(path.dirname, recursive: true) try fileSystem.write(path, contents: contents) } catch { - constructionDelegate.emit(Diagnostic(behavior: .error, location: .unknown, data: DiagnosticData("failed to save attachment: \(path.str)"))) + constructionDelegate.emit(Diagnostic(behavior: .error, location: .unknown, data: DiagnosticData("failed to save attachment: \(path.str). Error: \(error)"))) } return path } From 5bbe231719ce9b6adf5757a3a385aca46be067c3 Mon Sep 17 00:00:00 2001 From: Jake Petroules Date: Fri, 14 Feb 2025 15:30:25 +0900 Subject: [PATCH 28/54] Add FreeBSD support This allows building Swift Build for FreeBSD hosts, as well as building for a FreeBSD target from a FreeBSD host. Also adds some speculative support for targeting OpenBSD on OpenBSD hosts, since SwiftPM has minimal support. --- Package.swift | 4 +-- Sources/SWBCore/Settings/Settings.swift | 4 +++ Sources/SWBGenericUnixPlatform/Plugin.swift | 12 ++++++--- .../Specs/FreeBSDLibtool.xcspec | 27 +++++++++++++++++++ .../RunDestinationTestSupport.swift | 20 ++++++++++++++ .../SWBTestSupport/SkippedTestSupport.swift | 12 +++++++-- Sources/SWBUtil/Architecture.swift | 13 ++++++++- Sources/SWBUtil/FSProxy.swift | 8 ++++++ Sources/SWBUtil/Lock.swift | 2 +- Sources/SWBUtil/Process.swift | 2 ++ Sources/SWBUtil/ProcessInfo.swift | 8 +++++- .../BuildTaskBehaviorTests.swift | 2 +- ...mmandLineToolSpecDiscoveredInfoTests.swift | 2 +- .../SWBCoreTests/FileTextEncodingTests.swift | 3 +-- Tests/SWBCoreTests/SettingsTests.swift | 2 +- Tests/SWBTaskExecutionTests/PBXCpTests.swift | 2 +- Tests/SWBUtilTests/ElapsedTimerTests.swift | 3 ++- Tests/SWBUtilTests/FSProxyTests.swift | 4 +-- Tests/SWBUtilTests/FileHandleTests.swift | 2 +- Tests/SWBUtilTests/HeavyCacheTests.swift | 5 ++-- Tests/SWBUtilTests/MiscTests.swift | 2 +- Tests/SWBUtilTests/RateLimiterTests.swift | 4 ++- 22 files changed, 118 insertions(+), 25 deletions(-) create mode 100644 Sources/SWBGenericUnixPlatform/Specs/FreeBSDLibtool.xcspec diff --git a/Package.swift b/Package.swift index 2cbc3eff..d368e124 100644 --- a/Package.swift +++ b/Package.swift @@ -110,7 +110,7 @@ let package = Package( "SWBBuildSystem", "SWBServiceCore", "SWBTaskExecution", - .product(name: "SystemPackage", package: "swift-system", condition: .when(platforms: [.linux, .android, .windows])), + .product(name: "SystemPackage", package: "swift-system", condition: .when(platforms: [.linux, .openbsd, .android, .windows, .custom("freebsd")])), ], exclude: ["CMakeLists.txt"], swiftSettings: swiftSettings(languageMode: .v5)), @@ -201,7 +201,7 @@ let package = Package( "SWBCSupport", "SWBLibc", .product(name: "ArgumentParser", package: "swift-argument-parser"), - .product(name: "SystemPackage", package: "swift-system", condition: .when(platforms: [.linux, .android, .windows])), + .product(name: "SystemPackage", package: "swift-system", condition: .when(platforms: [.linux, .openbsd, .android, .windows, .custom("freebsd")])), ], exclude: ["CMakeLists.txt"], swiftSettings: swiftSettings(languageMode: .v5)), diff --git a/Sources/SWBCore/Settings/Settings.swift b/Sources/SWBCore/Settings/Settings.swift index 0bfabcf0..70d95c0d 100644 --- a/Sources/SWBCore/Settings/Settings.swift +++ b/Sources/SWBCore/Settings/Settings.swift @@ -5307,6 +5307,10 @@ extension OperatingSystem { return "windows" case .linux: return "linux" + case .freebsd: + return "freebsd" + case .openbsd: + return "openbsd" case .android: return "android" case .unknown: diff --git a/Sources/SWBGenericUnixPlatform/Plugin.swift b/Sources/SWBGenericUnixPlatform/Plugin.swift index 128fde75..a3839068 100644 --- a/Sources/SWBGenericUnixPlatform/Plugin.swift +++ b/Sources/SWBGenericUnixPlatform/Plugin.swift @@ -39,7 +39,11 @@ struct GenericUnixPlatformSpecsExtension: SpecificationsExtension { } func specificationDomains() -> [String: [String]] { - ["linux": ["generic-unix"]] + [ + "linux": ["generic-unix"], + "freebsd": ["generic-unix"], + "openbsd": ["generic-unix"], + ] } } @@ -73,9 +77,9 @@ struct GenericUnixSDKRegistryExtension: SDKRegistryExtension { let defaultProperties: [String: PropertyListItem] switch operatingSystem { - case .linux: + case .linux, .freebsd: defaultProperties = [ - // Workaround to avoid `-dependency_info` on Linux. + // Workaround to avoid `-dependency_info`. "LD_DEPENDENCY_INFO_FILE": .plString(""), "GENERATE_TEXT_BASED_STUBS": "NO", @@ -167,6 +171,6 @@ struct GenericUnixToolchainRegistryExtension: ToolchainRegistryExtension { extension OperatingSystem { /// Whether the Core is allowed to create a fallback toolchain, SDK, and platform for this operating system in cases where no others have been provided. var createFallbackSystemToolchain: Bool { - return self == .linux + return self == .linux || self == .freebsd || self == .openbsd } } diff --git a/Sources/SWBGenericUnixPlatform/Specs/FreeBSDLibtool.xcspec b/Sources/SWBGenericUnixPlatform/Specs/FreeBSDLibtool.xcspec new file mode 100644 index 00000000..8d247c89 --- /dev/null +++ b/Sources/SWBGenericUnixPlatform/Specs/FreeBSDLibtool.xcspec @@ -0,0 +1,27 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +( + { + Domain = freebsd; + Identifier = com.apple.pbx.linkers.libtool; + BasedOn = generic-unix:com.apple.pbx.linkers.libtool; + Type = Linker; + Options = ( + { + Name = "LIBTOOL_USE_RESPONSE_FILE"; + Type = Boolean; + DefaultValue = NO; + }, + ); + }, +) diff --git a/Sources/SWBTestSupport/RunDestinationTestSupport.swift b/Sources/SWBTestSupport/RunDestinationTestSupport.swift index 9f6ae3bf..247397b7 100644 --- a/Sources/SWBTestSupport/RunDestinationTestSupport.swift +++ b/Sources/SWBTestSupport/RunDestinationTestSupport.swift @@ -98,6 +98,10 @@ extension _RunDestinationInfo { windows case .linux: linux + case .freebsd: + freebsd + case .openbsd: + openbsd case .android: android case .unknown: @@ -259,6 +263,22 @@ extension _RunDestinationInfo { return .init(platform: "linux", sdk: "linux", sdkVariant: "linux", targetArchitecture: arch, supportedArchitectures: ["x86_64", "aarch64"], disableOnlyActiveArch: false) } + /// A run destination targeting FreeBSD generic device, using the public SDK. + package static var freebsd: Self { + guard let arch = Architecture.hostStringValue else { + preconditionFailure("Unknown architecture \(Architecture.host.stringValue ?? "")") + } + return .init(platform: "freebsd", sdk: "freebsd", sdkVariant: "freebsd", targetArchitecture: arch, supportedArchitectures: ["x86_64", "aarch64"], disableOnlyActiveArch: false) + } + + /// A run destination targeting OpenBSD generic device, using the public SDK. + package static var openbsd: Self { + guard let arch = Architecture.hostStringValue else { + preconditionFailure("Unknown architecture \(Architecture.host.stringValue ?? "")") + } + return .init(platform: "openbsd", sdk: "openbsd", sdkVariant: "openbsd", targetArchitecture: arch, supportedArchitectures: ["x86_64", "aarch64"], disableOnlyActiveArch: false) + } + /// A run destination targeting Android generic device, using the public SDK. package static var android: Self { return .init(platform: "android", sdk: "android", sdkVariant: "android", targetArchitecture: "undefined_arch", supportedArchitectures: ["armv7", "aarch64", "riscv64", "i686", "x86_64"], disableOnlyActiveArch: true) diff --git a/Sources/SWBTestSupport/SkippedTestSupport.swift b/Sources/SWBTestSupport/SkippedTestSupport.swift index 27040e16..bef3a66e 100644 --- a/Sources/SWBTestSupport/SkippedTestSupport.swift +++ b/Sources/SWBTestSupport/SkippedTestSupport.swift @@ -49,6 +49,10 @@ extension KnownSDK { return windows case .success(.linux): return linux + case .success(.freebsd): + return freebsd + case .success(.openbsd): + return openbsd case .success(.android): return android case .success(.unknown), .failure: @@ -69,6 +73,8 @@ extension KnownSDK { extension KnownSDK { package static let windows: Self = "windows" package static let linux: Self = "linux" + package static let freebsd: Self = "freebsd" + package static let openbsd: Self = "openbsd" package static let android: Self = "android" package static let qnx: Self = "qnx" package static let wasi: Self = "wasi" @@ -196,7 +202,7 @@ extension Trait where Self == Testing.ConditionTrait { } } - package static func requireSystemPackages(apt: String..., yum: String..., sourceLocation: SourceLocation = #_sourceLocation) -> Self { + package static func requireSystemPackages(apt: String..., yum: String..., freebsd: String..., sourceLocation: SourceLocation = #_sourceLocation) -> Self { enabled("required system packages are not installed") { func checkInstalled(hostOS: OperatingSystem, packageManagerPath: Path, args: [String], packages: [String], regex: Regex<(Substring, name: Substring)>) async throws -> Bool { if try ProcessInfo.processInfo.hostOperatingSystem() == hostOS && localFS.exists(packageManagerPath) { @@ -222,7 +228,9 @@ extension Trait where Self == Testing.ConditionTrait { // spelled `--installed` in newer versions of yum, but Amazon Linux 2 is on older versions let yum = try await checkInstalled(hostOS: .linux, packageManagerPath: Path("/usr/bin/yum"), args: ["list", "installed", "yum"], packages: yum, regex: #/(?.+)\./#) - return apt && yum + let freebsd = try await checkInstalled(hostOS: .freebsd, packageManagerPath: Path("/usr/sbin/pkg"), args: ["info"], packages: freebsd, regex: #/^Name(?:[ ]+): (?.+)$/#) + + return apt && yum && freebsd } } diff --git a/Sources/SWBUtil/Architecture.swift b/Sources/SWBUtil/Architecture.swift index 340491bd..d7518ec8 100644 --- a/Sources/SWBUtil/Architecture.swift +++ b/Sources/SWBUtil/Architecture.swift @@ -98,7 +98,18 @@ public struct Architecture: Sendable { if uname(&buf) == 0 { return withUnsafeBytes(of: &buf.machine) { buf in let data = Data(buf) - return String(decoding: data[0...(data.lastIndex(where: { $0 != 0 }) ?? 0)], as: UTF8.self) + let value = String(decoding: data[0...(data.lastIndex(where: { $0 != 0 }) ?? 0)], as: UTF8.self) + #if os(FreeBSD) + switch value { + case "amd64": + return "x86_64" + case "arm64": + return "aarch64" + default: + break + } + #endif + return value } } return nil diff --git a/Sources/SWBUtil/FSProxy.swift b/Sources/SWBUtil/FSProxy.swift index adf69e3a..9f0eb4c2 100644 --- a/Sources/SWBUtil/FSProxy.swift +++ b/Sources/SWBUtil/FSProxy.swift @@ -718,6 +718,9 @@ class LocalFS: FSProxy, @unchecked Sendable { #if os(Windows) // Implement ADS on Windows? See also https://github.com/swiftlang/swift-foundation/issues/1166 return [] + #elseif os(FreeBSD) + // FreeBSD blocked on https://github.com/swiftlang/swift/pull/77836 + return [] #elseif os(OpenBSD) // OpenBSD no longer supports extended attributes return [] @@ -758,6 +761,8 @@ class LocalFS: FSProxy, @unchecked Sendable { func setExtendedAttribute(_ path: Path, key: String, value: ByteString) throws { #if os(Windows) // Implement ADS on Windows? See also https://github.com/swiftlang/swift-foundation/issues/1166 + #elseif os(FreeBSD) + // FreeBSD blocked on https://github.com/swiftlang/swift/pull/77836 #elseif os(OpenBSD) // OpenBSD no longer supports extended attributes #else @@ -778,6 +783,9 @@ class LocalFS: FSProxy, @unchecked Sendable { #if os(Windows) // Implement ADS on Windows? See also https://github.com/swiftlang/swift-foundation/issues/1166 return nil + #elseif os(FreeBSD) + // FreeBSD blocked on https://github.com/swiftlang/swift/pull/77836 + return nil #elseif os(OpenBSD) // OpenBSD no longer supports extended attributes return nil diff --git a/Sources/SWBUtil/Lock.swift b/Sources/SWBUtil/Lock.swift index abc1664a..b45625c6 100644 --- a/Sources/SWBUtil/Lock.swift +++ b/Sources/SWBUtil/Lock.swift @@ -28,7 +28,7 @@ public final class Lock: @unchecked Sendable { #if os(Windows) @usableFromInline let mutex: UnsafeMutablePointer = UnsafeMutablePointer.allocate(capacity: 1) - #elseif os(OpenBSD) + #elseif os(FreeBSD) || os(OpenBSD) @usableFromInline let mutex: UnsafeMutablePointer = UnsafeMutablePointer.allocate(capacity: 1) #else diff --git a/Sources/SWBUtil/Process.swift b/Sources/SWBUtil/Process.swift index d07f8e9f..d832eb5a 100644 --- a/Sources/SWBUtil/Process.swift +++ b/Sources/SWBUtil/Process.swift @@ -70,6 +70,8 @@ extension Process { case .linux: // Amazon Linux 2 has glibc 2.26, and glibc 2.29 is needed for posix_spawn_file_actions_addchdir_np support FileManager.default.contents(atPath: "/etc/system-release").map { String(decoding: $0, as: UTF8.self) == "Amazon Linux release 2 (Karoo)\n" } ?? false + case .openbsd: + true default: false } diff --git a/Sources/SWBUtil/ProcessInfo.swift b/Sources/SWBUtil/ProcessInfo.swift index 85b4ef2c..e96f8631 100644 --- a/Sources/SWBUtil/ProcessInfo.swift +++ b/Sources/SWBUtil/ProcessInfo.swift @@ -99,6 +99,10 @@ extension ProcessInfo { return .windows #elseif os(Linux) return .linux + #elseif os(FreeBSD) + return .freebsd + #elseif os(OpenBSD) + return .openbsd #else if try FileManager.default.isReadableFile(atPath: systemVersionPlistURL.filePath.str) { switch try systemVersion().productName { @@ -129,6 +133,8 @@ public enum OperatingSystem: Hashable, Sendable { case visionOS(simulator: Bool) case windows case linux + case freebsd + case openbsd case android case unknown @@ -157,7 +163,7 @@ public enum OperatingSystem: Hashable, Sendable { return .macho case .windows: return .pe - case .linux, .android, .unknown: + case .linux, .freebsd, .openbsd, .android, .unknown: return .elf } } diff --git a/Tests/SWBBuildSystemTests/BuildTaskBehaviorTests.swift b/Tests/SWBBuildSystemTests/BuildTaskBehaviorTests.swift index 17f091fa..aa8390b6 100644 --- a/Tests/SWBBuildSystemTests/BuildTaskBehaviorTests.swift +++ b/Tests/SWBBuildSystemTests/BuildTaskBehaviorTests.swift @@ -308,7 +308,7 @@ fileprivate struct BuildTaskBehaviorTests: CoreBasedTests { } /// Check that we honor specs which are unsafe to interrupt. - @Test(.requireSDKs(.host), .skipHostOS(.windows, "no bash shell")) + @Test(.requireSDKs(.host), .skipHostOS(.windows, "no bash shell"), .skipHostOS(.freebsd, "Currently hangs on FreeBSD")) func unsafeToInterrupt() async throws { let fs = localFS let output = MakePlannedVirtualNode("") diff --git a/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift b/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift index 169c89c7..1b541d75 100644 --- a/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift +++ b/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift @@ -242,7 +242,7 @@ import SWBMacro } } - @Test(.skipHostOS(.windows), .requireSystemPackages(apt: "libtool", yum: "libtool")) + @Test(.skipHostOS(.windows), .requireSystemPackages(apt: "libtool", yum: "libtool", freebsd: "libtool")) func discoveredLibtoolSpecInfo() async throws { try await withSpec(LibtoolLinkerSpec.self, .deferred) { (info: DiscoveredLibtoolLinkerToolSpecInfo) in #expect(info.toolPath.basename == "libtool") diff --git a/Tests/SWBCoreTests/FileTextEncodingTests.swift b/Tests/SWBCoreTests/FileTextEncodingTests.swift index 190dca66..ee898097 100644 --- a/Tests/SWBCoreTests/FileTextEncodingTests.swift +++ b/Tests/SWBCoreTests/FileTextEncodingTests.swift @@ -26,8 +26,7 @@ import SWBTestSupport #expect(FileTextEncoding("utf8") != FileTextEncoding.utf8) } - @Test(.skipHostOS(.windows, "feature not available on Windows due to missing CF APIs"), - .skipHostOS(.linux, "feature not available on Linux due to missing CF APIs")) + @Test(.requireHostOS(.macOS)) // requires CoreFoundation which is macOS-only func encoding() throws { #expect(FileTextEncoding.utf8.stringEncoding == String.Encoding.utf8) #expect(FileTextEncoding.utf16.stringEncoding == String.Encoding.utf16) diff --git a/Tests/SWBCoreTests/SettingsTests.swift b/Tests/SWBCoreTests/SettingsTests.swift index 435100e8..b32b506f 100644 --- a/Tests/SWBCoreTests/SettingsTests.swift +++ b/Tests/SWBCoreTests/SettingsTests.swift @@ -1773,7 +1773,7 @@ import SWBMacro #expect(!core.platformRegistry.platforms.isEmpty) for developmentTeam in ["ABCDWXYZ", ""] { for platform in core.platformRegistry.platforms { - if ["android", "linux", "qnx", "windows"].contains(platform.name) { + if ["android", "freebsd", "linux", "qnx", "windows"].contains(platform.name) { continue } for sdk in platform.sdks { diff --git a/Tests/SWBTaskExecutionTests/PBXCpTests.swift b/Tests/SWBTaskExecutionTests/PBXCpTests.swift index 109bba4d..cfb37a75 100644 --- a/Tests/SWBTaskExecutionTests/PBXCpTests.swift +++ b/Tests/SWBTaskExecutionTests/PBXCpTests.swift @@ -553,7 +553,7 @@ fileprivate struct PBXCpTests: CoreBasedTests { } } - @Test + @Test(.skipHostOS(.freebsd, "Currently hangs on FreeBSD")) func skipCopyIfContentsEqual() async throws { try await withTemporaryDirectory { tmp in let src = tmp.join("src") diff --git a/Tests/SWBUtilTests/ElapsedTimerTests.swift b/Tests/SWBUtilTests/ElapsedTimerTests.swift index e9f6dab6..3c364148 100644 --- a/Tests/SWBUtilTests/ElapsedTimerTests.swift +++ b/Tests/SWBUtilTests/ElapsedTimerTests.swift @@ -13,9 +13,10 @@ import Foundation import SWBUtil import Testing +import SWBTestSupport @Suite fileprivate struct ElapsedTimerTests { - @Test + @Test(.skipHostOS(.freebsd, "Currently hangs on FreeBSD")) func time() async throws { do { let delta = try await ElapsedTimer.measure { diff --git a/Tests/SWBUtilTests/FSProxyTests.swift b/Tests/SWBUtilTests/FSProxyTests.swift index 557a33d1..1f56717d 100644 --- a/Tests/SWBUtilTests/FSProxyTests.swift +++ b/Tests/SWBUtilTests/FSProxyTests.swift @@ -488,7 +488,7 @@ import SWBTestSupport case .android, .linux: // This will _usually_ be correct on Linux-derived OSes (see above), but not always. #expect(current_gid == ownership.group) - case .macOS, .iOS, .tvOS, .watchOS, .visionOS: + case .macOS, .iOS, .tvOS, .watchOS, .visionOS, .freebsd, .openbsd: #expect(parentDirOwnership.group == ownership.group) case .windows: // POSIX permissions don't exist, so everything is hardcoded to zero. @@ -566,7 +566,7 @@ import SWBTestSupport } } - @Test(.skipHostOS(.windows)) + @Test(.skipHostOS(.windows), .skipHostOS(.freebsd, "Blocked on https://github.com/swiftlang/swift/pull/77836")) func extendedAttributesSupport() throws { try withTemporaryDirectory { (tmpDir: Path) in // Many filesystems on other platforms (e.g. various non-ext4 temporary filesystems on Linux) don't support xattrs and will return ENOTSUP. diff --git a/Tests/SWBUtilTests/FileHandleTests.swift b/Tests/SWBUtilTests/FileHandleTests.swift index e118f507..51b504b3 100644 --- a/Tests/SWBUtilTests/FileHandleTests.swift +++ b/Tests/SWBUtilTests/FileHandleTests.swift @@ -22,7 +22,7 @@ import SystemPackage #endif @Suite fileprivate struct FileHandleTests { - @Test + @Test(.skipHostOS(.freebsd, "Currently crashes on FreeBSD")) func asyncReadFileDescriptor() async throws { let fs = localFS try await withTemporaryDirectory(fs: fs) { testDataPath in diff --git a/Tests/SWBUtilTests/HeavyCacheTests.swift b/Tests/SWBUtilTests/HeavyCacheTests.swift index 49c496b4..e4f0c9fb 100644 --- a/Tests/SWBUtilTests/HeavyCacheTests.swift +++ b/Tests/SWBUtilTests/HeavyCacheTests.swift @@ -14,6 +14,7 @@ import Foundation import Testing @_spi(Testing) import SWBUtil import Synchronization +import SWBTestSupport @Suite fileprivate struct HeavyCacheTests { @@ -105,7 +106,7 @@ fileprivate struct HeavyCacheTests { } /// Check initial TTL. - @Test + @Test(.skipHostOS(.freebsd, "Currently hangs on FreeBSD")) func TTL_initial() async throws { let fudgeFactor = 10.0 let ttl = Duration.seconds(0.01) @@ -124,7 +125,7 @@ fileprivate struct HeavyCacheTests { } /// Check TTL set after the fact. - @Test + @Test(.skipHostOS(.freebsd, "Currently hangs on FreeBSD")) func TTL_after() async throws { let fudgeFactor = 10.0 let ttl = Duration.seconds(0.01) diff --git a/Tests/SWBUtilTests/MiscTests.swift b/Tests/SWBUtilTests/MiscTests.swift index 36240ad9..f93094a4 100644 --- a/Tests/SWBUtilTests/MiscTests.swift +++ b/Tests/SWBUtilTests/MiscTests.swift @@ -25,7 +25,7 @@ import SWBUtil #expect(SWBUtil.userCacheDir().str.hasPrefix("/var/folders")) case .android: #expect(SWBUtil.userCacheDir().str.hasPrefix("/data/local/tmp")) - case .linux, .unknown: + case .linux, .freebsd, .openbsd, .unknown: #expect(SWBUtil.userCacheDir().str.hasPrefix("/tmp")) } } diff --git a/Tests/SWBUtilTests/RateLimiterTests.swift b/Tests/SWBUtilTests/RateLimiterTests.swift index 932b94fb..3affb9b4 100644 --- a/Tests/SWBUtilTests/RateLimiterTests.swift +++ b/Tests/SWBUtilTests/RateLimiterTests.swift @@ -13,8 +13,10 @@ import Foundation import Testing import SWBUtil +import SWBTestSupport -@Suite fileprivate struct RateLimiterTests { +@Suite(.skipHostOS(.freebsd, "Currently hangs on FreeBSD")) +fileprivate struct RateLimiterTests { @Test func rateLimiterSeconds() async throws { let timer = ElapsedTimer() From 89c1983e7119ad7a12b66f0cb12119a7d8857a98 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Fri, 6 Jun 2025 09:43:02 -0700 Subject: [PATCH 29/54] Remove references to obsolete '.swiftPlaceholder' module type. This is a leftover from a prior design which had Swift Build specify paths to to-be-built modules to the driver. This designed turned out to be not viable for multiple reasons and was abandoned a couple of years ago. We are now workgin towards removing this code. --- Sources/SWBCore/LibSwiftDriver/LibSwiftDriver.swift | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Sources/SWBCore/LibSwiftDriver/LibSwiftDriver.swift b/Sources/SWBCore/LibSwiftDriver/LibSwiftDriver.swift index bd6cd847..3f963cb9 100644 --- a/Sources/SWBCore/LibSwiftDriver/LibSwiftDriver.swift +++ b/Sources/SWBCore/LibSwiftDriver/LibSwiftDriver.swift @@ -200,7 +200,9 @@ public final class SwiftModuleDependencyGraph: SwiftGlobalExplicitDependencyGrap if let modulePath = VirtualPath.lookup(details.compiledModulePath.path).absolutePath { swiftmodulePaths.append(modulePath.pathString) } - case .clang, .swiftPlaceholder: + case .clang: + fallthrough + default: break } } @@ -223,15 +225,13 @@ public final class SwiftModuleDependencyGraph: SwiftGlobalExplicitDependencyGrap } fileDependencies.append(contentsOf: moduleInfo.sourceFiles ?? []) switch moduleInfo.details { - case .swift: - break - case .swiftPlaceholder: - break case .swiftPrebuiltExternal(let details): if let modulePath = VirtualPath.lookup(details.compiledModulePath.path).absolutePath { fileDependencies.append(modulePath.pathString) } - case .clang: + case .swift, .clang: + fallthrough + default: break } } From a43d0e386691f65eb0360e3d9c187464b823fcbe Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Wed, 4 Jun 2025 16:22:32 -0700 Subject: [PATCH 30/54] Revert "Revert "XCTest discovery support for non-Darwin platforms"" --- .../Specs/DarwinProductTypes.xcspec | 13 + Sources/SWBCSupport/IndexStore.h | 194 ++++++ Sources/SWBCSupport/SWBCSupport.h | 1 + Sources/SWBCore/Settings/BuiltinMacros.swift | 2 + .../SpecImplementations/ProductTypes.swift | 10 +- .../Tools/SwiftCompiler.swift | 3 + .../SWBGenericUnixPlatform/Specs/Unix.xcspec | 35 +- .../SWBProjectModel/PIFGenerationModel.swift | 2 + Sources/SWBQNXPlatform/Specs/QNX.xcspec | 33 +- .../ProductPlanning/ProductPlan.swift | 2 +- .../InfoPlistTaskProducer.swift | 2 +- .../SwiftStandardLibrariesTaskProducer.swift | 2 +- Sources/SWBTestSupport/TestWorkspaces.swift | 6 +- .../Specs/ProductTypes.xcspec | 15 + .../TestEntryPointGenerationTaskAction.swift | 603 +++++++++++++++++- .../TestEntryPointGenerationTool.swift | 52 ++ .../TestEntryPointTaskProducer.swift | 51 +- Sources/SWBUtil/CMakeLists.txt | 1 + Sources/SWBUtil/IndexStore.swift | 389 +++++++++++ .../SWBWindowsPlatform/Specs/Windows.xcspec | 25 + .../ProjectModel/BuildSettings.swift | 2 + Sources/SwiftBuild/ProjectModel/Targets.swift | 1 + .../BuildOperationTests.swift | 52 +- .../UnitTestTaskConstructionTests.swift | 35 +- 24 files changed, 1442 insertions(+), 89 deletions(-) create mode 100644 Sources/SWBCSupport/IndexStore.h create mode 100644 Sources/SWBUtil/IndexStore.swift diff --git a/Sources/SWBApplePlatform/Specs/DarwinProductTypes.xcspec b/Sources/SWBApplePlatform/Specs/DarwinProductTypes.xcspec index e1bc5a17..d6e43e3f 100644 --- a/Sources/SWBApplePlatform/Specs/DarwinProductTypes.xcspec +++ b/Sources/SWBApplePlatform/Specs/DarwinProductTypes.xcspec @@ -461,4 +461,17 @@ ); Platforms = (driverkit); }, + { + _Domain = darwin; + Type = ProductType; + Identifier = com.apple.product-type.tool.swiftpm-test-runner; + Name = "SwiftPM Unit Test Runner"; + Description = "SwiftPM Unit Test Runner"; + DefaultBuildProperties = { + __SKIP_BUILD = YES; + }; + PackageTypes = ( + com.apple.package-type.mach-o-executable + ); + }, ) diff --git a/Sources/SWBCSupport/IndexStore.h b/Sources/SWBCSupport/IndexStore.h new file mode 100644 index 00000000..7d4b77b8 --- /dev/null +++ b/Sources/SWBCSupport/IndexStore.h @@ -0,0 +1,194 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#ifndef INDEXSTORE_H +#define INDEXSTORE_H + +#include +#include +#include +#include + +typedef void *indexstore_error_t; + +typedef struct { + const char *data; + size_t length; +} indexstore_string_ref_t; + +typedef void *indexstore_t; +typedef void *indexstore_symbol_t; + +typedef enum { + INDEXSTORE_SYMBOL_KIND_UNKNOWN = 0, + INDEXSTORE_SYMBOL_KIND_MODULE = 1, + INDEXSTORE_SYMBOL_KIND_NAMESPACE = 2, + INDEXSTORE_SYMBOL_KIND_NAMESPACEALIAS = 3, + INDEXSTORE_SYMBOL_KIND_MACRO = 4, + INDEXSTORE_SYMBOL_KIND_ENUM = 5, + INDEXSTORE_SYMBOL_KIND_STRUCT = 6, + INDEXSTORE_SYMBOL_KIND_CLASS = 7, + INDEXSTORE_SYMBOL_KIND_PROTOCOL = 8, + INDEXSTORE_SYMBOL_KIND_EXTENSION = 9, + INDEXSTORE_SYMBOL_KIND_UNION = 10, + INDEXSTORE_SYMBOL_KIND_TYPEALIAS = 11, + INDEXSTORE_SYMBOL_KIND_FUNCTION = 12, + INDEXSTORE_SYMBOL_KIND_VARIABLE = 13, + INDEXSTORE_SYMBOL_KIND_FIELD = 14, + INDEXSTORE_SYMBOL_KIND_ENUMCONSTANT = 15, + INDEXSTORE_SYMBOL_KIND_INSTANCEMETHOD = 16, + INDEXSTORE_SYMBOL_KIND_CLASSMETHOD = 17, + INDEXSTORE_SYMBOL_KIND_STATICMETHOD = 18, + INDEXSTORE_SYMBOL_KIND_INSTANCEPROPERTY = 19, + INDEXSTORE_SYMBOL_KIND_CLASSPROPERTY = 20, + INDEXSTORE_SYMBOL_KIND_STATICPROPERTY = 21, + INDEXSTORE_SYMBOL_KIND_CONSTRUCTOR = 22, + INDEXSTORE_SYMBOL_KIND_DESTRUCTOR = 23, + INDEXSTORE_SYMBOL_KIND_CONVERSIONFUNCTION = 24, + INDEXSTORE_SYMBOL_KIND_PARAMETER = 25, + INDEXSTORE_SYMBOL_KIND_USING = 26, + + INDEXSTORE_SYMBOL_KIND_COMMENTTAG = 1000, +} indexstore_symbol_kind_t; + +typedef enum { + INDEXSTORE_SYMBOL_PROPERTY_GENERIC = 1 << 0, + INDEXSTORE_SYMBOL_PROPERTY_TEMPLATE_PARTIAL_SPECIALIZATION = 1 << 1, + INDEXSTORE_SYMBOL_PROPERTY_TEMPLATE_SPECIALIZATION = 1 << 2, + INDEXSTORE_SYMBOL_PROPERTY_UNITTEST = 1 << 3, + INDEXSTORE_SYMBOL_PROPERTY_IBANNOTATED = 1 << 4, + INDEXSTORE_SYMBOL_PROPERTY_IBOUTLETCOLLECTION = 1 << 5, + INDEXSTORE_SYMBOL_PROPERTY_GKINSPECTABLE = 1 << 6, + INDEXSTORE_SYMBOL_PROPERTY_LOCAL = 1 << 7, + INDEXSTORE_SYMBOL_PROPERTY_PROTOCOL_INTERFACE = 1 << 8, + INDEXSTORE_SYMBOL_PROPERTY_SWIFT_ASYNC = 1 << 16, +} indexstore_symbol_property_t; + +typedef enum { + INDEXSTORE_SYMBOL_ROLE_DECLARATION = 1 << 0, + INDEXSTORE_SYMBOL_ROLE_DEFINITION = 1 << 1, + INDEXSTORE_SYMBOL_ROLE_REFERENCE = 1 << 2, + INDEXSTORE_SYMBOL_ROLE_READ = 1 << 3, + INDEXSTORE_SYMBOL_ROLE_WRITE = 1 << 4, + INDEXSTORE_SYMBOL_ROLE_CALL = 1 << 5, + INDEXSTORE_SYMBOL_ROLE_DYNAMIC = 1 << 6, + INDEXSTORE_SYMBOL_ROLE_ADDRESSOF = 1 << 7, + INDEXSTORE_SYMBOL_ROLE_IMPLICIT = 1 << 8, + INDEXSTORE_SYMBOL_ROLE_UNDEFINITION = 1 << 19, + + // Relation roles. + INDEXSTORE_SYMBOL_ROLE_REL_CHILDOF = 1 << 9, + INDEXSTORE_SYMBOL_ROLE_REL_BASEOF = 1 << 10, + INDEXSTORE_SYMBOL_ROLE_REL_OVERRIDEOF = 1 << 11, + INDEXSTORE_SYMBOL_ROLE_REL_RECEIVEDBY = 1 << 12, + INDEXSTORE_SYMBOL_ROLE_REL_CALLEDBY = 1 << 13, + INDEXSTORE_SYMBOL_ROLE_REL_EXTENDEDBY = 1 << 14, + INDEXSTORE_SYMBOL_ROLE_REL_ACCESSOROF = 1 << 15, + INDEXSTORE_SYMBOL_ROLE_REL_CONTAINEDBY = 1 << 16, + INDEXSTORE_SYMBOL_ROLE_REL_IBTYPEOF = 1 << 17, + INDEXSTORE_SYMBOL_ROLE_REL_SPECIALIZATIONOF = 1 << 18, +} indexstore_symbol_role_t; + +typedef void *indexstore_unit_dependency_t; + +typedef enum { + INDEXSTORE_UNIT_DEPENDENCY_UNIT = 1, + INDEXSTORE_UNIT_DEPENDENCY_RECORD = 2, + INDEXSTORE_UNIT_DEPENDENCY_FILE = 3, +} indexstore_unit_dependency_kind_t; + +typedef void *indexstore_symbol_relation_t; +typedef void *indexstore_occurrence_t; +typedef void *indexstore_record_reader_t; +typedef void *indexstore_unit_reader_t; + +typedef struct { + const char * + (*error_get_description)(indexstore_error_t); + + void + (*error_dispose)(indexstore_error_t); + + indexstore_t + (*store_create)(const char *store_path, indexstore_error_t *error); + + void + (*store_dispose)(indexstore_t); + + size_t + (*store_get_unit_name_from_output_path)(indexstore_t store, + const char *output_path, + char *name_buf, + size_t buf_size); + + indexstore_symbol_kind_t + (*symbol_get_kind)(indexstore_symbol_t); + + uint64_t + (*symbol_get_properties)(indexstore_symbol_t); + + indexstore_string_ref_t + (*symbol_get_name)(indexstore_symbol_t); + + uint64_t + (*symbol_relation_get_roles)(indexstore_symbol_relation_t); + + indexstore_symbol_t + (*symbol_relation_get_symbol)(indexstore_symbol_relation_t); + + indexstore_symbol_t + (*occurrence_get_symbol)(indexstore_occurrence_t); + + bool + (*occurrence_relations_apply_f)(indexstore_occurrence_t, + void *context, + bool(*applier)(void *context, indexstore_symbol_relation_t symbol_rel)); + + indexstore_record_reader_t + (*record_reader_create)(indexstore_t store, const char *record_name, + indexstore_error_t *error); + + void + (*record_reader_dispose)(indexstore_record_reader_t); + + bool + (*record_reader_occurrences_apply_f)(indexstore_record_reader_t, + void *context, + bool(*applier)(void *context, indexstore_occurrence_t occur)); + + indexstore_unit_reader_t + (*unit_reader_create)(indexstore_t store, const char *unit_name, + indexstore_error_t *error); + + void + (*unit_reader_dispose)(indexstore_unit_reader_t); + + indexstore_string_ref_t + (*unit_reader_get_module_name)(indexstore_unit_reader_t); + + indexstore_unit_dependency_kind_t + (*unit_dependency_get_kind)(indexstore_unit_dependency_t); + + indexstore_string_ref_t + (*unit_dependency_get_name)(indexstore_unit_dependency_t); + + bool + (*unit_reader_dependencies_apply)(indexstore_unit_reader_t, + bool(^applier)(indexstore_unit_dependency_t)); + + bool + (*unit_reader_dependencies_apply_f)(indexstore_unit_reader_t, + void *context, + bool(*applier)(void *context, indexstore_unit_dependency_t)); +} indexstore_functions_t; + +#endif diff --git a/Sources/SWBCSupport/SWBCSupport.h b/Sources/SWBCSupport/SWBCSupport.h index c020472c..18591c94 100644 --- a/Sources/SWBCSupport/SWBCSupport.h +++ b/Sources/SWBCSupport/SWBCSupport.h @@ -21,6 +21,7 @@ #include "CLibclang.h" #include "CLibRemarksHelper.h" +#include "IndexStore.h" #include "PluginAPI.h" #include "PluginAPI_functions.h" #include "PluginAPI_types.h" diff --git a/Sources/SWBCore/Settings/BuiltinMacros.swift b/Sources/SWBCore/Settings/BuiltinMacros.swift index 7fbab73d..21be4140 100644 --- a/Sources/SWBCore/Settings/BuiltinMacros.swift +++ b/Sources/SWBCore/Settings/BuiltinMacros.swift @@ -750,6 +750,7 @@ public final class BuiltinMacros { public static let INDEX_PREPARED_TARGET_MARKER_PATH = BuiltinMacros.declareStringMacro("INDEX_PREPARED_TARGET_MARKER_PATH") public static let INDEX_REGULAR_BUILD_PRODUCTS_DIR = BuiltinMacros.declareStringMacro("INDEX_REGULAR_BUILD_PRODUCTS_DIR") public static let INDEX_REGULAR_BUILD_INTERMEDIATES_DIR = BuiltinMacros.declareStringMacro("INDEX_REGULAR_BUILD_INTERMEDIATES_DIR") + public static let INDEX_STORE_LIBRARY_PATH = BuiltinMacros.declarePathMacro("INDEX_STORE_LIBRARY_PATH") public static let INFOPLIST_ENFORCE_MINIMUM_OS = BuiltinMacros.declareBooleanMacro("INFOPLIST_ENFORCE_MINIMUM_OS") public static let INFOPLIST_EXPAND_BUILD_SETTINGS = BuiltinMacros.declareBooleanMacro("INFOPLIST_EXPAND_BUILD_SETTINGS") public static let INFOPLIST_FILE = BuiltinMacros.declarePathMacro("INFOPLIST_FILE") @@ -1797,6 +1798,7 @@ public final class BuiltinMacros { INDEX_PREPARED_TARGET_MARKER_PATH, INDEX_REGULAR_BUILD_PRODUCTS_DIR, INDEX_REGULAR_BUILD_INTERMEDIATES_DIR, + INDEX_STORE_LIBRARY_PATH, INDEX_ENABLE_DATA_STORE, INDEX_PRECOMPS_DIR, INFOPLIST_ENFORCE_MINIMUM_OS, diff --git a/Sources/SWBCore/SpecImplementations/ProductTypes.swift b/Sources/SWBCore/SpecImplementations/ProductTypes.swift index 7cb6a1d5..28415e4b 100644 --- a/Sources/SWBCore/SpecImplementations/ProductTypes.swift +++ b/Sources/SWBCore/SpecImplementations/ProductTypes.swift @@ -321,7 +321,7 @@ public class ProductTypeSpec : Spec, SpecType, @unchecked Sendable { } /// Returns whether the product type supports embedding Swift standard libraries inside it. - public var supportsEmbeddingSwiftStandardLibraries: Bool { + public func supportsEmbeddingSwiftStandardLibraries(producer: CommandProducer) -> Bool { // Most product types don't support having the Swift libraries embedded in them. return false } @@ -381,7 +381,7 @@ public final class ApplicationProductTypeSpec : BundleProductTypeSpec, @unchecke return "PBXApplicationProductType" } - public override var supportsEmbeddingSwiftStandardLibraries: Bool { + public override func supportsEmbeddingSwiftStandardLibraries(producer: CommandProducer) -> Bool { return true } @@ -602,8 +602,8 @@ public final class XCTestBundleProductTypeSpec : BundleProductTypeSpec, @uncheck super.init(parser, basedOnSpec) } - public override var supportsEmbeddingSwiftStandardLibraries: Bool { - return true + public override func supportsEmbeddingSwiftStandardLibraries(producer: CommandProducer) -> Bool { + return producer.isApplePlatform } public class func usesXCTRunner(_ scope: MacroEvaluationScope) -> Bool { @@ -649,7 +649,7 @@ public final class XCTestBundleProductTypeSpec : BundleProductTypeSpec, @uncheck var (tableOpt, warnings, errors) = super.overridingBuildSettings(scope, platform: platform) var table = tableOpt ?? MacroValueAssignmentTable(namespace: scope.namespace) - let isDeviceBuild = platform?.isDeploymentPlatform == true && platform?.identifier != "com.apple.platform.macosx" + let isDeviceBuild = platform?.isDeploymentPlatform == true && platform?.name != scope.evaluate(BuiltinMacros.HOST_PLATFORM) if isDeviceBuild { // For tests running on devices (not simulators) we always want to generate dSYMs so that symbolication can give file and line information about test failures. table.push(BuiltinMacros.DEBUG_INFORMATION_FORMAT, literal: "dwarf-with-dsym") diff --git a/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift b/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift index 70e9ad3a..687820ec 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift @@ -3763,6 +3763,9 @@ public extension BuildPhaseWithBuildFiles { /// - Returns: If the build phase contains any Swift source files that are not filtered out via the platform filter or excluded source file name patterns. func containsSwiftSources(_ referenceLookupContext: any ReferenceLookupContext, _ specLookupContext: any SpecLookupContext, _ scope: MacroEvaluationScope, _ filePathResolver: FilePathResolver) -> Bool { guard let swiftFileType = specLookupContext.lookupFileType(identifier: "sourcecode.swift") else { return false } + if scope.evaluate(BuiltinMacros.GENERATE_TEST_ENTRY_POINT) { + return true + } return containsFiles(ofType: swiftFileType, referenceLookupContext, specLookupContext, scope, filePathResolver) } } diff --git a/Sources/SWBGenericUnixPlatform/Specs/Unix.xcspec b/Sources/SWBGenericUnixPlatform/Specs/Unix.xcspec index eb72ba9c..00bc75d5 100644 --- a/Sources/SWBGenericUnixPlatform/Specs/Unix.xcspec +++ b/Sources/SWBGenericUnixPlatform/Specs/Unix.xcspec @@ -24,41 +24,18 @@ SortNumber = 0; }, - // Test type bundle (bodged to be a tool) { Domain = generic-unix; Type = ProductType; Identifier = com.apple.product-type.bundle.unit-test; - Class = PBXToolProductType; - Name = "Command-line Tool"; - Description = "Standalone command-line tool"; - DefaultTargetName = "Command-line Tool"; + BasedOn = com.apple.product-type.library.dynamic; DefaultBuildProperties = { - FULL_PRODUCT_NAME = "$(EXECUTABLE_NAME)"; - EXECUTABLE_PREFIX = ""; - EXECUTABLE_SUFFIX = ".xctest"; - REZ_EXECUTABLE = YES; - INSTALL_PATH = "/usr/local/bin"; - FRAMEWORK_FLAG_PREFIX = "-framework"; - LIBRARY_FLAG_PREFIX = "-l"; - LIBRARY_FLAG_NOSPACE = YES; - GCC_DYNAMIC_NO_PIC = NO; - LD_NO_PIE = NO; - GCC_SYMBOLS_PRIVATE_EXTERN = YES; - GCC_INLINES_ARE_PRIVATE_EXTERN = YES; - STRIP_STYLE = "all"; - CODE_SIGNING_ALLOWED = NO; - IsUnitTest = YES; - SWIFT_FORCE_DYNAMIC_LINK_STDLIB = YES; - SWIFT_FORCE_STATIC_LINK_STDLIB = NO; - // Avoid warning for executable types - ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = NO; - GENERATE_TEST_ENTRY_POINT = YES; - GENERATED_TEST_ENTRY_POINT_PATH = "$(DERIVED_SOURCES_DIR)/test_entry_point.swift"; + // Index store data is required to discover XCTest tests + COMPILER_INDEX_STORE_ENABLE = YES; + SWIFT_INDEX_STORE_ENABLE = YES; + // Testability is needed to generate code to invoke discovered XCTest tests + SWIFT_ENABLE_TESTABILITY = YES; }; - PackageTypes = ( - com.apple.package-type.mach-o-executable // default - ); }, // Dynamic library (masquerading as a framework to placate Swift's project structure) diff --git a/Sources/SWBProjectModel/PIFGenerationModel.swift b/Sources/SWBProjectModel/PIFGenerationModel.swift index 698a7343..f3db2bce 100644 --- a/Sources/SWBProjectModel/PIFGenerationModel.swift +++ b/Sources/SWBProjectModel/PIFGenerationModel.swift @@ -295,6 +295,7 @@ public enum PIF { case executable = "com.apple.product-type.tool" case hostBuildTool = "com.apple.product-type.tool.host-build" case unitTest = "com.apple.product-type.bundle.unit-test" + case swiftpmTestRunner = "com.apple.product-type.tool.swiftpm-test-runner" case bundle = "com.apple.product-type.bundle" case packageProduct = "packageProduct" public var asString: String { return rawValue } @@ -1022,6 +1023,7 @@ public enum PIF { public var SWIFT_ADD_TOOLCHAIN_SWIFTSYNTAX_SEARCH_PATHS: String? public var SWIFT_FORCE_STATIC_LINK_STDLIB: String? public var SWIFT_FORCE_DYNAMIC_LINK_STDLIB: String? + public var SWIFT_INDEX_STORE_ENABLE: String? public var SWIFT_INSTALL_OBJC_HEADER: String? public var SWIFT_LOAD_BINARY_MACROS: [String]? public var SWIFT_MODULE_ALIASES: [String]? diff --git a/Sources/SWBQNXPlatform/Specs/QNX.xcspec b/Sources/SWBQNXPlatform/Specs/QNX.xcspec index 3c72c620..aea12e3b 100644 --- a/Sources/SWBQNXPlatform/Specs/QNX.xcspec +++ b/Sources/SWBQNXPlatform/Specs/QNX.xcspec @@ -24,39 +24,18 @@ SortNumber = 0; }, - // Test type bundle (bodged to be a tool) { Domain = qnx; Type = ProductType; Identifier = com.apple.product-type.bundle.unit-test; - Class = PBXToolProductType; - Name = "Command-line Tool"; - Description = "Standalone command-line tool"; - DefaultTargetName = "Command-line Tool"; + BasedOn = com.apple.product-type.library.dynamic; DefaultBuildProperties = { - FULL_PRODUCT_NAME = "$(EXECUTABLE_NAME)"; - EXECUTABLE_PREFIX = ""; - EXECUTABLE_SUFFIX = ".xctest"; - REZ_EXECUTABLE = YES; - INSTALL_PATH = "/usr/local/bin"; - FRAMEWORK_FLAG_PREFIX = "-framework"; - LIBRARY_FLAG_PREFIX = "-l"; - LIBRARY_FLAG_NOSPACE = YES; - GCC_DYNAMIC_NO_PIC = NO; - LD_NO_PIE = NO; - GCC_SYMBOLS_PRIVATE_EXTERN = YES; - GCC_INLINES_ARE_PRIVATE_EXTERN = YES; - STRIP_STYLE = "all"; - CODE_SIGNING_ALLOWED = NO; - IsUnitTest = YES; - SWIFT_FORCE_DYNAMIC_LINK_STDLIB = YES; - SWIFT_FORCE_STATIC_LINK_STDLIB = NO; - // Avoid warning for executable types - ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = NO; + // Index store data is required to discover XCTest tests + COMPILER_INDEX_STORE_ENABLE = YES; + SWIFT_INDEX_STORE_ENABLE = YES; + // Testability is needed to generate code to invoke discovered XCTest tests + SWIFT_ENABLE_TESTABILITY = YES; }; - PackageTypes = ( - com.apple.package-type.mach-o-executable // default - ); }, // Dynamic library (masquerading as a framework to placate Swift's project structure) diff --git a/Sources/SWBTaskConstruction/ProductPlanning/ProductPlan.swift b/Sources/SWBTaskConstruction/ProductPlanning/ProductPlan.swift index 88d2edfe..883073b0 100644 --- a/Sources/SWBTaskConstruction/ProductPlanning/ProductPlan.swift +++ b/Sources/SWBTaskConstruction/ProductPlanning/ProductPlan.swift @@ -31,7 +31,7 @@ package protocol GlobalProductPlanDelegate: CoreClientTargetDiagnosticProducingD package final class GlobalProductPlan: GlobalTargetInfoProvider { /// The build plan request. - let planRequest: BuildPlanRequest + package let planRequest: BuildPlanRequest /// The target task info for each configured target. private(set) var targetTaskInfos: [ConfiguredTarget: TargetTaskInfo] diff --git a/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/InfoPlistTaskProducer.swift b/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/InfoPlistTaskProducer.swift index 1f740497..820a55f5 100644 --- a/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/InfoPlistTaskProducer.swift +++ b/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/InfoPlistTaskProducer.swift @@ -57,7 +57,7 @@ private extension ProductTypeSpec break } - fatalError("unknown product type") + fatalError("unknown product type \(type(of: self))") } } diff --git a/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/SwiftStandardLibrariesTaskProducer.swift b/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/SwiftStandardLibrariesTaskProducer.swift index bba8b051..c74e1843 100644 --- a/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/SwiftStandardLibrariesTaskProducer.swift +++ b/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/SwiftStandardLibrariesTaskProducer.swift @@ -41,7 +41,7 @@ final class SwiftStandardLibrariesTaskProducer: PhasedTaskProducer, TaskProducer let buildingAnySwiftSourceFiles = (context.configuredTarget?.target as? BuildPhaseTarget)?.sourcesBuildPhase?.containsSwiftSources(context.workspaceContext.workspace, context, scope, context.filePathResolver) ?? false // Determine whether we want to embed swift libraries. - var shouldEmbedSwiftLibraries = (buildingAnySwiftSourceFiles && productType.supportsEmbeddingSwiftStandardLibraries) + var shouldEmbedSwiftLibraries = (buildingAnySwiftSourceFiles && productType.supportsEmbeddingSwiftStandardLibraries(producer: context)) // If ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES then we will override our earlier reasoning if the product is a wrapper. if !shouldEmbedSwiftLibraries && scope.evaluate(BuiltinMacros.ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES) { diff --git a/Sources/SWBTestSupport/TestWorkspaces.swift b/Sources/SWBTestSupport/TestWorkspaces.swift index c225eb2c..6837bcf7 100644 --- a/Sources/SWBTestSupport/TestWorkspaces.swift +++ b/Sources/SWBTestSupport/TestWorkspaces.swift @@ -924,6 +924,7 @@ package final class TestStandardTarget: TestInternalTarget, Sendable { case extensionKitExtension case xcodeExtension case unitTest + case swiftpmTestRunner case uiTest case multiDeviceUITest case systemExtension @@ -972,6 +973,8 @@ package final class TestStandardTarget: TestInternalTarget, Sendable { return "com.apple.product-type.xcode-extension" case .unitTest: return "com.apple.product-type.bundle.unit-test" + case .swiftpmTestRunner: + return "com.apple.product-type.tool.swiftpm-test-runner" case .uiTest: return "com.apple.product-type.bundle.ui-testing" case .multiDeviceUITest: @@ -1015,7 +1018,8 @@ package final class TestStandardTarget: TestInternalTarget, Sendable { .appClip: return "\(name).app" case .commandLineTool, - .hostBuildTool: + .hostBuildTool, + .swiftpmTestRunner: return "\(name)" case .framework, .staticFramework: diff --git a/Sources/SWBUniversalPlatform/Specs/ProductTypes.xcspec b/Sources/SWBUniversalPlatform/Specs/ProductTypes.xcspec index 0067050f..bd6cf9c8 100644 --- a/Sources/SWBUniversalPlatform/Specs/ProductTypes.xcspec +++ b/Sources/SWBUniversalPlatform/Specs/ProductTypes.xcspec @@ -312,4 +312,19 @@ IsUnitTest = YES; WantsBundleIdentifierEditing = NO; }, + // SwiftPM test runner + { Type = ProductType; + Identifier = com.apple.product-type.tool.swiftpm-test-runner; + BasedOn = com.apple.product-type.tool; + Name = "SwiftPM Unit Test Runner"; + Description = "SwiftPM Unit Test Runner"; + DefaultBuildProperties = { + ENABLE_TESTING_SEARCH_PATHS = YES; + GENERATE_TEST_ENTRY_POINT = YES; + GENERATED_TEST_ENTRY_POINT_PATH = "$(DERIVED_SOURCES_DIR)/test_entry_point.swift"; + }; + PackageTypes = ( + com.apple.package-type.mach-o-executable + ); + }, ) diff --git a/Sources/SWBUniversalPlatform/TestEntryPointGenerationTaskAction.swift b/Sources/SWBUniversalPlatform/TestEntryPointGenerationTaskAction.swift index 57dff473..9765719a 100644 --- a/Sources/SWBUniversalPlatform/TestEntryPointGenerationTaskAction.swift +++ b/Sources/SWBUniversalPlatform/TestEntryPointGenerationTaskAction.swift @@ -23,10 +23,37 @@ class TestEntryPointGenerationTaskAction: TaskAction { override func performTaskAction(_ task: any ExecutableTask, dynamicExecutionDelegate: any DynamicTaskExecutionDelegate, executionDelegate: any TaskExecutionDelegate, clientDelegate: any TaskExecutionClientDelegate, outputDelegate: any TaskOutputDelegate) async -> CommandResult { do { let options = try Options.parse(Array(task.commandLineAsStrings.dropFirst())) - try executionDelegate.fs.write(options.output, contents: #""" + + var tests: [IndexStore.TestCaseClass] = [] + var objects: [Path] = [] + for linkerFilelist in options.linkerFilelist { + let filelistContents = String(String(decoding: try executionDelegate.fs.read(linkerFilelist), as: UTF8.self)) + let entries = filelistContents.split(separator: "\n", omittingEmptySubsequences: true).map { Path($0) }.map { + for indexUnitBasePath in options.indexUnitBasePath { + if let remappedPath = generateIndexOutputPath(from: $0, basePath: indexUnitBasePath) { + return remappedPath + } + } + return $0 + } + objects.append(contentsOf: entries) + } + let indexStoreAPI = try IndexStoreAPI(dylib: options.indexStoreLibraryPath) + for indexStore in options.indexStore { + let store = try IndexStore.open(store: indexStore, api: indexStoreAPI) + let testInfo = try store.listTests(in: objects) + tests.append(contentsOf: testInfo) + } + + try executionDelegate.fs.write(options.output, contents: ByteString(encodingAsUTF8: """ #if canImport(Testing) import Testing #endif + + \(testObservationFragment) + + import XCTest + \(discoveredTestsFragment(tests: tests)) @main @available(macOS 10.15, iOS 11, watchOS 4, tvOS 11, visionOS 1, *) @@ -44,6 +71,16 @@ class TestEntryPointGenerationTaskAction: TaskAction { return "xctest" } + private static func testOutputPath() -> String? { + var iterator = CommandLine.arguments.makeIterator() + while let argument = iterator.next() { + if argument == "--testing-output-path", let outputPath = iterator.next() { + return outputPath + } + } + return nil + } + #if os(Linux) @_silgen_name("$ss13_runAsyncMainyyyyYaKcF") private static func _runAsyncMain(_ asyncFun: @Sendable @escaping () async throws -> ()) @@ -57,6 +94,16 @@ class TestEntryPointGenerationTaskAction: TaskAction { } } #endif + if testingLibrary == "xctest" { + #if !os(Windows) && \(options.enableExperimentalTestOutput) + _ = Self.testOutputPath().map { SwiftPMXCTestObserver(testOutputPath: testOutputPath) } + #endif + #if os(WASI) + await XCTMain(__allDiscoveredTests()) as Never + #else + XCTMain(__allDiscoveredTests()) as Never + #endif + } } #else static func main() async { @@ -66,18 +113,564 @@ class TestEntryPointGenerationTaskAction: TaskAction { await Testing.__swiftPMEntryPoint() as Never } #endif + if testingLibrary == "xctest" { + #if !os(Windows) && \(options.enableExperimentalTestOutput) + _ = Self.testOutputPath().map { SwiftPMXCTestObserver(testOutputPath: testOutputPath) } + #endif + #if os(WASI) + await XCTMain(__allDiscoveredTests()) as Never + #else + XCTMain(__allDiscoveredTests()) as Never + #endif + } } #endif } - """#) + """)) + return .succeeded } catch { outputDelegate.emitError("\(error)") return .failed } } -} -private struct Options: ParsableArguments { - @Option var output: Path + private struct Options: ParsableArguments { + @Option var output: Path + @Option var indexStoreLibraryPath: Path + @Option var linkerFilelist: [Path] + @Option var indexStore: [Path] + @Option var indexUnitBasePath: [Path] + @Flag var enableExperimentalTestOutput: Bool = false + } + + private func discoveredTestsFragment(tests: [IndexStore.TestCaseClass]) -> String { + var fragment = "" + for moduleName in Set(tests.map { $0.module }).sorted() { + fragment += "@testable import \(moduleName)\n" + } + fragment += """ + @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings") + public func __allDiscoveredTests() -> [XCTestCaseEntry] { + return [ + + """ + for testClass in tests { + + let testTuples = testClass.testMethods.map { method in + let basename = method.name.hasSuffix("()") ? String(method.name.dropLast(2)) : method.name + if method.isAsync { + return " (\"\(basename)\", asyncTest(\(testClass.name).\(basename)))" + } else { + return " (\"\(basename)\", \(testClass.name).\(basename))" + } + } + fragment += " testCase([\(testTuples.joined(separator: ",\n"))]),\n" + } + fragment += """ + ] + } + """ + return fragment + } + + private var testObservationFragment: String = + """ + #if !os(Windows) // Test observation is not supported on Windows + import Foundation + import XCTest + + public final class SwiftPMXCTestObserver: NSObject { + let testOutputPath: String + + public init(testOutputPath: String) { + self.testOutputPath = testOutputPath + super.init() + XCTestObservationCenter.shared.addTestObserver(self) + } + } + + extension SwiftPMXCTestObserver: XCTestObservation { + private func write(record: any Encodable) { + let lock = FileLock(at: URL(fileURLWithPath: self.testOutputPath + ".lock")) + _ = try? lock.withLock { + self._write(record: record) + } + } + + private func _write(record: any Encodable) { + if let data = try? JSONEncoder().encode(record) { + if let fileHandle = FileHandle(forWritingAtPath: self.testOutputPath) { + defer { fileHandle.closeFile() } + fileHandle.seekToEndOfFile() + fileHandle.write("\\n".data(using: .utf8)!) + fileHandle.write(data) + } else { + _ = try? data.write(to: URL(fileURLWithPath: self.testOutputPath)) + } + } + } + + public func testBundleWillStart(_ testBundle: Bundle) { + let record = TestBundleEventRecord(bundle: .init(testBundle), event: .start) + write(record: TestEventRecord(bundleEvent: record)) + } + + public func testSuiteWillStart(_ testSuite: XCTestSuite) { + let record = TestSuiteEventRecord(suite: .init(testSuite), event: .start) + write(record: TestEventRecord(suiteEvent: record)) + } + + public func testCaseWillStart(_ testCase: XCTestCase) { + let record = TestCaseEventRecord(testCase: .init(testCase), event: .start) + write(record: TestEventRecord(caseEvent: record)) + } + + #if canImport(Darwin) + public func testCase(_ testCase: XCTestCase, didRecord issue: XCTIssue) { + let record = TestCaseFailureRecord(testCase: .init(testCase), issue: .init(issue), failureKind: .unexpected) + write(record: TestEventRecord(caseFailure: record)) + } + + public func testCase(_ testCase: XCTestCase, didRecord expectedFailure: XCTExpectedFailure) { + let record = TestCaseFailureRecord(testCase: .init(testCase), issue: .init(expectedFailure.issue), failureKind: .expected(failureReason: expectedFailure.failureReason)) + write(record: TestEventRecord(caseFailure: record)) + } + #else + public func testCase(_ testCase: XCTestCase, didFailWithDescription description: String, inFile filePath: String?, atLine lineNumber: Int) { + let issue = TestIssue(description: description, inFile: filePath, atLine: lineNumber) + let record = TestCaseFailureRecord(testCase: .init(testCase), issue: issue, failureKind: .unexpected) + write(record: TestEventRecord(caseFailure: record)) + } + #endif + + public func testCaseDidFinish(_ testCase: XCTestCase) { + let record = TestCaseEventRecord(testCase: .init(testCase), event: .finish) + write(record: TestEventRecord(caseEvent: record)) + } + + #if canImport(Darwin) + public func testSuite(_ testSuite: XCTestSuite, didRecord issue: XCTIssue) { + let record = TestSuiteFailureRecord(suite: .init(testSuite), issue: .init(issue), failureKind: .unexpected) + write(record: TestEventRecord(suiteFailure: record)) + } + + public func testSuite(_ testSuite: XCTestSuite, didRecord expectedFailure: XCTExpectedFailure) { + let record = TestSuiteFailureRecord(suite: .init(testSuite), issue: .init(expectedFailure.issue), failureKind: .expected(failureReason: expectedFailure.failureReason)) + write(record: TestEventRecord(suiteFailure: record)) + } + #else + public func testSuite(_ testSuite: XCTestSuite, didFailWithDescription description: String, inFile filePath: String?, atLine lineNumber: Int) { + let issue = TestIssue(description: description, inFile: filePath, atLine: lineNumber) + let record = TestSuiteFailureRecord(suite: .init(testSuite), issue: issue, failureKind: .unexpected) + write(record: TestEventRecord(suiteFailure: record)) + } + #endif + + public func testSuiteDidFinish(_ testSuite: XCTestSuite) { + let record = TestSuiteEventRecord(suite: .init(testSuite), event: .finish) + write(record: TestEventRecord(suiteEvent: record)) + } + + public func testBundleDidFinish(_ testBundle: Bundle) { + let record = TestBundleEventRecord(bundle: .init(testBundle), event: .finish) + write(record: TestEventRecord(bundleEvent: record)) + } + } + + // FIXME: Copied from `Lock.swift` in TSCBasic, would be nice if we had a better way + + #if canImport(Glibc) + @_exported import Glibc + #elseif canImport(Musl) + @_exported import Musl + #elseif os(Windows) + @_exported import CRT + @_exported import WinSDK + #elseif os(WASI) + @_exported import WASILibc + #elseif canImport(Android) + @_exported import Android + #else + @_exported import Darwin.C + #endif + + import Foundation + + public final class FileLock { + #if os(Windows) + private var handle: HANDLE? + #else + private var fileDescriptor: CInt? + #endif + + private let lockFile: URL + + public init(at lockFile: URL) { + self.lockFile = lockFile + } + + public func lock() throws { + #if os(Windows) + if handle == nil { + let h: HANDLE = lockFile.path.withCString(encodedAs: UTF16.self, { + CreateFileW( + $0, + UInt32(GENERIC_READ) | UInt32(GENERIC_WRITE), + UInt32(FILE_SHARE_READ) | UInt32(FILE_SHARE_WRITE), + nil, + DWORD(OPEN_ALWAYS), + DWORD(FILE_ATTRIBUTE_NORMAL), + nil + ) + }) + if h == INVALID_HANDLE_VALUE { + throw FileSystemError(errno: Int32(GetLastError()), lockFile) + } + self.handle = h + } + var overlapped = OVERLAPPED() + overlapped.Offset = 0 + overlapped.OffsetHigh = 0 + overlapped.hEvent = nil + if !LockFileEx(handle, DWORD(LOCKFILE_EXCLUSIVE_LOCK), 0, + UInt32.max, UInt32.max, &overlapped) { + throw ProcessLockError.unableToAquireLock(errno: Int32(GetLastError())) + } + #elseif os(WASI) + // WASI doesn't support flock + #else + if fileDescriptor == nil { + let fd = open(lockFile.path, O_WRONLY | O_CREAT | O_CLOEXEC, 0o666) + if fd == -1 { + fatalError("errno: \\(errno), lockFile: \\(lockFile)") + } + self.fileDescriptor = fd + } + while true { + if flock(fileDescriptor!, LOCK_EX) == 0 { + break + } + if errno == EINTR { continue } + fatalError("unable to acquire lock, errno: \\(errno)") + } + #endif + } + + public func unlock() { + #if os(Windows) + var overlapped = OVERLAPPED() + overlapped.Offset = 0 + overlapped.OffsetHigh = 0 + overlapped.hEvent = nil + UnlockFileEx(handle, 0, UInt32.max, UInt32.max, &overlapped) + #elseif os(WASI) + // WASI doesn't support flock + #else + guard let fd = fileDescriptor else { return } + flock(fd, LOCK_UN) + #endif + } + + deinit { + #if os(Windows) + guard let handle = handle else { return } + CloseHandle(handle) + #elseif os(WASI) + // WASI doesn't support flock + #else + guard let fd = fileDescriptor else { return } + close(fd) + #endif + } + + public func withLock(_ body: () throws -> T) throws -> T { + try lock() + defer { unlock() } + return try body() + } + + public func withLock(_ body: () async throws -> T) async throws -> T { + try lock() + defer { unlock() } + return try await body() + } + } + + // FIXME: Copied from `XCTEvents.swift`, would be nice if we had a better way + + struct TestEventRecord: Codable { + let caseFailure: TestCaseFailureRecord? + let suiteFailure: TestSuiteFailureRecord? + + let bundleEvent: TestBundleEventRecord? + let suiteEvent: TestSuiteEventRecord? + let caseEvent: TestCaseEventRecord? + + init( + caseFailure: TestCaseFailureRecord? = nil, + suiteFailure: TestSuiteFailureRecord? = nil, + bundleEvent: TestBundleEventRecord? = nil, + suiteEvent: TestSuiteEventRecord? = nil, + caseEvent: TestCaseEventRecord? = nil + ) { + self.caseFailure = caseFailure + self.suiteFailure = suiteFailure + self.bundleEvent = bundleEvent + self.suiteEvent = suiteEvent + self.caseEvent = caseEvent + } + } + + // MARK: - Records + + struct TestAttachment: Codable { + let name: String? + // TODO: Handle `userInfo: [AnyHashable : Any]?` + let uniformTypeIdentifier: String + let payload: Data? + } + + struct TestBundleEventRecord: Codable { + let bundle: TestBundle + let event: TestEvent + } + + struct TestCaseEventRecord: Codable { + let testCase: TestCase + let event: TestEvent + } + + struct TestCaseFailureRecord: Codable, CustomStringConvertible { + let testCase: TestCase + let issue: TestIssue + let failureKind: TestFailureKind + + var description: String { + return "\\(issue.sourceCodeContext.description)\\(testCase) \\(issue.compactDescription)" + } + } + + struct TestSuiteEventRecord: Codable { + let suite: TestSuiteRecord + let event: TestEvent + } + + struct TestSuiteFailureRecord: Codable { + let suite: TestSuiteRecord + let issue: TestIssue + let failureKind: TestFailureKind + } + + // MARK: Primitives + + struct TestBundle: Codable { + let bundleIdentifier: String? + let bundlePath: String + } + + struct TestCase: Codable { + let name: String + } + + struct TestErrorInfo: Codable { + let description: String + let type: String + } + + enum TestEvent: Codable { + case start + case finish + } + + enum TestFailureKind: Codable, Equatable { + case unexpected + case expected(failureReason: String?) + + var isExpected: Bool { + switch self { + case .expected: return true + case .unexpected: return false + } + } + } + + struct TestIssue: Codable { + let type: TestIssueType + let compactDescription: String + let detailedDescription: String? + let associatedError: TestErrorInfo? + let sourceCodeContext: TestSourceCodeContext + let attachments: [TestAttachment] + } + + enum TestIssueType: Codable { + case assertionFailure + case performanceRegression + case system + case thrownError + case uncaughtException + case unmatchedExpectedFailure + case unknown + } + + struct TestLocation: Codable, CustomStringConvertible { + let file: String + let line: Int + + var description: String { + return "\\(file):\\(line) " + } + } + + struct TestSourceCodeContext: Codable, CustomStringConvertible { + let callStack: [TestSourceCodeFrame] + let location: TestLocation? + + var description: String { + return location?.description ?? "" + } + } + + struct TestSourceCodeFrame: Codable { + let address: UInt64 + let symbolInfo: TestSourceCodeSymbolInfo? + let symbolicationError: TestErrorInfo? + } + + struct TestSourceCodeSymbolInfo: Codable { + let imageName: String + let symbolName: String + let location: TestLocation? + } + + struct TestSuiteRecord: Codable { + let name: String + } + + // MARK: XCTest compatibility + + extension TestIssue { + init(description: String, inFile filePath: String?, atLine lineNumber: Int) { + let location: TestLocation? + if let filePath = filePath { + location = .init(file: filePath, line: lineNumber) + } else { + location = nil + } + self.init(type: .assertionFailure, compactDescription: description, detailedDescription: description, associatedError: nil, sourceCodeContext: .init(callStack: [], location: location), attachments: []) + } + } + + import XCTest + + #if canImport(Darwin) // XCTAttachment is unavailable in swift-corelibs-xctest. + extension TestAttachment { + init(_ attachment: XCTAttachment) { + self.init( + name: attachment.name, + uniformTypeIdentifier: attachment.uniformTypeIdentifier, + payload: attachment.value(forKey: "payload") as? Data + ) + } + } + #endif + + extension TestBundle { + init(_ testBundle: Bundle) { + self.init( + bundleIdentifier: testBundle.bundleIdentifier, + bundlePath: testBundle.bundlePath + ) + } + } + + extension TestCase { + init(_ testCase: XCTestCase) { + self.init(name: testCase.name) + } + } + + extension TestErrorInfo { + init(_ error: any Swift.Error) { + self.init(description: "\\(error)", type: "\\(Swift.type(of: error))") + } + } + + #if canImport(Darwin) // XCTIssue is unavailable in swift-corelibs-xctest. + extension TestIssue { + init(_ issue: XCTIssue) { + self.init( + type: .init(issue.type), + compactDescription: issue.compactDescription, + detailedDescription: issue.detailedDescription, + associatedError: issue.associatedError.map { .init($0) }, + sourceCodeContext: .init(issue.sourceCodeContext), + attachments: issue.attachments.map { .init($0) } + ) + } + } + + extension TestIssueType { + init(_ type: XCTIssue.IssueType) { + switch type { + case .assertionFailure: self = .assertionFailure + case .thrownError: self = .thrownError + case .uncaughtException: self = .uncaughtException + case .performanceRegression: self = .performanceRegression + case .system: self = .system + case .unmatchedExpectedFailure: self = .unmatchedExpectedFailure + @unknown default: self = .unknown + } + } + } + #endif + + #if canImport(Darwin) // XCTSourceCodeLocation/XCTSourceCodeContext/XCTSourceCodeFrame/XCTSourceCodeSymbolInfo is unavailable in swift-corelibs-xctest. + extension TestLocation { + init(_ location: XCTSourceCodeLocation) { + self.init( + file: location.fileURL.absoluteString, + line: location.lineNumber + ) + } + } + + extension TestSourceCodeContext { + init(_ context: XCTSourceCodeContext) { + self.init( + callStack: context.callStack.map { .init($0) }, + location: context.location.map { .init($0) } + ) + } + } + + extension TestSourceCodeFrame { + init(_ frame: XCTSourceCodeFrame) { + self.init( + address: frame.address, + symbolInfo: (try? frame.symbolInfo()).map { .init($0) }, + symbolicationError: frame.symbolicationError.map { .init($0) } + ) + } + } + + extension TestSourceCodeSymbolInfo { + init(_ symbolInfo: XCTSourceCodeSymbolInfo) { + self.init( + imageName: symbolInfo.imageName, + symbolName: symbolInfo.symbolName, + location: symbolInfo.location.map { .init($0) } + ) + } + } + #endif + + extension TestSuiteRecord { + init(_ testSuite: XCTestSuite) { + self.init(name: testSuite.name) + } + } + #endif + """ } diff --git a/Sources/SWBUniversalPlatform/TestEntryPointGenerationTool.swift b/Sources/SWBUniversalPlatform/TestEntryPointGenerationTool.swift index 007611ad..5ee78857 100644 --- a/Sources/SWBUniversalPlatform/TestEntryPointGenerationTool.swift +++ b/Sources/SWBUniversalPlatform/TestEntryPointGenerationTool.swift @@ -17,7 +17,59 @@ import SWBCore final class TestEntryPointGenerationToolSpec: GenericCommandLineToolSpec, SpecIdentifierType, @unchecked Sendable { static let identifier = "org.swift.test-entry-point-generator" + override func commandLineFromTemplate(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, optionContext: (any DiscoveredCommandLineToolSpecInfo)?, specialArgs: [String] = [], lookup: ((MacroDeclaration) -> MacroExpression?)? = nil) -> [CommandLineArgument] { + var args = super.commandLineFromTemplate(cbc, delegate, optionContext: optionContext, specialArgs: specialArgs, lookup: lookup) + for (toolchainPath, toolchainLibrarySearchPath) in cbc.producer.toolchains.map({ ($0.path, $0.librarySearchPaths) }) { + if let path = toolchainLibrarySearchPath.findLibrary(operatingSystem: cbc.producer.hostOperatingSystem, basename: "IndexStore") { + args.append(contentsOf: ["--index-store-library-path", .path(path)]) + } + for input in cbc.inputs { + if input.fileType.conformsTo(identifier: "text") { + args.append(contentsOf: ["--linker-filelist", .path(input.absolutePath)]) + } else if input.fileType.conformsTo(identifier: "compiled.mach-o") { + // Do nothing + } else { + delegate.error("Unexpected input of type '\(input.fileType)' to test entry point generation") + } + } + } + return args + } + override func createTaskAction(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate) -> (any PlannedTaskAction)? { TestEntryPointGenerationTaskAction() } + + public func constructTasks(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, indexStorePaths: [Path], indexUnitBasePaths: [Path]) async { + var commandLine = commandLineFromTemplate(cbc, delegate, optionContext: nil) + + for indexStorePath in indexStorePaths { + commandLine.append(contentsOf: ["--index-store", .path(indexStorePath)]) + } + + for basePath in indexUnitBasePaths { + commandLine.append(contentsOf: ["--index-unit-base-path", .path(basePath)]) + } + + delegate.createTask( + type: self, + dependencyData: nil, + payload: nil, + ruleInfo: defaultRuleInfo(cbc, delegate), + additionalSignatureData: "", + commandLine: commandLine, + additionalOutput: [], + environment: environmentFromSpec(cbc, delegate), + workingDirectory: cbc.producer.defaultWorkingDirectory, + inputs: cbc.inputs.map { delegate.createNode($0.absolutePath) }, + outputs: cbc.outputs.map { delegate.createNode($0) }, + mustPrecede: [], + action: createTaskAction(cbc, delegate), + execDescription: resolveExecutionDescription(cbc, delegate), + preparesForIndexing: true, + enableSandboxing: enableSandboxing, + llbuildControlDisabled: true, + additionalTaskOrderingOptions: [] + ) + } } diff --git a/Sources/SWBUniversalPlatform/TestEntryPointTaskProducer.swift b/Sources/SWBUniversalPlatform/TestEntryPointTaskProducer.swift index fe4b56ef..23ec56f0 100644 --- a/Sources/SWBUniversalPlatform/TestEntryPointTaskProducer.swift +++ b/Sources/SWBUniversalPlatform/TestEntryPointTaskProducer.swift @@ -13,6 +13,7 @@ import SWBCore import SWBTaskConstruction import SWBMacro +import SWBUtil class TestEntryPointTaskProducer: PhasedTaskProducer, TaskProducer { func generateTasks() async -> [any PlannedTask] { @@ -21,8 +22,54 @@ class TestEntryPointTaskProducer: PhasedTaskProducer, TaskProducer { await self.appendGeneratedTasks(&tasks) { delegate in let scope = context.settings.globalScope let outputPath = scope.evaluate(BuiltinMacros.GENERATED_TEST_ENTRY_POINT_PATH) - let cbc = CommandBuildContext(producer: context, scope: scope, inputs: [], outputs: [outputPath]) - await context.testEntryPointGenerationToolSpec.constructTasks(cbc, delegate) + + guard let configuredTarget = context.configuredTarget else { + context.error("Cannot generate a test entry point without a target") + return + } + var indexStoreDirectories: OrderedSet = [] + var linkerFileLists: OrderedSet = [] + var indexUnitBasePaths: OrderedSet = [] + var binaryPaths: OrderedSet = [] + for directDependency in context.globalProductPlan.dependencies(of: configuredTarget) { + let settings = context.globalProductPlan.planRequest.buildRequestContext.getCachedSettings(directDependency.parameters, target: directDependency.target) + guard settings.productType?.conformsTo(identifier: "com.apple.product-type.bundle.unit-test") == true else { + continue + } + guard settings.globalScope.evaluate(BuiltinMacros.SWIFT_INDEX_STORE_ENABLE) else { + context.error("Cannot perform test discovery for '\(directDependency.target.name)' because index while building is disabled") + continue + } + let path = settings.globalScope.evaluate(BuiltinMacros.SWIFT_INDEX_STORE_PATH) + guard !path.isEmpty else { + continue + } + indexStoreDirectories.append(path) + + for arch in settings.globalScope.evaluate(BuiltinMacros.ARCHS) { + for variant in settings.globalScope.evaluate(BuiltinMacros.BUILD_VARIANTS) { + let innerScope = settings.globalScope + .subscope(binding: BuiltinMacros.archCondition, to: arch) + .subscope(binding: BuiltinMacros.variantCondition, to: variant) + let linkerFileListPath = innerScope.evaluate(BuiltinMacros.__INPUT_FILE_LIST_PATH__) + if !linkerFileListPath.isEmpty { + linkerFileLists.append(linkerFileListPath) + } + let objroot = innerScope.evaluate(BuiltinMacros.OBJROOT) + if !objroot.isEmpty { + indexUnitBasePaths.append(objroot) + } + + let binaryPath = innerScope.evaluate(BuiltinMacros.TARGET_BUILD_DIR).join(innerScope.evaluate(BuiltinMacros.EXECUTABLE_PATH)).normalize() + binaryPaths.append(binaryPath) + } + } + } + + let inputs: [FileToBuild] = linkerFileLists.map { FileToBuild(absolutePath: $0, fileType: self.context.workspaceContext.core.specRegistry.getSpec("text") as! FileTypeSpec) } + binaryPaths.map { FileToBuild(absolutePath: $0, fileType: self.context.workspaceContext.core.specRegistry.getSpec("compiled.mach-o") as! FileTypeSpec) } + + let cbc = CommandBuildContext(producer: context, scope: scope, inputs: inputs, outputs: [outputPath]) + await context.testEntryPointGenerationToolSpec.constructTasks(cbc, delegate, indexStorePaths: indexStoreDirectories.elements, indexUnitBasePaths: indexUnitBasePaths.elements) } } return tasks diff --git a/Sources/SWBUtil/CMakeLists.txt b/Sources/SWBUtil/CMakeLists.txt index 91348fd9..9d2d611d 100644 --- a/Sources/SWBUtil/CMakeLists.txt +++ b/Sources/SWBUtil/CMakeLists.txt @@ -46,6 +46,7 @@ add_library(SWBUtil HashContext.swift Headermap.swift HeavyCache.swift + IndexStore.swift Int.swift InterningArena.swift IO.swift diff --git a/Sources/SWBUtil/IndexStore.swift b/Sources/SWBUtil/IndexStore.swift new file mode 100644 index 00000000..46cbd8bc --- /dev/null +++ b/Sources/SWBUtil/IndexStore.swift @@ -0,0 +1,389 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SWBCSupport +import Foundation + +public final class IndexStore { + + public struct TestCaseClass { + public struct TestMethod: Hashable, Comparable { + public let name: String + public let isAsync: Bool + + public static func < (lhs: IndexStore.TestCaseClass.TestMethod, rhs: IndexStore.TestCaseClass.TestMethod) -> Bool { + return (lhs.name, (lhs.isAsync ? 1 : 0)) < (rhs.name, (rhs.isAsync ? 1 : 0)) + } + } + + public var name: String + public var module: String + public var testMethods: [TestMethod] + @available(*, deprecated, message: "use testMethods instead") public var methods: [String] + } + + fileprivate var impl: IndexStoreImpl { _impl as! IndexStoreImpl } + private let _impl: Any + + fileprivate init(_ impl: IndexStoreImpl) { + self._impl = impl + } + + static public func open(store path: Path, api: IndexStoreAPI) throws -> IndexStore { + let impl = try IndexStoreImpl.open(store: path, api: api.impl) + return IndexStore(impl) + } + + public func listTests(in objectFiles: [Path]) throws -> [TestCaseClass] { + return try impl.listTests(in: objectFiles) + } + + @available(*, deprecated, message: "use listTests(in:) instead") + public func listTests(inObjectFile object: Path) throws -> [TestCaseClass] { + return try impl.listTests(inObjectFile: object) + } +} + +public final class IndexStoreAPI { + fileprivate var impl: IndexStoreAPIImpl { + _impl as! IndexStoreAPIImpl + } + private let _impl: Any + + public init(dylib path: Path) throws { + self._impl = try IndexStoreAPIImpl(dylib: path) + } +} + +private final class IndexStoreImpl { + typealias TestCaseClass = IndexStore.TestCaseClass + + let api: IndexStoreAPIImpl + + let store: indexstore_t + + private init(store: indexstore_t, api: IndexStoreAPIImpl) { + self.store = store + self.api = api + } + + static public func open(store path: Path, api: IndexStoreAPIImpl) throws -> IndexStoreImpl { + if let store = try api.call({ api.fn.store_create(path.str, &$0) }) { + return IndexStoreImpl(store: store, api: api) + } + throw StubError.error("Unable to open store at \(path.str)") + } + + public func listTests(in objectFiles: [Path]) throws -> [TestCaseClass] { + var inheritance = [String: [String: String]]() + var testMethods = [String: [String: [(name: String, async: Bool)]]]() + + for objectFile in objectFiles { + // Get the records of this object file. + guard let unitReader = try? self.api.call ({ self.api.fn.unit_reader_create(store, unitName(object: objectFile), &$0) }) else { + continue + } + let records = try getRecords(unitReader: unitReader) + let moduleName = self.api.fn.unit_reader_get_module_name(unitReader).str + for record in records { + // get tests info + let testsInfo = try self.getTestsInfo(record: record) + // merge results across module + for (className, parentClassName) in testsInfo.inheritance { + inheritance[moduleName, default: [:]][className] = parentClassName + } + for (className, classTestMethods) in testsInfo.testMethods { + testMethods[moduleName, default: [:]][className, default: []].append(contentsOf: classTestMethods) + } + } + } + + // merge across inheritance in module boundries + func flatten(moduleName: String, className: String) -> [String: (name: String, async: Bool)] { + var allMethods = [String: (name: String, async: Bool)]() + + if let parentClassName = inheritance[moduleName]?[className] { + let parentMethods = flatten(moduleName: moduleName, className: parentClassName) + allMethods.merge(parentMethods, uniquingKeysWith: { (lhs, _) in lhs }) + } + + for method in testMethods[moduleName]?[className] ?? [] { + allMethods[method.name] = (name: method.name, async: method.async) + } + + return allMethods + } + + var testCaseClasses = [TestCaseClass]() + for (moduleName, classMethods) in testMethods { + for className in classMethods.keys { + let methods = flatten(moduleName: moduleName, className: className) + .map { (name, info) in TestCaseClass.TestMethod(name: name, isAsync: info.async) } + .sorted() + testCaseClasses.append(TestCaseClass(name: className, module: moduleName, testMethods: methods, methods: methods.map(\.name))) + } + } + + return testCaseClasses + } + + + @available(*, deprecated, message: "use listTests(in:) instead") + public func listTests(inObjectFile object: Path) throws -> [TestCaseClass] { + // Get the records of this object file. + let unitReader = try api.call{ self.api.fn.unit_reader_create(store, unitName(object: object), &$0) } + let records = try getRecords(unitReader: unitReader) + + // Get the test classes. + var inheritance = [String: String]() + var testMethods = [String: [(name: String, async: Bool)]]() + + for record in records { + let testsInfo = try self.getTestsInfo(record: record) + inheritance.merge(testsInfo.inheritance, uniquingKeysWith: { (lhs, _) in lhs }) + testMethods.merge(testsInfo.testMethods, uniquingKeysWith: { (lhs, _) in lhs }) + } + + func flatten(className: String) -> [(method: String, async: Bool)] { + var results = [(String, Bool)]() + if let parentClassName = inheritance[className] { + let parentMethods = flatten(className: parentClassName) + results.append(contentsOf: parentMethods) + } + if let methods = testMethods[className] { + results.append(contentsOf: methods) + } + return results + } + + let moduleName = self.api.fn.unit_reader_get_module_name(unitReader).str + + var testCaseClasses = [TestCaseClass]() + for className in testMethods.keys { + let methods = flatten(className: className) + .map { TestCaseClass.TestMethod(name: $0.method, isAsync: $0.async) } + .sorted() + testCaseClasses.append(TestCaseClass(name: className, module: moduleName, testMethods: methods, methods: methods.map(\.name))) + } + + return testCaseClasses + } + + private func getTestsInfo(record: String) throws -> (inheritance: [String: String], testMethods: [String: [(name: String, async: Bool)]] ) { + let recordReader = try api.call{ self.api.fn.record_reader_create(store, record, &$0) } + + // scan for inheritance + + let inheritanceStoreRef = StoreRef([String: String](), api: self.api) + let inheritancePointer = unsafeBitCast(Unmanaged.passUnretained(inheritanceStoreRef), to: UnsafeMutableRawPointer.self) + + _ = self.api.fn.record_reader_occurrences_apply_f(recordReader, inheritancePointer) { inheritancePointer , occ -> Bool in + let inheritanceStoreRef = Unmanaged>.fromOpaque(inheritancePointer!).takeUnretainedValue() + let fn = inheritanceStoreRef.api.fn + + // Get the symbol. + let sym = fn.occurrence_get_symbol(occ) + let symbolProperties = fn.symbol_get_properties(sym) + // We only care about symbols that are marked unit tests and are instance methods. + if symbolProperties & UInt64(INDEXSTORE_SYMBOL_PROPERTY_UNITTEST.rawValue) == 0 { + return true + } + if fn.symbol_get_kind(sym) != INDEXSTORE_SYMBOL_KIND_CLASS{ + return true + } + + let parentClassName = fn.symbol_get_name(sym).str + + let childClassNameStoreRef = StoreRef("", api: inheritanceStoreRef.api) + let childClassNamePointer = unsafeBitCast(Unmanaged.passUnretained(childClassNameStoreRef), to: UnsafeMutableRawPointer.self) + _ = fn.occurrence_relations_apply_f(occ!, childClassNamePointer) { childClassNamePointer, relation in + guard let relation = relation else { return true } + let childClassNameStoreRef = Unmanaged>.fromOpaque(childClassNamePointer!).takeUnretainedValue() + let fn = childClassNameStoreRef.api.fn + + // Look for the base class. + if fn.symbol_relation_get_roles(relation) != UInt64(INDEXSTORE_SYMBOL_ROLE_REL_BASEOF.rawValue) { + return true + } + + let childClassNameSym = fn.symbol_relation_get_symbol(relation) + childClassNameStoreRef.instance = fn.symbol_get_name(childClassNameSym).str + return true + } + + if !childClassNameStoreRef.instance.isEmpty { + inheritanceStoreRef.instance[childClassNameStoreRef.instance] = parentClassName + } + + return true + } + + // scan for methods + + let testMethodsStoreRef = StoreRef([String: [(name: String, async: Bool)]](), api: api) + let testMethodsPointer = unsafeBitCast(Unmanaged.passUnretained(testMethodsStoreRef), to: UnsafeMutableRawPointer.self) + + _ = self.api.fn.record_reader_occurrences_apply_f(recordReader, testMethodsPointer) { testMethodsPointer , occ -> Bool in + let testMethodsStoreRef = Unmanaged>.fromOpaque(testMethodsPointer!).takeUnretainedValue() + let fn = testMethodsStoreRef.api.fn + + // Get the symbol. + let sym = fn.occurrence_get_symbol(occ) + let symbolProperties = fn.symbol_get_properties(sym) + // We only care about symbols that are marked unit tests and are instance methods. + if symbolProperties & UInt64(INDEXSTORE_SYMBOL_PROPERTY_UNITTEST.rawValue) == 0 { + return true + } + if fn.symbol_get_kind(sym) != INDEXSTORE_SYMBOL_KIND_INSTANCEMETHOD { + return true + } + + let classNameStoreRef = StoreRef("", api: testMethodsStoreRef.api) + let classNamePointer = unsafeBitCast(Unmanaged.passUnretained(classNameStoreRef), to: UnsafeMutableRawPointer.self) + + _ = fn.occurrence_relations_apply_f(occ!, classNamePointer) { classNamePointer, relation in + guard let relation = relation else { return true } + let classNameStoreRef = Unmanaged>.fromOpaque(classNamePointer!).takeUnretainedValue() + let fn = classNameStoreRef.api.fn + + // Look for the class. + if fn.symbol_relation_get_roles(relation) != UInt64(INDEXSTORE_SYMBOL_ROLE_REL_CHILDOF.rawValue) { + return true + } + + let classNameSym = fn.symbol_relation_get_symbol(relation) + classNameStoreRef.instance = fn.symbol_get_name(classNameSym).str + return true + } + + if !classNameStoreRef.instance.isEmpty { + let methodName = fn.symbol_get_name(sym).str + let isAsync = symbolProperties & UInt64(INDEXSTORE_SYMBOL_PROPERTY_SWIFT_ASYNC.rawValue) != 0 + testMethodsStoreRef.instance[classNameStoreRef.instance, default: []].append((name: methodName, async: isAsync)) + } + + return true + } + + return ( + inheritance: inheritanceStoreRef.instance, + testMethods: testMethodsStoreRef.instance + ) + + } + + private func getRecords(unitReader: indexstore_unit_reader_t?) throws -> [String] { + let builder = StoreRef([String](), api: api) + + let ctx = unsafeBitCast(Unmanaged.passUnretained(builder), to: UnsafeMutableRawPointer.self) + _ = self.api.fn.unit_reader_dependencies_apply_f(unitReader, ctx) { ctx , unit -> Bool in + let store = Unmanaged>.fromOpaque(ctx!).takeUnretainedValue() + let fn = store.api.fn + if fn.unit_dependency_get_kind(unit) == INDEXSTORE_UNIT_DEPENDENCY_RECORD { + store.instance.append(fn.unit_dependency_get_name(unit).str) + } + return true + } + + return builder.instance + } + + private func unitName(object: Path) -> String { + let initialSize = 64 + var buf = UnsafeMutablePointer.allocate(capacity: initialSize) + let len = self.api.fn.store_get_unit_name_from_output_path(store, object.str, buf, initialSize) + + if len + 1 > initialSize { + buf.deallocate() + buf = UnsafeMutablePointer.allocate(capacity: len + 1) + _ = self.api.fn.store_get_unit_name_from_output_path(store, object.str, buf, len + 1) + } + + defer { + buf.deallocate() + } + + return String(cString: buf) + } +} + +private class StoreRef { + let api: IndexStoreAPIImpl + var instance: T + init(_ instance: T, api: IndexStoreAPIImpl) { + self.instance = instance + self.api = api + } +} + +private final class IndexStoreAPIImpl { + + /// The path of the index store dylib. + private let path: Path + + /// Handle of the dynamic library. + private let dylib: LibraryHandle + + /// The index store API functions. + fileprivate let fn: indexstore_functions_t + + fileprivate func call(_ fn: (inout indexstore_error_t?) -> T) throws -> T { + var error: indexstore_error_t? = nil + let ret = fn(&error) + + if let error = error { + if let desc = self.fn.error_get_description(error) { + throw StubError.error(String(cString: desc)) + } + throw StubError.error("Unable to get description for error: \(error)") + } + + return ret + } + + public init(dylib path: Path) throws { + self.path = path + self.dylib = try Library.open(path) + + var api = indexstore_functions_t() + api.store_create = Library.lookup(dylib, "indexstore_store_create") + api.store_get_unit_name_from_output_path = Library.lookup(dylib, "indexstore_store_get_unit_name_from_output_path") + api.unit_reader_create = Library.lookup(dylib, "indexstore_unit_reader_create") + api.error_get_description = Library.lookup(dylib, "indexstore_error_get_description") + api.unit_reader_dependencies_apply_f = Library.lookup(dylib, "indexstore_unit_reader_dependencies_apply_f") + api.unit_reader_get_module_name = Library.lookup(dylib, "indexstore_unit_reader_get_module_name") + api.unit_dependency_get_kind = Library.lookup(dylib, "indexstore_unit_dependency_get_kind") + api.unit_dependency_get_name = Library.lookup(dylib, "indexstore_unit_dependency_get_name") + api.record_reader_create = Library.lookup(dylib, "indexstore_record_reader_create") + api.symbol_get_name = Library.lookup(dylib, "indexstore_symbol_get_name") + api.symbol_get_properties = Library.lookup(dylib, "indexstore_symbol_get_properties") + api.symbol_get_kind = Library.lookup(dylib, "indexstore_symbol_get_kind") + api.record_reader_occurrences_apply_f = Library.lookup(dylib, "indexstore_record_reader_occurrences_apply_f") + api.occurrence_get_symbol = Library.lookup(dylib, "indexstore_occurrence_get_symbol") + api.occurrence_relations_apply_f = Library.lookup(dylib, "indexstore_occurrence_relations_apply_f") + api.symbol_relation_get_symbol = Library.lookup(dylib, "indexstore_symbol_relation_get_symbol") + api.symbol_relation_get_roles = Library.lookup(dylib, "indexstore_symbol_relation_get_roles") + + self.fn = api + } +} + +extension indexstore_string_ref_t { + fileprivate var str: String { + return String( + bytesNoCopy: UnsafeMutableRawPointer(mutating: data), + length: length, + encoding: .utf8, + freeWhenDone: false + )! + } +} diff --git a/Sources/SWBWindowsPlatform/Specs/Windows.xcspec b/Sources/SWBWindowsPlatform/Specs/Windows.xcspec index df09990f..9c58f4f4 100644 --- a/Sources/SWBWindowsPlatform/Specs/Windows.xcspec +++ b/Sources/SWBWindowsPlatform/Specs/Windows.xcspec @@ -42,6 +42,31 @@ BasedOn = com.apple.product-type.tool; }, + { + Domain = windows; + Type = ProductType; + Identifier = com.apple.product-type.bundle.unit-test; + BasedOn = com.apple.product-type.library.dynamic; + DefaultBuildProperties = { + // Index store data is required to discover XCTest tests + COMPILER_INDEX_STORE_ENABLE = YES; + SWIFT_INDEX_STORE_ENABLE = YES; + // Testability is needed to generate code to invoke discovered XCTest tests + SWIFT_ENABLE_TESTABILITY = YES; + }; + }, + + { + Domain = windows; + Type = ProductType; + Identifier = com.apple.product-type.tool.swiftpm-test-runner; + BasedOn = default:com.apple.product-type.tool.swiftpm-test-runner; + DefaultBuildProperties = { + EXECUTABLE_SUFFIX = ".$(EXECUTABLE_EXTENSION)"; + EXECUTABLE_EXTENSION = "exe"; + }; + }, + { Domain = windows; Type = ProductType; diff --git a/Sources/SwiftBuild/ProjectModel/BuildSettings.swift b/Sources/SwiftBuild/ProjectModel/BuildSettings.swift index a8909c03..ae10d9de 100644 --- a/Sources/SwiftBuild/ProjectModel/BuildSettings.swift +++ b/Sources/SwiftBuild/ProjectModel/BuildSettings.swift @@ -99,6 +99,7 @@ extension ProjectModel { case SUPPORTS_TEXT_BASED_API case SUPPRESS_WARNINGS case SWIFT_ENABLE_BARE_SLASH_REGEX + case SWIFT_INDEX_STORE_ENABLE case SWIFT_INSTALL_MODULE case SWIFT_PACKAGE_NAME case SWIFT_USER_MODULE_VERSION @@ -146,6 +147,7 @@ extension ProjectModel { case SPECIALIZATION_SDK_OPTIONS case SWIFT_VERSION case SWIFT_ACTIVE_COMPILATION_CONDITIONS + case DYLIB_INSTALL_NAME_BASE } public enum Platform: Hashable, CaseIterable, Sendable { diff --git a/Sources/SwiftBuild/ProjectModel/Targets.swift b/Sources/SwiftBuild/ProjectModel/Targets.swift index 0f8a201a..efd4749c 100644 --- a/Sources/SwiftBuild/ProjectModel/Targets.swift +++ b/Sources/SwiftBuild/ProjectModel/Targets.swift @@ -316,6 +316,7 @@ extension ProjectModel { case executable = "com.apple.product-type.tool" case hostBuildTool = "com.apple.product-type.tool.host-build" case unitTest = "com.apple.product-type.bundle.unit-test" + case swiftpmTestRunner = "com.apple.product-type.tool.swiftpm-test-runner" case bundle = "com.apple.product-type.bundle" case packageProduct = "packageProduct" } diff --git a/Tests/SWBBuildSystemTests/BuildOperationTests.swift b/Tests/SWBBuildSystemTests/BuildOperationTests.swift index e6c0caaf..f3a0c446 100644 --- a/Tests/SWBBuildSystemTests/BuildOperationTests.swift +++ b/Tests/SWBBuildSystemTests/BuildOperationTests.swift @@ -399,7 +399,7 @@ fileprivate struct BuildOperationTests: CoreBasedTests { @Test(.requireSDKs(.host), .skipHostOS(.macOS), .skipHostOS(.windows, "cannot find testing library")) func unitTestWithGeneratedEntryPoint() async throws { - try await withTemporaryDirectory { (tmpDir: Path) in + try await withTemporaryDirectory(removeTreeOnDeinit: false) { (tmpDir: Path) in let testProject = try await TestProject( "TestProject", sourceRoot: tmpDir, @@ -417,14 +417,32 @@ fileprivate struct BuildOperationTests: CoreBasedTests { "SDKROOT": "$(HOST_PLATFORM)", "SUPPORTED_PLATFORMS": "$(HOST_PLATFORM)", "SWIFT_VERSION": swiftVersion, + "INDEX_DATA_STORE_DIR": "\(tmpDir.join("index").str)", + "LINKER_DRIVER": "swiftc" ]) ], targets: [ TestStandardTarget( - "test", + "UnitTestRunner", + type: .swiftpmTestRunner, + buildConfigurations: [ + TestBuildConfiguration("Debug", + buildSettings: [:]), + ], + buildPhases: [ + TestSourcesBuildPhase(), + TestFrameworksBuildPhase([ + "MyTests.so" + ]) + ], + dependencies: ["MyTests"] + ), + TestStandardTarget( + "MyTests", type: .unitTest, buildConfigurations: [ TestBuildConfiguration("Debug", buildSettings: [ + "DYLIB_INSTALL_NAME_BASE": "$ORIGIN", "LD_RUNPATH_SEARCH_PATHS": "@loader_path/", ]) ], @@ -433,10 +451,10 @@ fileprivate struct BuildOperationTests: CoreBasedTests { TestFrameworksBuildPhase([ TestBuildFile(.target("library")), ]) - ], - dependencies: [ + ], dependencies: [ "library" - ] + ], + productReferenceName: "MyTests.so" ), TestStandardTarget( "library", @@ -444,6 +462,7 @@ fileprivate struct BuildOperationTests: CoreBasedTests { buildConfigurations: [ TestBuildConfiguration("Debug", buildSettings: [ "DYLIB_INSTALL_NAME_BASE": "$ORIGIN", + "LD_RUNPATH_SEARCH_PATHS": "@loader_path/", // FIXME: Find a way to make these default "EXECUTABLE_PREFIX": "lib", @@ -457,7 +476,7 @@ fileprivate struct BuildOperationTests: CoreBasedTests { ]) let core = try await getCore() let tester = try await BuildOperationTester(core, testProject, simulated: false) - + try localFS.createDirectory(tmpDir.join("index")) let projectDir = tester.workspace.projects[0].sourceRoot try await tester.fs.writeFileContents(projectDir.join("library.swift")) { stream in @@ -467,12 +486,19 @@ fileprivate struct BuildOperationTests: CoreBasedTests { try await tester.fs.writeFileContents(projectDir.join("test.swift")) { stream in stream <<< """ import Testing + import XCTest import library @Suite struct MySuite { - @Test func myTest() async throws { + @Test func myTest() { #expect(foo() == 42) } } + + final class MYXCTests: XCTestCase { + func testFoo() { + XCTAssertTrue(true) + } + } """ } @@ -483,13 +509,19 @@ fileprivate struct BuildOperationTests: CoreBasedTests { let toolchain = try #require(try await getCore().toolchainRegistry.defaultToolchain) let environment: Environment if destination.platform == "linux" { - environment = ["LD_LIBRARY_PATH": toolchain.path.join("usr/lib/swift/linux").str] + environment = ["LD_LIBRARY_PATH": "\(toolchain.path.join("usr/lib/swift/linux").str):\(projectDir.join("build").join("Debug\(destination.builtProductsDirSuffix)"))"] } else { environment = .init() } - let executionResult = try await Process.getOutput(url: URL(fileURLWithPath: projectDir.join("build").join("Debug\(destination.builtProductsDirSuffix)").join(core.hostOperatingSystem.imageFormat.executableName(basename: "test.xctest")).str), arguments: ["--testing-library", "swift-testing"], environment: environment) - #expect(String(decoding: executionResult.stderr, as: UTF8.self).contains("Test run started")) + do { + let executionResult = try await Process.getOutput(url: URL(fileURLWithPath: projectDir.join("build").join("Debug\(destination.builtProductsDirSuffix)").join(core.hostOperatingSystem.imageFormat.executableName(basename: "UnitTestRunner")).str), arguments: [], environment: environment) + #expect(String(decoding: executionResult.stdout, as: UTF8.self).contains("Executed 1 test, with 0 failures")) + } + do { + let executionResult = try await Process.getOutput(url: URL(fileURLWithPath: projectDir.join("build").join("Debug\(destination.builtProductsDirSuffix)").join(core.hostOperatingSystem.imageFormat.executableName(basename: "UnitTestRunner")).str), arguments: ["--testing-library", "swift-testing"], environment: environment) + #expect(String(decoding: executionResult.stderr, as: UTF8.self).contains("Test run with 1 test in 1 suite passed")) + } } } } diff --git a/Tests/SWBTaskConstructionTests/UnitTestTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/UnitTestTaskConstructionTests.swift index d40bc363..ce4515df 100644 --- a/Tests/SWBTaskConstructionTests/UnitTestTaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/UnitTestTaskConstructionTests.swift @@ -300,7 +300,7 @@ fileprivate struct UnitTestTaskConstructionTests: CoreBasedTests { } @Test(.requireSDKs(.linux)) - func unitTestTarget_linux() async throws { + func unitTestRunnerTarget_linux() async throws { let swiftCompilerPath = try await self.swiftCompilerPath let swiftVersion = try await self.swiftVersion let testProject = TestProject( @@ -319,9 +319,26 @@ fileprivate struct UnitTestTaskConstructionTests: CoreBasedTests { "PRODUCT_NAME": "$(TARGET_NAME)", "SDKROOT": "linux", "SWIFT_VERSION": swiftVersion, + "INDEX_DATA_STORE_DIR": "/index", + "LINKER_DRIVER": "swiftc" ]), ], targets: [ + TestStandardTarget( + "UnitTestRunner", + type: .swiftpmTestRunner, + buildConfigurations: [ + TestBuildConfiguration("Debug", + buildSettings: [:]), + ], + buildPhases: [ + TestSourcesBuildPhase(), + TestFrameworksBuildPhase([ + "UnitTestTarget.so" + ]) + ], + dependencies: ["UnitTestTarget"], + ), TestStandardTarget( "UnitTestTarget", type: .unitTest, @@ -335,7 +352,8 @@ fileprivate struct UnitTestTaskConstructionTests: CoreBasedTests { "TestTwo.swift", ]), ], - dependencies: [] + dependencies: [], + productReferenceName: "UnitTestTarget.so" ), ]) let core = try await getCore() @@ -346,14 +364,17 @@ fileprivate struct UnitTestTaskConstructionTests: CoreBasedTests { try await fs.writeFileContents(swiftCompilerPath) { $0 <<< "binary" } await tester.checkBuild(runDestination: .linux, fs: fs) { results in - results.checkTarget("UnitTestTarget") { target in + results.checkTarget("UnitTestRunner") { target in results.checkTask(.matchTarget(target), .matchRuleType("GenerateTestEntryPoint")) { task in - task.checkCommandLineMatches([.suffix("builtin-generateTestEntryPoint"), "--output", .suffix("test_entry_point.swift")]) + task.checkCommandLineMatches([.suffix("builtin-generateTestEntryPoint"), "--output", .suffix("test_entry_point.swift"), "--index-store-library-path", .suffix("libIndexStore.so"), "--linker-filelist", .suffix("UnitTestTarget.LinkFileList"), "--index-store", "/index", "--index-unit-base-path", "/tmp/Test/aProject/build"]) + task.checkInputs([ + .pathPattern(.suffix("UnitTestTarget.LinkFileList")), + .pathPattern(.suffix("UnitTestTarget.so")), + .namePattern(.any), + .namePattern(.any) + ]) task.checkOutputs([.pathPattern(.suffix("test_entry_point.swift"))]) } - results.checkTask(.matchTarget(target), .matchRuleType("SwiftDriver Compilation")) { task in - task.checkInputs(contain: [.pathPattern(.suffix("test_entry_point.swift"))]) - } } results.checkNoDiagnostics() From 86a22ecac10b95256c01b27d3cf32ec5ab8a5bff Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Wed, 4 Jun 2025 13:28:49 -0700 Subject: [PATCH 31/54] Fix issue due to duplicated indexstore bindings in Swift Build and TSC --- Sources/SWBCSupport/IndexStore.h | 6 +----- Sources/SWBUtil/Cache.swift | 12 ++++++++++++ Sources/SWBUtil/IndexStore.swift | 4 ++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Sources/SWBCSupport/IndexStore.h b/Sources/SWBCSupport/IndexStore.h index 7d4b77b8..c1546932 100644 --- a/Sources/SWBCSupport/IndexStore.h +++ b/Sources/SWBCSupport/IndexStore.h @@ -181,14 +181,10 @@ typedef struct { indexstore_string_ref_t (*unit_dependency_get_name)(indexstore_unit_dependency_t); - bool - (*unit_reader_dependencies_apply)(indexstore_unit_reader_t, - bool(^applier)(indexstore_unit_dependency_t)); - bool (*unit_reader_dependencies_apply_f)(indexstore_unit_reader_t, void *context, bool(*applier)(void *context, indexstore_unit_dependency_t)); -} indexstore_functions_t; +} swiftbuild_indexstore_functions_t; #endif diff --git a/Sources/SWBUtil/Cache.swift b/Sources/SWBUtil/Cache.swift index ad5b932d..eaa9e4e0 100644 --- a/Sources/SWBUtil/Cache.swift +++ b/Sources/SWBUtil/Cache.swift @@ -87,6 +87,13 @@ public final class Cache: NSObject, KeyValueStorage, NSCac return nil } set { + #if os(Linux) + if let newValue = newValue { + cache.value.setObject(ValueWrapper(newValue), forKey: KeyWrapper(key)) + } else { + cache.value.removeObject(forKey: KeyWrapper(key)) + } + #else if let newValue, let cacheableValue = newValue as? (any CacheableValue) { cache.value.setObject(ValueWrapper(newValue), forKey: KeyWrapper(key), cost: cacheableValue.cost) } else if let newValue = newValue { @@ -94,6 +101,7 @@ public final class Cache: NSObject, KeyValueStorage, NSCac } else { cache.value.removeObject(forKey: KeyWrapper(key)) } + #endif } } @@ -112,12 +120,16 @@ public final class Cache: NSObject, KeyValueStorage, NSCac } let value = try body() + #if os(Linux) + cache.value.setObject(ValueWrapper(value), forKey: wrappedKey) + #else if let cacheableValue = value as? (any CacheableValue) { cache.value.setObject(ValueWrapper(value), forKey: wrappedKey, cost: cacheableValue.cost) } else { cache.value.setObject(ValueWrapper(value), forKey: wrappedKey) } + #endif return value } diff --git a/Sources/SWBUtil/IndexStore.swift b/Sources/SWBUtil/IndexStore.swift index 46cbd8bc..8b4515fc 100644 --- a/Sources/SWBUtil/IndexStore.swift +++ b/Sources/SWBUtil/IndexStore.swift @@ -334,7 +334,7 @@ private final class IndexStoreAPIImpl { private let dylib: LibraryHandle /// The index store API functions. - fileprivate let fn: indexstore_functions_t + fileprivate let fn: swiftbuild_indexstore_functions_t fileprivate func call(_ fn: (inout indexstore_error_t?) -> T) throws -> T { var error: indexstore_error_t? = nil @@ -354,7 +354,7 @@ private final class IndexStoreAPIImpl { self.path = path self.dylib = try Library.open(path) - var api = indexstore_functions_t() + var api = swiftbuild_indexstore_functions_t() api.store_create = Library.lookup(dylib, "indexstore_store_create") api.store_get_unit_name_from_output_path = Library.lookup(dylib, "indexstore_store_get_unit_name_from_output_path") api.unit_reader_create = Library.lookup(dylib, "indexstore_unit_reader_create") From 380f251d50e61927be1325a5b2812c273e4b7262 Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Sat, 7 Jun 2025 12:01:18 -0700 Subject: [PATCH 32/54] Add settings to control frame pointer omission --- .../SWBUniversalPlatform/Specs/Clang.xcspec | 19 +++++ .../SWBUniversalPlatform/Specs/Swift.xcspec | 19 +++++ .../TaskConstructionTests.swift | 74 +++++++++++++++++++ 3 files changed, 112 insertions(+) diff --git a/Sources/SWBUniversalPlatform/Specs/Clang.xcspec b/Sources/SWBUniversalPlatform/Specs/Clang.xcspec index f7ce248e..484a1d3c 100644 --- a/Sources/SWBUniversalPlatform/Specs/Clang.xcspec +++ b/Sources/SWBUniversalPlatform/Specs/Clang.xcspec @@ -2934,6 +2934,25 @@ NO = (); }; }, + { + Name = "CLANG_OMIT_FRAME_POINTERS"; + Type = Enumeration; + Values = ( + "compiler-default", + YES, + NO, + ); + CommandLineArgs = { + YES = ( + "-fomit-frame-pointer", + ); + NO = ( + "-fno-omit-frame-pointer", + ); + "<>" = (); + }; + DefaultValue = "compiler-default"; + }, // Index-while-building options, not visible in build settings. { Name = "CLANG_INDEX_STORE_PATH"; diff --git a/Sources/SWBUniversalPlatform/Specs/Swift.xcspec b/Sources/SWBUniversalPlatform/Specs/Swift.xcspec index cccdc078..ff751506 100644 --- a/Sources/SWBUniversalPlatform/Specs/Swift.xcspec +++ b/Sources/SWBUniversalPlatform/Specs/Swift.xcspec @@ -995,6 +995,25 @@ DefaultValue = "$(DEBUG_INFORMATION_VERSION)"; Condition = "$(GCC_GENERATE_DEBUGGING_SYMBOLS) && $(DEBUG_INFORMATION_FORMAT) != \"\""; }, + { + Name = "SWIFT_OMIT_FRAME_POINTERS"; + Type = Enumeration; + Values = ( + "compiler-default", + YES, + NO, + ); + CommandLineArgs = { + YES = ( + "-Xcc", "-fomit-frame-pointer", + ); + NO = ( + "-Xcc", "-fno-omit-frame-pointer", + ); + "<>" = (); + }; + DefaultValue = "compiler-default"; + }, { Name = "CLANG_MODULE_CACHE_PATH"; Type = Path; diff --git a/Tests/SWBTaskConstructionTests/TaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/TaskConstructionTests.swift index a3931342..a0edbbe6 100644 --- a/Tests/SWBTaskConstructionTests/TaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/TaskConstructionTests.swift @@ -8583,6 +8583,80 @@ fileprivate struct TaskConstructionTests: CoreBasedTests { } } + @Test(.requireSDKs(.host)) + func framePointerControl() async throws { + try await withTemporaryDirectory { tmpDir in + let testProject = try await TestProject( + "aProject", + sourceRoot: tmpDir, + groupTree: TestGroup( + "SomeFiles", path: "Sources", + children: [ + TestFile("SourceFile.c"), + TestFile("Source.swift"), + ]), + buildConfigurations: [ + TestBuildConfiguration("Debug", buildSettings: [ + "SWIFT_EXEC": swiftCompilerPath.str, + "SWIFT_VERSION": swiftVersion, + ]) + ], + targets: [ + TestStandardTarget( + "Library", + type: .dynamicLibrary, + buildConfigurations: [ + TestBuildConfiguration("Debug") + ], + buildPhases: [ + TestSourcesBuildPhase([ + "SourceFile.c", + "Source.swift" + ]) + ] + )] + ) + + let fs = PseudoFS() + + let core = try await getCore() + let tester = try TaskConstructionTester(core, testProject) + + await tester.checkBuild(BuildParameters(configuration: "Debug", overrides: [:]), runDestination: .host, fs: fs) { results in + results.checkTask(.matchRuleType("CompileC")) { task in + task.checkCommandLineDoesNotContain("-fomit-frame-pointer") + task.checkCommandLineDoesNotContain("-fno-omit-frame-pointer") + } + results.checkTask(.matchRuleType("SwiftDriver Compilation")) { task in + task.checkCommandLineDoesNotContain("-fomit-frame-pointer") + task.checkCommandLineDoesNotContain("-fno-omit-frame-pointer") + } + } + + await tester.checkBuild(BuildParameters(configuration: "Debug", overrides: ["CLANG_OMIT_FRAME_POINTERS": "YES", "SWIFT_OMIT_FRAME_POINTERS": "YES"]), runDestination: .host, fs: fs) { results in + results.checkTask(.matchRuleType("CompileC")) { task in + task.checkCommandLineContains(["-fomit-frame-pointer"]) + task.checkCommandLineDoesNotContain("-fno-omit-frame-pointer") + } + results.checkTask(.matchRuleType("SwiftDriver Compilation")) { task in + task.checkCommandLineContains(["-Xcc", "-fomit-frame-pointer"]) + task.checkCommandLineDoesNotContain("-fno-omit-frame-pointer") + } + } + + await tester.checkBuild(BuildParameters(configuration: "Debug", overrides: ["CLANG_OMIT_FRAME_POINTERS": "NO", "SWIFT_OMIT_FRAME_POINTERS": "NO"]), runDestination: .host, fs: fs) { results in + results.checkTask(.matchRuleType("CompileC")) { task in + task.checkCommandLineDoesNotContain("-fomit-frame-pointer") + task.checkCommandLineContains(["-fno-omit-frame-pointer"]) + } + results.checkTask(.matchRuleType("SwiftDriver Compilation")) { task in + task.checkCommandLineDoesNotContain("-fomit-frame-pointer") + task.checkCommandLineContains(["-Xcc", "-fno-omit-frame-pointer"]) + } + } + } + } + @Test(.requireSDKs(.macOS)) func warningSuppression() async throws { try await withTemporaryDirectory { tmpDir in From 24b64352141c21f6a3e814e4762ed4b50255a208 Mon Sep 17 00:00:00 2001 From: Jake Petroules Date: Sun, 8 Jun 2025 18:46:23 -0700 Subject: [PATCH 33/54] Simplify implementation of Dispatch-async bridge Stress testing this approach over use of a long-lived DispatchIO channel appears to resolve the nondeterministic failures with the file descriptor being destroyed, and is easier to reason about. Closes #21 --- Sources/SWBUtil/Dispatch+Async.swift | 112 +++--------------- Sources/SWBUtil/FileHandle+Async.swift | 8 +- Sources/SWBUtil/Misc+Async.swift | 12 ++ Sources/SWBUtil/Process+Async.swift | 8 +- Sources/SWBUtil/Process.swift | 8 +- Sources/SWBUtil/SWBDispatch.swift | 27 ----- Tests/SWBUtilTests/FileHandleTests.swift | 32 ++--- .../ConsoleCommands/CLIConnection.swift | 14 ++- .../ConsoleCommands/ServiceConsoleTests.swift | 2 +- 9 files changed, 70 insertions(+), 153 deletions(-) diff --git a/Sources/SWBUtil/Dispatch+Async.swift b/Sources/SWBUtil/Dispatch+Async.swift index bf70a86d..fa81ddef 100644 --- a/Sources/SWBUtil/Dispatch+Async.swift +++ b/Sources/SWBUtil/Dispatch+Async.swift @@ -57,114 +57,38 @@ extension DispatchFD { } } } -} -extension AsyncThrowingStream where Element == UInt8, Failure == any Error { /// Returns an async stream which reads bytes from the specified file descriptor. Unlike `FileHandle.bytes`, it does not block the caller. @available(macOS, deprecated: 15.0, message: "Use the AsyncSequence-returning overload.") @available(iOS, deprecated: 18.0, message: "Use the AsyncSequence-returning overload.") @available(tvOS, deprecated: 18.0, message: "Use the AsyncSequence-returning overload.") @available(watchOS, deprecated: 11.0, message: "Use the AsyncSequence-returning overload.") @available(visionOS, deprecated: 2.0, message: "Use the AsyncSequence-returning overload.") - public static func _dataStream(reading fileDescriptor: DispatchFD, on queue: SWBQueue) -> AsyncThrowingStream { - AsyncThrowingStream { continuation in - let newFD: DispatchFD - do { - newFD = try fileDescriptor._duplicate() - } catch { - continuation.finish(throwing: error) - return - } - - let io = SWBDispatchIO.stream(fileDescriptor: newFD, queue: queue) { error in - do { - try newFD._close() - if error != 0 { - continuation.finish(throwing: POSIXError(error, context: "dataStream(reading: \(fileDescriptor))#1")) - } - } catch { - continuation.finish(throwing: error) - } - } - io.setLimit(lowWater: 0) - io.setLimit(highWater: 4096) - - continuation.onTermination = { termination in - if case .cancelled = termination { - io.close(flags: .stop) - } else { - io.close() - } - } - - io.read(offset: 0, length: .max, queue: queue) { done, data, error in - guard error == 0 else { - continuation.finish(throwing: POSIXError(error, context: "dataStream(reading: \(fileDescriptor))#2")) - return - } - - let data = data ?? .empty - for element in data { - continuation.yield(element) - } - - if done { - continuation.finish() + public func _dataStream() -> AsyncThrowingStream { + AsyncThrowingStream { + while !Task.isCancelled { + let chunk = try await readChunk(upToLength: 4096) + if chunk.isEmpty { + return nil } + return chunk } + throw CancellationError() } } -} -@available(macOS 15.0, iOS 18.0, tvOS 18.0, watchOS 11.0, visionOS 2.0, *) -extension AsyncSequence where Element == UInt8, Failure == any Error { /// Returns an async stream which reads bytes from the specified file descriptor. Unlike `FileHandle.bytes`, it does not block the caller. - public static func dataStream(reading fileDescriptor: DispatchFD, on queue: SWBQueue) -> any AsyncSequence { - AsyncThrowingStream { continuation in - let newFD: DispatchFD - do { - newFD = try fileDescriptor._duplicate() - } catch { - continuation.finish(throwing: error) - return - } - - let io = SWBDispatchIO.stream(fileDescriptor: newFD, queue: queue) { error in - do { - try newFD._close() - if error != 0 { - let context = "dataStream(reading: \(fileDescriptor) \"\(Result { try fileDescriptor._filePath() })\")#1" - continuation.finish(throwing: POSIXError(error, context: context)) - } - } catch { - continuation.finish(throwing: error) - } - } - io.setLimit(lowWater: 0) - io.setLimit(highWater: 4096) - - continuation.onTermination = { termination in - if case .cancelled = termination { - io.close(flags: .stop) - } else { - io.close() - } - } - - io.read(offset: 0, length: .max, queue: queue) { done, data, error in - guard error == 0 else { - let context = "dataStream(reading: \(fileDescriptor) \"\(Result { try fileDescriptor._filePath() })\")#2" - continuation.finish(throwing: POSIXError(error, context: context)) - return - } - - let data = data ?? .empty - continuation.yield(data) - - if done { - continuation.finish() + @available(macOS 15.0, iOS 18.0, tvOS 18.0, watchOS 11.0, visionOS 2.0, *) + public func dataStream() -> some AsyncSequence { + AsyncThrowingStream { + while !Task.isCancelled { + let chunk = try await readChunk(upToLength: 4096) + if chunk.isEmpty { + return nil } + return chunk } - }.flattened + throw CancellationError() + } } } diff --git a/Sources/SWBUtil/FileHandle+Async.swift b/Sources/SWBUtil/FileHandle+Async.swift index 01746218..2f0874e9 100644 --- a/Sources/SWBUtil/FileHandle+Async.swift +++ b/Sources/SWBUtil/FileHandle+Async.swift @@ -19,13 +19,13 @@ extension FileHandle { @available(tvOS, deprecated: 18.0, message: "Use the AsyncSequence-returning overload.") @available(watchOS, deprecated: 11.0, message: "Use the AsyncSequence-returning overload.") @available(visionOS, deprecated: 2.0, message: "Use the AsyncSequence-returning overload.") - public func _bytes(on queue: SWBQueue) -> AsyncThrowingStream { - ._dataStream(reading: DispatchFD(fileHandle: self), on: queue) + public func _bytes() -> AsyncThrowingStream { + DispatchFD(fileHandle: self)._dataStream() } /// Replacement for `bytes` which uses DispatchIO to avoid blocking the caller. @available(macOS 15.0, iOS 18.0, tvOS 18.0, watchOS 11.0, visionOS 2.0, *) - public func bytes(on queue: SWBQueue) -> any AsyncSequence { - AsyncThrowingStream.dataStream(reading: DispatchFD(fileHandle: self), on: queue) + public func bytes() -> some AsyncSequence { + DispatchFD(fileHandle: self).dataStream() } } diff --git a/Sources/SWBUtil/Misc+Async.swift b/Sources/SWBUtil/Misc+Async.swift index 22818c12..ff01e3cf 100644 --- a/Sources/SWBUtil/Misc+Async.swift +++ b/Sources/SWBUtil/Misc+Async.swift @@ -23,6 +23,18 @@ extension AsyncSequence { } } +extension AsyncSequence where Element: RandomAccessCollection { + @inlinable + public func collect() async rethrows -> [Element.Element] { + var items = [Element.Element]() + var it = makeAsyncIterator() + while let e = try await it.next() { + items.append(contentsOf: e) + } + return items + } +} + extension TaskGroup where Element == Void { /// Concurrency-friendly replacement for `DispatchQueue.concurrentPerform(iterations:execute:)`. public static func concurrentPerform(iterations: Int, maximumParallelism: Int, execute work: @Sendable @escaping (Int) async -> Element) async { diff --git a/Sources/SWBUtil/Process+Async.swift b/Sources/SWBUtil/Process+Async.swift index aa22ecc1..93cecc3e 100644 --- a/Sources/SWBUtil/Process+Async.swift +++ b/Sources/SWBUtil/Process+Async.swift @@ -35,20 +35,20 @@ extension Process { @available(tvOS, deprecated: 18.0, message: "Use the AsyncSequence-returning overload.") @available(watchOS, deprecated: 11.0, message: "Use the AsyncSequence-returning overload.") @available(visionOS, deprecated: 2.0, message: "Use the AsyncSequence-returning overload.") - public func _makeStream(for keyPath: ReferenceWritableKeyPath, using pipe: Pipe) -> AsyncThrowingStream { + public func _makeStream(for keyPath: ReferenceWritableKeyPath, using pipe: Pipe) -> AsyncThrowingStream { precondition(!isRunning) // the pipe setters will raise `NSInvalidArgumentException` anyways self[keyPath: keyPath] = pipe - return pipe.fileHandleForReading._bytes(on: .global()) + return pipe.fileHandleForReading._bytes() } /// Returns an ``AsyncStream`` configured to read the standard output or error stream of the process. /// /// - note: This method will mutate the `standardOutput` or `standardError` property of the Process object, replacing any existing `Pipe` or `FileHandle` which may be set. It must be called before the process is started. @available(macOS 15.0, iOS 18.0, tvOS 18.0, watchOS 11.0, visionOS 2.0, *) - public func makeStream(for keyPath: ReferenceWritableKeyPath, using pipe: Pipe) -> any AsyncSequence { + public func makeStream(for keyPath: ReferenceWritableKeyPath, using pipe: Pipe) -> some AsyncSequence { precondition(!isRunning) // the pipe setters will raise `NSInvalidArgumentException` anyways self[keyPath: keyPath] = pipe - return pipe.fileHandleForReading.bytes(on: .global()) + return pipe.fileHandleForReading.bytes() } } diff --git a/Sources/SWBUtil/Process.swift b/Sources/SWBUtil/Process.swift index d832eb5a..12434bf2 100644 --- a/Sources/SWBUtil/Process.swift +++ b/Sources/SWBUtil/Process.swift @@ -119,7 +119,7 @@ extension Process { let (exitStatus, output) = try await _getOutput(url: url, arguments: arguments, currentDirectoryURL: currentDirectoryURL, environment: environment, interruptible: interruptible) { process in process.standardOutputPipe = pipe process.standardErrorPipe = pipe - return pipe.fileHandleForReading.bytes(on: .global()) + return pipe.fileHandleForReading.bytes() } collect: { stream in try await stream.collect() } @@ -131,7 +131,7 @@ extension Process { let (exitStatus, output) = try await _getOutput(url: url, arguments: arguments, currentDirectoryURL: currentDirectoryURL, environment: environment, interruptible: interruptible) { process in process.standardOutputPipe = pipe process.standardErrorPipe = pipe - return pipe.fileHandleForReading._bytes(on: .global()) + return pipe.fileHandleForReading._bytes() } collect: { stream in try await stream.collect() } @@ -161,9 +161,11 @@ extension Process { let streams = setup(process) + async let outputTask = await collect(streams) + try await process.run(interruptible: interruptible) - let output = try await collect(streams) + let output = try await outputTask #if !canImport(Darwin) // Clear the pipes to prevent file descriptor leaks on platforms using swift-corelibs-foundation diff --git a/Sources/SWBUtil/SWBDispatch.swift b/Sources/SWBUtil/SWBDispatch.swift index 26aad751..0aa561b2 100644 --- a/Sources/SWBUtil/SWBDispatch.swift +++ b/Sources/SWBUtil/SWBDispatch.swift @@ -49,33 +49,6 @@ public struct DispatchFD { rawValue = fileHandle.fileDescriptor #endif } - - internal func _duplicate() throws -> DispatchFD { - #if os(Windows) - return self - #else - return try DispatchFD(fileDescriptor: FileDescriptor(rawValue: rawValue).duplicate()) - #endif - } - - internal func _close() throws { - #if !os(Windows) - try FileDescriptor(rawValue: rawValue).close() - #endif - } - - // Only exists to help debug a rare concurrency issue where the file descriptor goes invalid - internal func _filePath() throws -> String { - #if canImport(Darwin) - var buffer = [CChar](repeating: 0, count: Int(MAXPATHLEN)) - if fcntl(rawValue, F_GETPATH, &buffer) == -1 { - throw POSIXError(errno, "fcntl", String(rawValue), "F_GETPATH") - } - return String(cString: buffer) - #else - return String() - #endif - } } // @unchecked: rdar://130051790 (DispatchData should be Sendable) diff --git a/Tests/SWBUtilTests/FileHandleTests.swift b/Tests/SWBUtilTests/FileHandleTests.swift index 51b504b3..6a837b96 100644 --- a/Tests/SWBUtilTests/FileHandleTests.swift +++ b/Tests/SWBUtilTests/FileHandleTests.swift @@ -38,21 +38,21 @@ import SystemPackage let fh = FileHandle(fileDescriptor: fd.rawValue, closeOnDealloc: false) try await fd.closeAfter { if #available(macOS 15, iOS 18, tvOS 18, watchOS 11, visionOS 2, *) { - var it = fh.bytes(on: .global()).makeAsyncIterator() + var it = fh.bytes().makeAsyncIterator() var bytesOfFile: [UInt8] = [] await #expect(throws: Never.self) { - while let byte = try await it.next() { - bytesOfFile.append(byte) + while let chunk = try await it.next() { + bytesOfFile.append(contentsOf: chunk) } } #expect(bytesOfFile.count == 1448) #expect(plist.bytes == bytesOfFile) } else { - var it = fh._bytes(on: .global()).makeAsyncIterator() + var it = fh._bytes().makeAsyncIterator() var bytesOfFile: [UInt8] = [] await #expect(throws: Never.self) { - while let byte = try await it.next() { - bytesOfFile.append(byte) + while let chunk = try await it.next() { + bytesOfFile.append(contentsOf: chunk) } } #expect(bytesOfFile.count == 1448) @@ -72,7 +72,7 @@ import SystemPackage let fh = FileHandle(fileDescriptor: fd.rawValue, closeOnDealloc: false) if #available(macOS 15, iOS 18, tvOS 18, watchOS 11, visionOS 2, *) { - var it = fh.bytes(on: .global()).makeAsyncIterator() + var it = fh.bytes().makeAsyncIterator() try fd.close() await #expect(throws: (any Error).self) { @@ -80,7 +80,7 @@ import SystemPackage } } } else { - var it = fh._bytes(on: .global()).makeAsyncIterator() + var it = fh._bytes().makeAsyncIterator() try fd.close() await #expect(throws: (any Error).self) { @@ -99,21 +99,21 @@ import SystemPackage try await fd.closeAfter { let fh = FileHandle(fileDescriptor: fd.rawValue, closeOnDealloc: false) if #available(macOS 15, iOS 18, tvOS 18, watchOS 11, visionOS 2, *) { - var it = fh.bytes(on: .global()).makeAsyncIterator() + var it = fh.bytes().makeAsyncIterator() var bytes: [UInt8] = [] - while let byte = try await it.next() { - bytes.append(byte) - if bytes.count == 100 { + while let chunk = try await it.next() { + bytes.append(contentsOf: chunk) + if bytes.count >= 100 { condition.signal() throw CancellationError() } } } else { - var it = fh._bytes(on: .global()).makeAsyncIterator() + var it = fh._bytes().makeAsyncIterator() var bytes: [UInt8] = [] - while let byte = try await it.next() { - bytes.append(byte) - if bytes.count == 100 { + while let chunk = try await it.next() { + bytes.append(contentsOf: chunk) + if bytes.count >= 100 { condition.signal() throw CancellationError() } diff --git a/Tests/SwiftBuildTests/ConsoleCommands/CLIConnection.swift b/Tests/SwiftBuildTests/ConsoleCommands/CLIConnection.swift index 3b987a81..7de7fb62 100644 --- a/Tests/SwiftBuildTests/ConsoleCommands/CLIConnection.swift +++ b/Tests/SwiftBuildTests/ConsoleCommands/CLIConnection.swift @@ -32,8 +32,8 @@ final class CLIConnection { private let monitorHandle: FileHandle private let temporaryDirectory: NamedTemporaryDirectory private let exitPromise: Promise - private let outputStream: AsyncThrowingStream - private var outputStreamIterator: AsyncCLIConnectionResponseSequence>.AsyncIterator + private let outputStream: AsyncThrowingStream + private var outputStreamIterator: AsyncCLIConnectionResponseSequence>>.AsyncIterator static var swiftbuildToolSearchPaths: [URL] { var searchPaths: [URL] = [] @@ -138,8 +138,8 @@ final class CLIConnection { // Close the session handle, so the FD will close once the service stops. try sessionHandle.close() - outputStream = monitorHandle._bytes(on: .global()) - outputStreamIterator = outputStream.cliResponses.makeAsyncIterator() + outputStream = monitorHandle._bytes() + outputStreamIterator = outputStream.flattened.cliResponses.makeAsyncIterator() #endif } @@ -253,6 +253,9 @@ public struct AsyncCLIConnectionResponseSequence: AsyncSequ // BSDs send EOF, Linux raises EIO... #if os(Linux) || os(Android) if error.code == EIO { + if reply.isEmpty { + return nil + } break } #endif @@ -282,6 +285,9 @@ public struct AsyncCLIConnectionResponseSequence: AsyncSequ // BSDs send EOF, Linux raises EIO... #if os(Linux) || os(Android) if error.code == EIO { + if reply.isEmpty { + return nil + } break } #endif diff --git a/Tests/SwiftBuildTests/ConsoleCommands/ServiceConsoleTests.swift b/Tests/SwiftBuildTests/ConsoleCommands/ServiceConsoleTests.swift index 017090a5..26985a79 100644 --- a/Tests/SwiftBuildTests/ConsoleCommands/ServiceConsoleTests.swift +++ b/Tests/SwiftBuildTests/ConsoleCommands/ServiceConsoleTests.swift @@ -33,7 +33,7 @@ fileprivate struct ServiceConsoleTests { let standardOutput = task._makeStream(for: \.standardOutputPipe, using: outputPipe) let promise: Promise = try task.launch() - let data = try await standardOutput.reduce(into: [], { $0.append($1) }) + let data = try await standardOutput.reduce(into: [], { $0.append(contentsOf: $1) }) let output = String(decoding: data, as: UTF8.self) // Verify there were no errors. From 0b3c02837e08b9a32f0e7b8c3a628d8412958c85 Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Tue, 10 Jun 2025 10:20:12 -0700 Subject: [PATCH 34/54] Adapt test code to SE-0481 --- Tests/SwiftBuildTests/BuildOperationTests.swift | 2 -- 1 file changed, 2 deletions(-) diff --git a/Tests/SwiftBuildTests/BuildOperationTests.swift b/Tests/SwiftBuildTests/BuildOperationTests.swift index e9d94dbf..5191e372 100644 --- a/Tests/SwiftBuildTests/BuildOperationTests.swift +++ b/Tests/SwiftBuildTests/BuildOperationTests.swift @@ -1701,7 +1701,6 @@ fileprivate struct BuildOperationTests: CoreBasedTests { let service: SWBBuildService? = try await SWBBuildService() await deferrable.addBlock { [weak service] in await service?.close() - service = nil } func start() async throws -> (SWBBuildOperation, AsyncStream) { @@ -1711,7 +1710,6 @@ fileprivate struct BuildOperationTests: CoreBasedTests { await #expect(throws: Never.self) { try await session?.close() } - session = nil } let testTarget: TestStandardTarget From 427374a651a8f868bc64c0a8c16e1036f9a27400 Mon Sep 17 00:00:00 2001 From: Dave Inglis Date: Fri, 6 Jun 2025 09:18:01 -0400 Subject: [PATCH 35/54] Update LocalFS to use FileManager for all operations LocalFS was using some posix APIs (stat, mkdir, chmod, chown, etc) with some being used in Windows which mean long filenames would fail in some cases, this cleans that up making all platforms that same --- .../Tools/SwiftCompiler.swift | 2 +- Sources/SWBLLBuild/LowLevelBuildSystem.swift | 33 +- .../ODRAssetPackManifestTaskAction.swift | 2 +- Sources/SWBUtil/FSProxy.swift | 450 ++++++------------ Sources/SWBUtil/FilesSignature.swift | 19 +- Sources/SWBUtil/PbxCp.swift | 4 +- .../BuildOperationTests.swift | 4 +- .../SwiftDriverTests.swift | 16 +- .../FileCopyTaskTests.swift | 6 +- Tests/SWBUtilTests/FSProxyTests.swift | 57 ++- 10 files changed, 235 insertions(+), 358 deletions(-) diff --git a/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift b/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift index 687820ec..2464e9c3 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift @@ -526,7 +526,7 @@ public final class SwiftCommandOutputParser: TaskOutputParser { serializedDiagnosticsPaths.filter { path in // rdar://91295617 (Swift produces empty serialized diagnostics if there are none which is not parseable by clang_loadDiagnostics) do { - return try fs.exists(path) && fs.getFileInfo(path).statBuf.st_size > 0 + return try fs.exists(path) && fs.getFileInfo(path).size > 0 } catch { return false } diff --git a/Sources/SWBLLBuild/LowLevelBuildSystem.swift b/Sources/SWBLLBuild/LowLevelBuildSystem.swift index e2193a66..d23d6f98 100644 --- a/Sources/SWBLLBuild/LowLevelBuildSystem.swift +++ b/Sources/SWBLLBuild/LowLevelBuildSystem.swift @@ -11,11 +11,7 @@ //===----------------------------------------------------------------------===// public import SWBUtil -#if os(Windows) -private import SWBLibc -#else public import SWBLibc -#endif // Re-export all APIs from llbuild bindings. @_exported public import llbuild @@ -25,7 +21,34 @@ public import SWBLibc #endif // Filesystem adaptors for SWBLLBuild.FileSystem. -extension SWBUtil.FileInfo: SWBLLBuild.FileInfo {} +extension SWBUtil.FileInfo: SWBLLBuild.FileInfo { + + public init(_ statBuf: stat) { + // This should be remove from llbuild FileInfo protocol as it just not needed, would also be nice to remove the stat requirement too. + preconditionFailure() + } + + public var statBuf: stat { + var statBuf: stat = stat() + + statBuf.st_dev = numericCast(self.deviceID) + statBuf.st_ino = numericCast(self.iNode) + statBuf.st_mode = numericCast(self.permissions) + statBuf.st_size = numericCast(self.size) + #if canImport(Darwin) + statBuf.st_mtimespec.tv_sec = numericCast(self.modificationTimestamp) + statBuf.st_mtimespec.tv_nsec = self.modificationNanoseconds + #elseif os(Windows) + statBuf.st_mtime = self.modificationTimestamp + #elseif canImport(Glibc) || canImport(Musl) || canImport(Android) + statBuf.st_mtim.tv_sec = numericCast(self.modificationTimestamp) + statBuf.st_mtim.tv_nsec = self.modificationNanoseconds + #else + #error("Not implemented for this platform") + #endif + return statBuf + } +} public final class FileSystemImpl: FileSystem { diff --git a/Sources/SWBTaskExecution/TaskActions/ODRAssetPackManifestTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/ODRAssetPackManifestTaskAction.swift index 72f1cfb3..24142a60 100644 --- a/Sources/SWBTaskExecution/TaskActions/ODRAssetPackManifestTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/ODRAssetPackManifestTaskAction.swift @@ -108,7 +108,7 @@ fileprivate extension FSProxy { try traverse(path) { subPath -> Void in let info = try getLinkFileInfo(subPath) - uncompressedSize += Int(info.statBuf.st_size) + uncompressedSize += Int(info.size) newestModTime = max(newestModTime, info.modificationDate) } diff --git a/Sources/SWBUtil/FSProxy.swift b/Sources/SWBUtil/FSProxy.swift index 9f0eb4c2..92a9bf56 100644 --- a/Sources/SWBUtil/FSProxy.swift +++ b/Sources/SWBUtil/FSProxy.swift @@ -10,7 +10,7 @@ // //===----------------------------------------------------------------------===// -public import SWBLibc +import SWBLibc #if canImport(System) public import System @@ -18,6 +18,7 @@ public import System public import SystemPackage #endif +public import struct Foundation.CocoaError public import struct Foundation.Data public import struct Foundation.Date public import struct Foundation.FileAttributeKey @@ -28,82 +29,91 @@ public import struct Foundation.URL public import struct Foundation.URLResourceKey public import struct Foundation.URLResourceValues public import struct Foundation.UUID +public import struct Foundation.FileAttributeType +public import struct Foundation.FileAttributeKey +public import struct Foundation.TimeInterval +public import class Foundation.NSDictionary +#if canImport(Darwin) +import struct ObjectiveC.ObjCBool +#endif #if os(Windows) -// Windows' POSIX layer does not have S_IFLNK, so define it. -// We only need it for PseudoFS. -fileprivate let S_IFLNK: Int32 = 0o0120000 +public import struct WinSDK.HANDLE #endif + /// File system information for a particular file. /// /// This is a simple wrapper for stat() information. public struct FileInfo: Equatable, Sendable { - public let statBuf: stat + public let fileAttrs: [FileAttributeKey: any Sendable] - public init(_ statBuf: stat) { - self.statBuf = statBuf + public init(_ fileAttrs: [FileAttributeKey: any Sendable]) { + self.fileAttrs = fileAttrs + } + + func _readFileAttributePrimitive(_ value: Any?, as type: T.Type) -> T? { + guard let value else { return nil } + if let exact = value as? T { + return exact + } else if let binInt = value as? (any BinaryInteger), let result = T(exactly: binInt) { + return result + } + return nil } public var isFile: Bool { - #if os(Windows) - return (statBuf.st_mode & UInt16(ucrt.S_IFREG)) != 0 - #else - return (statBuf.st_mode & S_IFREG) != 0 - #endif + return (fileAttrs[.type] as! FileAttributeType == .typeRegular) } public var isDirectory: Bool { - #if os(Windows) - return (statBuf.st_mode & UInt16(ucrt.S_IFDIR)) != 0 - #else - return (statBuf.st_mode & S_IFDIR) != 0 - #endif + return fileAttrs[.type] as! FileAttributeType == .typeDirectory } public var isSymlink: Bool { - #if os(Windows) - return (statBuf.st_mode & UInt16(S_IFLNK)) == S_IFLNK - #else - return (statBuf.st_mode & S_IFMT) == S_IFLNK - #endif + return fileAttrs[.type] as! FileAttributeType == .typeSymbolicLink } - public var isExecutable: Bool { - #if os(Windows) - // Per https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/stat-functions, "user execute bits are set according to the filename extension". - // Don't use FileManager.isExecutableFile due to https://github.com/swiftlang/swift-foundation/issues/860 - return (statBuf.st_mode & UInt16(_S_IEXEC)) != 0 - #else - return (statBuf.st_mode & S_IXUSR) != 0 - #endif + public var size: Int64 { + return _readFileAttributePrimitive(fileAttrs[.size], as: Int64.self) ?? 0 } - public var permissions: Int { - return Int(statBuf.st_mode & 0o777) + public var permissions: UInt16 { + return _readFileAttributePrimitive(fileAttrs[.posixPermissions], as: UInt16.self) ?? 0 } - public var owner: Int { - return Int(statBuf.st_uid) + public var owner: UInt { + return _readFileAttributePrimitive(fileAttrs[.ownerAccountID], as: UInt.self) ?? 0 } - public var group: Int { - return Int(statBuf.st_gid) + public var group: UInt { + return _readFileAttributePrimitive(fileAttrs[.groupOwnerAccountID], as: UInt.self) ?? 0 } - public var modificationTimestamp: time_t { - return statBuf.st_mtimespec.tv_sec + public var modificationDate: Date { + return fileAttrs[.modificationDate] as! Date } - public var modificationDate: Date { - let secs = statBuf.st_mtimespec.tv_sec - let nsecs = statBuf.st_mtimespec.tv_nsec - // Using reference date, instead of 1970, which offers a bit more nanosecond precision since it is a lower absolute number. - return Date(timeIntervalSinceReferenceDate: Double(secs) - Date.timeIntervalBetween1970AndReferenceDate + (1.0e-9 * Double(nsecs))) + public var modificationTimestamp: Int64 { + let date = fileAttrs[.modificationDate] as! Date + return Int64(date.timeIntervalSince1970) + } + + public var modificationNanoseconds: Int { + let date = fileAttrs[.modificationDate] as! Date + return Int(date.timeIntervalSince1970 * 1_000_000_000.0 - Double(date.timeIntervalSince1970) * 1_000_000_000.0) + } + + public var iNode: UInt64 { + return _readFileAttributePrimitive(fileAttrs[.systemFileNumber], as: UInt64.self) ?? 0 + } + + public var deviceID: Int32 { + return _readFileAttributePrimitive(fileAttrs[.systemNumber], as: Int32.self) ?? 0 } public static func ==(lhs: FileInfo, rhs: FileInfo) -> Bool { - return lhs.statBuf == rhs.statBuf + return NSDictionary(dictionary: lhs.fileAttrs).isEqual(NSDictionary(dictionary: rhs.fileAttrs)) } } @@ -297,19 +307,19 @@ public extension FSProxy { } func getFileSize(_ path: Path) throws -> ByteCount { - try ByteCount(Int64(getFileInfo(path).statBuf.st_size)) + try ByteCount(Int64(getFileInfo(path).size)) } } fileprivate extension FSProxy { - func createFileInfo(_ statBuf: stat) -> FileInfo { + func createFileInfo(_ fileAttrs: [FileAttributeKey: any Sendable]) -> FileInfo { if fileSystemMode == .deviceAgnostic { - var buf = statBuf - buf.st_ino = 0 - buf.st_dev = 0 + var buf = fileAttrs + buf[.systemFileNumber] = 0 + buf[.systemNumber] = 0 return FileInfo(buf) } - return FileInfo(statBuf) + return FileInfo(fileAttrs) } } @@ -348,28 +358,30 @@ class LocalFS: FSProxy, @unchecked Sendable { /// Check whether a filesystem entity exists at the given path. func exists(_ path: Path) -> Bool { - var statBuf = stat() - if stat(path.str, &statBuf) < 0 { - return false - } - return true + fileManager.fileExists(atPath: path.str) } /// Check whether the given path is a directory. /// /// If the given path is a symlink to a directory, then this will return true if the destination of the symlink is a directory. func isDirectory(_ path: Path) -> Bool { - var statBuf = stat() - if stat(path.str, &statBuf) < 0 { - return false +#if canImport(Darwin) + var isDirectory: ObjCBool = false + if fileManager.fileExists(atPath: path.str, isDirectory: &isDirectory) { + return isDirectory.boolValue } - return createFileInfo(statBuf).isDirectory +#else + var isDirectory = false + if fileManager.fileExists(atPath: path.str, isDirectory: &isDirectory) { + return isDirectory + } +#endif + return false } /// Check whether a given path is a symlink. /// - parameter destinationExists: If the path is a symlink, then this `inout` parameter will be set to `true` if the destination exists. Otherwise it will be set to `false`. func isSymlink(_ path: Path, _ destinationExists: inout Bool) -> Bool { - #if os(Windows) do { let destination = try fileManager.destinationOfSymbolicLink(atPath: path.str) destinationExists = exists((path.isAbsolute ? path.dirname : Path.currentDirectory).join(destination)) @@ -378,22 +390,6 @@ class LocalFS: FSProxy, @unchecked Sendable { destinationExists = false return false } - #else - destinationExists = false - var statBuf = stat() - if lstat(path.str, &statBuf) < 0 { - return false - } - guard createFileInfo(statBuf).isSymlink else { - return false - } - statBuf = stat() - if stat(path.str, &statBuf) < 0 { - return true - } - destinationExists = true - return true - #endif } func listdir(_ path: Path) throws -> [String] { @@ -403,71 +399,53 @@ class LocalFS: FSProxy, @unchecked Sendable { /// Creates a directory at the given path. Throws an exception if it cannot do so. /// - parameter recursive: If `false`, then the parent directory at `path` must already exist in order to create the directory. If it doesn't, then it will return without creating the directory (it will not throw an exception). If `true`, then the directory hierarchy of `path` will be created if possible. func createDirectory(_ path: Path, recursive: Bool) throws { - // Try to create the directory. - #if os(Windows) - do { - return try fileManager.createDirectory(atPath: path.str, withIntermediateDirectories: recursive) - } catch { - throw StubError.error("Could not create directory at path '\(path.str)': \(error)") - } - #else - let result = mkdir(path.str, S_IRWXU | S_IRWXG | S_IRWXO) - - // If it succeeded, we are done. - if result == 0 { - return + guard path.isAbsolute else { + throw StubError.error("Cannot recursively create directory at non-absolute path: \(path.str)") } - - // If the failure was because something exists at this path, then we examine it to see whether it means we're okay. - if errno == EEXIST { - var destinationExists = false - if isDirectory(path) { - // If the item at the path is a directory, then we're good. This includes if it's a symlink which points to a directory. - return - } - else if isSymlink(path, &destinationExists) { - // If the item at the path is a symlink, then we check whether it's a broken symlink or points to something that is not a directory. - if destinationExists { - // The destination does exist, so it's not a directory. - throw StubError.error("File is a symbolic link which references a path which is not a directory: \(path.str)") + // If something exists at this path, then we examine it to see whether it means we're okay. + do { + try fileManager.createDirectory(atPath: path.str, withIntermediateDirectories: false) + } catch let error as CocoaError { + if error.code == .fileWriteFileExists || error.code == .fileWriteUnknown { + var destinationExists = false + if isDirectory(path) { + // If the item at the path is a directory, then we're good. This includes if it's a symlink which points to a directory. + return + } + else if isSymlink(path, &destinationExists) { + // If the item at the path is a symlink, then we check whether it's a broken symlink or points to something that is not a directory. + if destinationExists { + // The destination does exist, so it's not a directory. + throw StubError.error("File is a symbolic link which references a path which is not a directory: \(path.str)") + } + else { + // The destination does not exist - throw an exception because we have a broken symlink. + throw StubError.error("File is a broken symbolic link: \(path.str)") + } } else { - // The destination does not exist - throw an exception because we have a broken symlink. - throw StubError.error("File is a broken symbolic link: \(path.str)") + /// The path exists but is not a directory + throw StubError.error("File exists but is not a directory: \(path.str)") } } - else { - /// The path exists but is not a directory - throw StubError.error("File exists but is not a directory: \(path.str)") - } - } - - // If we are recursive and not the root path, then... - if recursive && !path.isRoot { - // If it failed due to ENOENT (e.g., a missing parent), then attempt to create the parent and retry. - if errno == ENOENT { - // Attempt to create the parent. - guard path.isAbsolute else { - throw StubError.error("Cannot recursively create directory at non-absolute path: \(path.str)") - } - try createDirectory(path.dirname, recursive: true) - - // Re-attempt creation, non-recursively. - try createDirectory(path) + if recursive && !path.isRoot { + if error.code == .fileNoSuchFile { + // Attempt to create the parent. + try createDirectory(path.dirname, recursive: true) - // We are done. - return - } + // Re-attempt creation, non-recursively. + try createDirectory(path) - // If our parent is not a directory, then report that. - if !isDirectory(path.dirname) { - throw StubError.error("File exists but is not a directory: \(path.dirname.str)") + // We are done. + return + } + // If our parent is not a directory, then report that. + if !isDirectory(path.dirname) { + throw StubError.error("File exists but is not a directory: \(path.dirname.str)") + } } + throw error } - - // Otherwise, we failed due to some other error. Report it. - throw POSIXError(errno, context: "mkdir", path.str, "S_IRWXU | S_IRWXG | S_IRWXO") - #endif } func createTemporaryDirectory(parent: Path) throws -> Path { @@ -569,49 +547,21 @@ class LocalFS: FSProxy, @unchecked Sendable { } func remove(_ path: Path) throws { - guard unlink(path.str) == 0 else { - throw POSIXError(errno, context: "unlink", path.str) - } + try fileManager.removeItem(atPath: path.str) } func removeDirectory(_ path: Path) throws { if isDirectory(path) { - #if os(Windows) try fileManager.removeItem(atPath: path.str) - #else - var paths = [path] - try traverse(path) { paths.append($0) } - for path in paths.reversed() { - guard SWBLibc.remove(path.str) == 0 else { - throw POSIXError(errno, context: "remove", path.str) - } - } - #endif } } func setFilePermissions(_ path: Path, permissions: Int) throws { - #if os(Windows) - // permissions work differently on Windows - #else - try eintrLoop { - guard chmod(path.str, mode_t(permissions)) == 0 else { - throw POSIXError(errno, context: "chmod", path.str, String(mode_t(permissions))) - } - } - #endif + try fileManager.setAttributes([.posixPermissions: Int(permissions)], ofItemAtPath: path.str) } func setFileOwnership(_ path: Path, owner: Int, group: Int) throws { - #if os(Windows) - // permissions work differently on Windows - #else - try eintrLoop { - guard chown(path.str, uid_t(owner), gid_t(group)) == 0 else { - throw POSIXError(errno, context: "chown", path.str, String(uid_t(owner)), String(gid_t(group))) - } - } - #endif + try fileManager.setAttributes([.ownerAccountID: owner, .groupOwnerAccountID: group], ofItemAtPath: path.str) } func touch(_ path: Path) throws { @@ -627,24 +577,23 @@ class LocalFS: FSProxy, @unchecked Sendable { } func getFileInfo(_ path: Path) throws -> FileInfo { - var buf = stat() - - try eintrLoop { - guard stat(path.str, &buf) == 0 else { - throw POSIXError(errno, context: "stat", path.str) + if isSymlink(path) { + var destinationPath = try fileManager.destinationOfSymbolicLink(atPath: path.str) + if !Path(destinationPath).isAbsolute { + destinationPath = path.dirname.join(Path(destinationPath)).str } + return createFileInfo(try fileManager.attributesOfItem(atPath: destinationPath)) } - - return createFileInfo(buf) + return createFileInfo(try fileManager.attributesOfItem(atPath: path.str)) } func getFilePermissions(_ path: Path) throws -> Int { - return try getFileInfo(path).permissions + return try Int(getFileInfo(path).permissions) } func getFileOwnership(_ path: Path) throws -> (owner: Int, group: Int) { let fileInfo = try getFileInfo(path) - return (fileInfo.owner, fileInfo.group) + return (Int(fileInfo.owner), Int(fileInfo.group)) } func getFileTimestamp(_ path: Path) throws -> Int { @@ -652,7 +601,7 @@ class LocalFS: FSProxy, @unchecked Sendable { } func isExecutable(_ path: Path) throws -> Bool { - return try getFileInfo(path).isExecutable + return fileManager.isExecutableFile(atPath: path.str) } func isFile(_ path: Path) throws -> Bool { @@ -660,27 +609,7 @@ class LocalFS: FSProxy, @unchecked Sendable { } func getLinkFileInfo(_ path: Path) throws -> FileInfo { - var buf = stat() - #if os(Windows) - try eintrLoop { - guard stat(path.str, &buf) == 0 else { - throw POSIXError(errno, context: "lstat", path.str) - } - } - - var destinationExists = false - if isSymlink(path, &destinationExists) { - buf.st_mode &= ~UInt16(ucrt.S_IFREG) - buf.st_mode |= UInt16(S_IFLNK) - } - #else - try eintrLoop { - guard lstat(path.str, &buf) == 0 else { - throw POSIXError(errno, context: "lstat", path.str) - } - } - #endif - return createFileInfo(buf) + return try createFileInfo(fileManager.attributesOfItem(atPath: path.str)) } @discardableResult func traverse(_ path: Path, _ f: (Path) throws -> T?) throws -> [T] { @@ -700,11 +629,7 @@ class LocalFS: FSProxy, @unchecked Sendable { } func symlink(_ path: Path, target: Path) throws { - #if os(Windows) try fileManager.createSymbolicLink(atPath: path.str, withDestinationPath: target.str) - #else - guard SWBLibc.symlink(target.str, path.str) == 0 else { throw POSIXError(errno, context: "symlink", target.str, path.str) } - #endif } func setIsExcludedFromBackup(_ path: Path, _ value: Bool) throws { @@ -854,18 +779,7 @@ class LocalFS: FSProxy, @unchecked Sendable { } func readlink(_ path: Path) throws -> Path { - #if os(Windows) return try Path(fileManager.destinationOfSymbolicLink(atPath: path.str)) - #else - let buf = UnsafeMutablePointer.allocate(capacity: Int(PATH_MAX) + 1) - defer { buf.deallocate() } - let result = SWBLibc.readlink(path.str, buf, Int(PATH_MAX)) - guard result >= 0 else { - throw POSIXError(errno, context: "readlink", path.str) - } - buf[result] = 0 - return Path(String.init(cString: buf)) - #endif } func getFreeDiskSpace(_ path: Path) throws -> ByteCount? { @@ -1329,41 +1243,31 @@ public class PseudoFS: FSProxy, @unchecked Sendable { guard let node = getNode(path) else { throw POSIXError(ENOENT) } switch node.contents { case .file(let contents): - var info = stat() - #if os(Windows) - info.st_mtimespec = timespec(tv_sec: Int64(node.timestamp), tv_nsec: 0) - #else - info.st_mtimespec = timespec(tv_sec: time_t(node.timestamp), tv_nsec: 0) - #endif - info.st_size = off_t(contents.bytes.count) - info.st_dev = node.device - info.st_ino = node.inode + let info: [FileAttributeKey: any Sendable] = [ + .modificationDate : Date(timeIntervalSince1970: TimeInterval(node.timestamp)), + .type: FileAttributeType.typeRegular, + .size: contents.bytes.count, + .posixPermissions: 0, + .systemNumber: node.device, + .systemFileNumber: node.inode] return createFileInfo(info) case .directory(let dir): - var info = stat() - #if os(Windows) - info.st_mode = UInt16(ucrt.S_IFDIR) - info.st_mtimespec = timespec(tv_sec: Int64(node.timestamp), tv_nsec: 0) - #else - info.st_mode = S_IFDIR - info.st_mtimespec = timespec(tv_sec: time_t(node.timestamp), tv_nsec: 0) - #endif - info.st_size = off_t(dir.contents.count) - info.st_dev = node.device - info.st_ino = node.inode + let info: [FileAttributeKey: any Sendable] = [ + .modificationDate: Date(timeIntervalSince1970: TimeInterval(node.timestamp)), + .type: FileAttributeType.typeDirectory, + .size: dir.contents.count, + .posixPermissions: 0, + .systemNumber: node.device, + .systemFileNumber: node.inode] return createFileInfo(info) case .symlink(_): - var info = stat() - #if os(Windows) - info.st_mode = UInt16(S_IFLNK) - info.st_mtimespec = timespec(tv_sec: Int64(node.timestamp), tv_nsec: 0) - #else - info.st_mode = S_IFLNK - info.st_mtimespec = timespec(tv_sec: time_t(node.timestamp), tv_nsec: 0) - #endif - info.st_size = off_t(0) - info.st_dev = node.device - info.st_ino = node.inode + let info: [FileAttributeKey: any Sendable] = [ + .modificationDate: Date(timeIntervalSince1970: TimeInterval(node.timestamp)), + .type: FileAttributeType.typeSymbolicLink, + .size: 0, + .posixPermissions: 0, + .systemNumber: node.device, + .systemFileNumber: node.inode] return createFileInfo(info) } } @@ -1481,72 +1385,6 @@ public func createFS(simulated: Bool, ignoreFileSystemDeviceInodeChanges: Bool) } } -fileprivate extension stat { - static func ==(lhs: stat, rhs: stat) -> Bool { - return ( - lhs.st_dev == rhs.st_dev && - lhs.st_ino == rhs.st_ino && - lhs.st_mode == rhs.st_mode && - lhs.st_nlink == rhs.st_nlink && - lhs.st_uid == rhs.st_uid && - lhs.st_gid == rhs.st_gid && - lhs.st_rdev == rhs.st_rdev && - lhs.st_atimespec == rhs.st_atimespec && - lhs.st_mtimespec == rhs.st_mtimespec && - lhs.st_ctimespec == rhs.st_ctimespec && - lhs.st_size == rhs.st_size) - } -} - -extension timespec: Equatable { - public static func ==(lhs: timespec, rhs: timespec) -> Bool { - return lhs.tv_sec == rhs.tv_sec && lhs.tv_nsec == rhs.tv_nsec - } -} - -#if os(Windows) -public struct timespec: Sendable { - public let tv_sec: Int64 - public let tv_nsec: Int64 -} - -extension stat { - public var st_atim: timespec { - get { timespec(tv_sec: st_atime, tv_nsec: 0) } - set { st_atime = newValue.tv_sec } - } - - public var st_mtim: timespec { - get { timespec(tv_sec: st_mtime, tv_nsec: 0) } - set { st_mtime = newValue.tv_sec } - } - - public var st_ctim: timespec { - get { timespec(tv_sec: st_ctime, tv_nsec: 0) } - set { st_ctime = newValue.tv_sec } - } -} -#endif - -#if !canImport(Darwin) -extension stat { - public var st_atimespec: timespec { - get { st_atim } - set { st_atim = newValue } - } - - public var st_mtimespec: timespec { - get { st_mtim } - set { st_mtim = newValue } - } - - public var st_ctimespec: timespec { - get { st_ctim } - set { st_ctim = newValue } - } -} -#endif - #if os(Windows) extension HANDLE { /// Runs a closure and then closes the HANDLE, even if an error occurs. diff --git a/Sources/SWBUtil/FilesSignature.swift b/Sources/SWBUtil/FilesSignature.swift index 8cbc25c7..6b716f5d 100644 --- a/Sources/SWBUtil/FilesSignature.swift +++ b/Sources/SWBUtil/FilesSignature.swift @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// import SWBLibc +internal import Foundation /// Represents an opaque signature of a list of files. /// @@ -51,18 +52,18 @@ fileprivate extension FSProxy { /// /// The signature returned is a byte string constructed from an MD5 of properties of all of the files, so the order of `paths` is significant, and a different signature may be returned for different orderings. func filesSignature(_ paths: [Path]) -> ByteString { - var stats: [(Path, stat?)] = [] + var stats: [(Path, FileInfo?)] = [] for path in paths { if isDirectory(path) { do { try traverse(path) { subPath in - stats.append((subPath, try? getFileInfo(subPath).statBuf)) + stats.append((subPath, try? getFileInfo(subPath))) } } catch { stats.append((path, nil)) } } else { - stats.append((path, try? getFileInfo(path).statBuf)) + stats.append((path, try? getFileInfo(path))) } } @@ -70,17 +71,17 @@ fileprivate extension FSProxy { } /// Returns the signature of a list of files. - func filesSignature(_ statInfos: [(Path, stat?)]) -> ByteString { + func filesSignature(_ statInfos: [(Path, FileInfo?)]) -> ByteString { let md5Context = InsecureHashContext() for (path, statInfo) in statInfos { md5Context.add(string: path.str) if let statInfo { md5Context.add(string: "stat") - md5Context.add(number: statInfo.st_ino) - md5Context.add(number: statInfo.st_dev) - md5Context.add(number: statInfo.st_size) - md5Context.add(number: statInfo.st_mtimespec.tv_sec) - md5Context.add(number: statInfo.st_mtimespec.tv_nsec) + md5Context.add(number: statInfo.iNode) + md5Context.add(number: statInfo.deviceID) + md5Context.add(number: statInfo.size) + md5Context.add(number: statInfo.modificationTimestamp) + md5Context.add(number: statInfo.modificationNanoseconds) } else { md5Context.add(string: "") } diff --git a/Sources/SWBUtil/PbxCp.swift b/Sources/SWBUtil/PbxCp.swift index f5be8a02..ee9b67a3 100644 --- a/Sources/SWBUtil/PbxCp.swift +++ b/Sources/SWBUtil/PbxCp.swift @@ -404,14 +404,14 @@ fileprivate func copyEntry(_ srcPath: Path, _ srcTopLevelPath: Path, _ srcParent } else if fileInfo.isFile { try await copyRegular(srcPath, srcParentPath, dstPath, options: options, verbose: verbose, indentationLevel: indentationLevel, outStream: outStream) if verbose { - let size = fileInfo.statBuf.st_size + let size = fileInfo.size textOutput(" \(size) bytes", indentTo: indentationLevel, outStream: outStream) } return 1 } else if fileInfo.isDirectory { return try await copyDirectory(srcPath, srcTopLevelPath, srcParentPath, dstPath, options: options, verbose: verbose, indentationLevel: indentationLevel, outStream: outStream) } else { - throw StubError.error("\(srcPath): unsupported or unknown stat mode (0x\(String(format: "%02x", fileInfo.statBuf.st_mode))") + throw StubError.error("\(srcPath): unsupported or unknown file type: \(fileInfo.fileAttrs[.type] as! String)") } } diff --git a/Tests/SWBBuildSystemTests/BuildOperationTests.swift b/Tests/SWBBuildSystemTests/BuildOperationTests.swift index f3a0c446..8b0c82a4 100644 --- a/Tests/SWBBuildSystemTests/BuildOperationTests.swift +++ b/Tests/SWBBuildSystemTests/BuildOperationTests.swift @@ -4782,7 +4782,7 @@ That command depends on command in Target 'agg2' (project \'aProject\'): script try await tester.checkBuild(runDestination: .macOS, persistent: true) { results in if !SWBFeatureFlag.performOwnershipAnalysis.value { for _ in 0..<4 { - results.checkError(.contains("No such file or directory (2) (for task: [\"Copy\"")) + results.checkError(.contains("couldn’t be opened because there is no such file. (for task: [\"Copy\"")) } } results.checkError(.contains("unterminated string literal")) @@ -5076,7 +5076,7 @@ That command depends on command in Target 'agg2' (project \'aProject\'): script } if !SWBFeatureFlag.performOwnershipAnalysis.value { for fname in ["aFramework.swiftmodule", "aFramework.swiftdoc", "aFramework.swiftsourceinfo", "aFramework.abi.json"] { - results.checkError(.contains("\(tmpDirPath.str)/Test/aProject/build/aProject.build/Debug/aFramework.build/Objects-normal/x86_64/\(fname)): No such file or directory (2)")) + results.checkError(.contains("The file “\(fname)” couldn’t be opened because there is no such file.")) } } results.checkError("Build input file cannot be found: \'\(tmpDirPath.str)/Test/aProject/File.swift\'. Did you forget to declare this file as an output of a script phase or custom build rule which produces it? (for task: [\"ExtractAppIntentsMetadata\"])") diff --git a/Tests/SWBBuildSystemTests/SwiftDriverTests.swift b/Tests/SWBBuildSystemTests/SwiftDriverTests.swift index 062e17ea..e6209702 100644 --- a/Tests/SWBBuildSystemTests/SwiftDriverTests.swift +++ b/Tests/SWBBuildSystemTests/SwiftDriverTests.swift @@ -2489,10 +2489,10 @@ fileprivate struct SwiftDriverTests: CoreBasedTests { if !SWBFeatureFlag.performOwnershipAnalysis.value { results.checkErrors([ - .contains("No such file or directory"), - .contains("No such file or directory"), - .contains("No such file or directory"), - .contains("No such file or directory"), + .contains("couldn’t be opened because there is no such file."), + .contains("couldn’t be opened because there is no such file."), + .contains("couldn’t be opened because there is no such file."), + .contains("couldn’t be opened because there is no such file."), ]) } @@ -2585,10 +2585,10 @@ fileprivate struct SwiftDriverTests: CoreBasedTests { if !SWBFeatureFlag.performOwnershipAnalysis.value { results.checkErrors([ - .contains("No such file or directory"), - .contains("No such file or directory"), - .contains("No such file or directory"), - .contains("No such file or directory"), + .contains("couldn’t be opened because there is no such file."), + .contains("couldn’t be opened because there is no such file."), + .contains("couldn’t be opened because there is no such file."), + .contains("couldn’t be opened because there is no such file."), ]) } diff --git a/Tests/SWBTaskExecutionTests/FileCopyTaskTests.swift b/Tests/SWBTaskExecutionTests/FileCopyTaskTests.swift index da70238e..d27708d8 100644 --- a/Tests/SWBTaskExecutionTests/FileCopyTaskTests.swift +++ b/Tests/SWBTaskExecutionTests/FileCopyTaskTests.swift @@ -122,7 +122,11 @@ fileprivate struct FileCopyTaskTests { #expect(result == .failed) // Examine the error messages. - XCTAssertMatch(outputDelegate.errors, [.suffix("MissingFile.bogus): No such file or directory (2)")]) + #if canImport(Darwin) + XCTAssertMatch(outputDelegate.errors, [.suffix("The file “MissingFile.bogus” couldn’t be opened because there is no such file.")]) + #else + XCTAssertMatch(outputDelegate.errors, [.suffix("The operation could not be completed. The file doesn’t exist.")]) + #endif } } } diff --git a/Tests/SWBUtilTests/FSProxyTests.swift b/Tests/SWBUtilTests/FSProxyTests.swift index 1f56717d..7a1aa90a 100644 --- a/Tests/SWBUtilTests/FSProxyTests.swift +++ b/Tests/SWBUtilTests/FSProxyTests.swift @@ -36,7 +36,7 @@ import SWBTestSupport // MARK: LocalFS Tests - @Test(.skipHostOS(.windows)) // FIXME: error handling is different on Windows + @Test func localCreateDirectory() throws { try withTemporaryDirectory { (tmpDir: Path) in // Create a directory inside the tmpDir. @@ -111,7 +111,11 @@ import SWBTestSupport } catch { didThrow = true + #if os(Windows) #expect(error.localizedDescription == "File exists but is not a directory: \(filePath.str)") + #else + #expect(error.localizedDescription == "File exists but is not a directory: \(dirPath.str)") + #endif } #expect(didThrow) } @@ -179,7 +183,11 @@ import SWBTestSupport } catch { didThrow = true - #expect(error.localizedDescription == "File exists but is not a directory: \(symlinkPath.str)") + #if os(Windows) + #expect(error.localizedDescription == "File is a symbolic link which references a path which is not a directory: \(symlinkPath.str)") + #else + #expect(error.localizedDescription == "File exists but is not a directory: \(dirPath.str)") + #endif } #expect(didThrow) } @@ -200,7 +208,7 @@ import SWBTestSupport #expect { try localFS.createDirectory(filePath, recursive: true) } throws: { error in - error.localizedDescription == "Cannot recursively create directory at non-absolute path: foo/bar/baz" + error.localizedDescription == "Cannot recursively create directory at non-absolute path: \(filePath.str)" } } } @@ -390,7 +398,7 @@ import SWBTestSupport #expect(!fileInfo.isSymlink) let linkFileInfo = try localFS.getLinkFileInfo(file) - #expect(fileInfo.statBuf.st_ino == linkFileInfo.statBuf.st_ino) + #expect(fileInfo.iNode == linkFileInfo.iNode) #expect(!linkFileInfo.isSymlink) // Test absolute and relative targets @@ -406,7 +414,7 @@ import SWBTestSupport #expect(try localFS.read(sym) == data) let symFileInfo = try localFS.getFileInfo(sym) - #expect(symFileInfo.statBuf.st_ino == fileInfo.statBuf.st_ino) + #expect(symFileInfo.iNode == fileInfo.iNode) #expect(!symFileInfo.isSymlink) let symLinkFileInfo = try localFS.getLinkFileInfo(sym) @@ -450,9 +458,7 @@ import SWBTestSupport // not working on Windows for some reason let hostOS = try ProcessInfo.processInfo.hostOperatingSystem() - withKnownIssue { - #expect(fsModDate == fileMgrModDate) - } when: { hostOS == .windows } + #expect(fsModDate == fileMgrModDate) } } @@ -849,18 +855,20 @@ import SWBTestSupport // Check that file stat information differs. #expect(try fs.getFileInfo(Path.root.join("subdir/a.txt")) != fs.getFileInfo(Path.root.join("subdir/b.txt"))) -#if !os(Windows) // Check that we can get stat info on the directory. let s = try fs.getFileInfo(Path.root.join("subdir")) - #expect(s.statBuf.st_mode & S_IFDIR == S_IFDIR) - #expect(s.statBuf.st_size == 2) + #expect(s.isDirectory) + #expect(s.size == 2) // Check that the stat info changes if we mutate the directory. try fs.remove(Path.root.join("subdir/b.txt")) try fs.write(Path.root.join("subdir/c.txt"), contents: "c") let s2 = try fs.getFileInfo(Path.root.join("subdir")) #expect(s != s2) -#endif + + let f = try fs.getFileInfo(Path.root.join("subdir")) + let f2 = try fs.getFileInfo(Path.root.join("subdir")) + #expect(f == f2) } @Test @@ -1141,10 +1149,13 @@ import SWBTestSupport } func _testCopyTree(_ fs: any FSProxy, basePath: Path) throws { - func compareFileInfo(_ lhs: FileInfo, _ rhs: FileInfo, sourceLocation: SourceLocation = #_sourceLocation) { + func compareFileInfo(_ lhsPath: Path, _ rhsPAth: Path, sourceLocation: SourceLocation = #_sourceLocation) throws { + let lhs = try fs.getFileInfo(lhsPath) + let rhs = try fs.getFileInfo(rhsPAth) + + #expect(FileManager.default.isExecutableFile(atPath: lhsPath.str) == FileManager.default.isExecutableFile(atPath: rhsPAth.str), sourceLocation: sourceLocation) #expect(lhs.group == rhs.group, sourceLocation: sourceLocation) #expect(lhs.isDirectory == rhs.isDirectory, sourceLocation: sourceLocation) - #expect(lhs.isExecutable == rhs.isExecutable, sourceLocation: sourceLocation) #expect(lhs.isSymlink == rhs.isSymlink, sourceLocation: sourceLocation) if fs is PseudoFS { // There is no guarantee that the implementation of copy() will preserve the modification timestamp on either files and/or directories, on any real filesystem, so only make this assertion for the pseudo filesystem which we wholly control. @@ -1188,11 +1199,11 @@ import SWBTestSupport #expect(try fs.getFilePermissions(subdirDst.join("dir0/file0")) == file0Perms) #expect(try fs.getFilePermissions(subdirDst.join("dir0/dir0_0/file1")) == file1Perms) } - compareFileInfo(try fs.getFileInfo(subdirDst.join("dir0")), try fs.getFileInfo(subdir.join("dir0"))) - compareFileInfo(try fs.getFileInfo(subdirDst.join("dir0/file0")), try fs.getFileInfo(subdir.join("dir0/file0"))) - compareFileInfo(try fs.getFileInfo(subdirDst.join("dir0/dir0_0/file1")), try fs.getFileInfo(subdir.join("dir0/dir0_0/file1"))) - compareFileInfo(try fs.getFileInfo(subdirDst.join("dir0/dir0_0")), try fs.getFileInfo(subdir.join("dir0/dir0_0"))) - compareFileInfo(try fs.getFileInfo(subdirDst.join("dir1")), try fs.getFileInfo(subdir.join("dir1"))) + try compareFileInfo(subdirDst.join("dir0"), subdir.join("dir0")) + try compareFileInfo(subdirDst.join("dir0/file0"), subdir.join("dir0/file0")) + try compareFileInfo(subdirDst.join("dir0/dir0_0/file1"), subdir.join("dir0/dir0_0/file1")) + try compareFileInfo(subdirDst.join("dir0/dir0_0"), subdir.join("dir0/dir0_0")) + try compareFileInfo(subdirDst.join("dir1"), subdir.join("dir1")) // Test the file contents. #expect(try ByteString(data0) == fs.read(subdirDst.join("dir0/file0"))) @@ -1212,9 +1223,9 @@ import SWBTestSupport let sig0a_orig = fs.filesSignature([file0]) // Validate that the inode/device info is only 0 when the info should be ignored. - let inode = try fs.getFileInfo(file0).statBuf.st_ino + let inode = try fs.getFileInfo(file0).iNode #expect((inode == 0) == shouldIgnoreDeviceInodeChanges) - let device = try fs.getFileInfo(file0).statBuf.st_dev + let device = try fs.getFileInfo(file0).deviceID #expect((device == 0) == shouldIgnoreDeviceInodeChanges) // Copy the file and copy it back, keeping the attributes of the file intact. NOTE!! Do not change this from copy/remove to move as that will **not** necessarily change the st_ino value. By copying the file, we can guarantee that a new file inode must be created. @@ -1255,9 +1266,9 @@ import SWBTestSupport // Validate that the inode/device info is only 0 when the info should be ignored. for file in [dir0, dir1, file0] { - let inode = try fs.getFileInfo(file).statBuf.st_ino + let inode = try fs.getFileInfo(file).iNode #expect((inode == 0) == shouldIgnoreDeviceInodeChanges) - let device = try fs.getFileInfo(file).statBuf.st_dev + let device = try fs.getFileInfo(file).deviceID #expect((device == 0) == shouldIgnoreDeviceInodeChanges) } From e357809a3001ad20481c9f11a1956d85e5bf0e13 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 11 Jun 2025 10:07:26 -0500 Subject: [PATCH 36/54] Ensure the path to Swift Testing's macro plugin is specified correctly when using a non-default Xcode toolchain (#574) This resolves an issue which can occur when using Xcode 26 Beta with the downloadable Metal toolchain installed, if a target imports Swift Testing. A toolchain override is specified in this scenario, and this causes the existing logic modified by this PR to incorrectly believe it's using a non-Xcode toolchain, which in turn causes the wrong Swift macro plugin flags to be passed, ultimately leading to a failure locating the `TestingMacros` plugin. The fix is to recognize all toolchains which have Xcode's prefix, and ensure the default toolchain prefix is used whenever any Xcode toolchain is in use. Fixes rdar://152440128 --- Sources/SWBCore/Settings/Settings.swift | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/Sources/SWBCore/Settings/Settings.swift b/Sources/SWBCore/Settings/Settings.swift index 70d95c0d..048a1f6d 100644 --- a/Sources/SWBCore/Settings/Settings.swift +++ b/Sources/SWBCore/Settings/Settings.swift @@ -4273,17 +4273,20 @@ private class SettingsBuilder { } let toolchainPath = Path(scope.evaluateAsString(BuiltinMacros.TOOLCHAIN_DIR)) - guard let toolchain = core.toolchainRegistry.toolchains.first(where: { $0.path == toolchainPath }) else { + guard let toolchain = core.toolchainRegistry.toolchains.first(where: { $0.path == toolchainPath }), + let defaultToolchain = core.toolchainRegistry.defaultToolchain + else { return [] } enum ToolchainStyle { - case xcodeDefault + case xcode(isDefault: Bool) case other init(_ toolchain: Toolchain) { - if toolchain.identifier == ToolchainRegistry.defaultToolchainIdentifier { - self = .xcodeDefault + if toolchain.identifier.hasPrefix(ToolchainRegistry.appleToolchainIdentifierPrefix) { + let isDefault = toolchain.identifier == ToolchainRegistry.defaultToolchainIdentifier + self = .xcode(isDefault: isDefault) } else { self = .other } @@ -4292,11 +4295,12 @@ private class SettingsBuilder { let testingPluginsPath = "/usr/lib/swift/host/plugins/testing" switch (ToolchainStyle(toolchain)) { - case .xcodeDefault: - // This target is building using the same toolchain as the one used - // to build the testing libraries which it is using, so it can use - // non-external plugin flags. - return ["-plugin-path", "$(TOOLCHAIN_DIR)\(testingPluginsPath)"] + case let .xcode(isDefault): + // This target is using a built-in Xcode toolchain, and that should + // match the toolchain which was used to build the testing libraries + // this target is using, so it can use non-external plugin flags. + let toolchainPathPrefix = isDefault ? "$(TOOLCHAIN_DIR)" : defaultToolchain.path.str + return ["-plugin-path", "\(toolchainPathPrefix)\(testingPluginsPath)"] case .other: // This target is using the testing libraries from Xcode, // which were built using the XcodeDefault toolchain, but it's using From 18fcc44220b96b035ec09572ffeef1ea6bc95884 Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Wed, 11 Jun 2025 13:47:10 -0700 Subject: [PATCH 37/54] Fixup cross platform specs handling of DEAD_CODE_STRIPPING --- .../Specs/UnixLd.xcspec | 10 ++++ Sources/SWBQNXPlatform/Specs/QNXLd.xcspec | 5 ++ Sources/SWBUniversalPlatform/Specs/Ld.xcspec | 5 +- .../Specs/WasmLd.xcspec | 10 ++++ .../SWBWindowsPlatform/Specs/WindowsLd.xcspec | 10 ++++ .../TaskConstructionTests.swift | 60 ++++++++++++++++++- 6 files changed, 98 insertions(+), 2 deletions(-) diff --git a/Sources/SWBGenericUnixPlatform/Specs/UnixLd.xcspec b/Sources/SWBGenericUnixPlatform/Specs/UnixLd.xcspec index e773b048..97316ed8 100644 --- a/Sources/SWBGenericUnixPlatform/Specs/UnixLd.xcspec +++ b/Sources/SWBGenericUnixPlatform/Specs/UnixLd.xcspec @@ -102,6 +102,16 @@ }; Condition = "$(ALTERNATE_LINKER) == gold"; }, + { + Name = "DEAD_CODE_STRIPPING"; + Type = Boolean; + DefaultValue = NO; + Condition = "$(MACH_O_TYPE) != mh_object"; + CommandLineArgs = { + YES = ("-Xlinker", "--gc-sections"); + NO = (); + }; + }, { // Frameworks are Mac specific Name = "SYSTEM_FRAMEWORK_SEARCH_PATHS"; diff --git a/Sources/SWBQNXPlatform/Specs/QNXLd.xcspec b/Sources/SWBQNXPlatform/Specs/QNXLd.xcspec index 30f9c24b..6c66a1ca 100644 --- a/Sources/SWBQNXPlatform/Specs/QNXLd.xcspec +++ b/Sources/SWBQNXPlatform/Specs/QNXLd.xcspec @@ -96,6 +96,11 @@ }; Condition = "$(ALTERNATE_LINKER) == gold"; }, + { + Name = "DEAD_CODE_STRIPPING"; + Type = Boolean; + Condition = "NO"; + }, { // Frameworks are Mac specific Name = "SYSTEM_FRAMEWORK_SEARCH_PATHS"; diff --git a/Sources/SWBUniversalPlatform/Specs/Ld.xcspec b/Sources/SWBUniversalPlatform/Specs/Ld.xcspec index d30e08ad..657c22b1 100644 --- a/Sources/SWBUniversalPlatform/Specs/Ld.xcspec +++ b/Sources/SWBUniversalPlatform/Specs/Ld.xcspec @@ -459,7 +459,10 @@ Type = Boolean; DefaultValue = NO; Condition = "$(MACH_O_TYPE) != mh_object"; - CommandLineFlag = "-dead_strip"; + CommandLineArgs = { + YES = ("-dead_strip"); + NO = (); + }; }, { Name = "BUNDLE_LOADER"; diff --git a/Sources/SWBWebAssemblyPlatform/Specs/WasmLd.xcspec b/Sources/SWBWebAssemblyPlatform/Specs/WasmLd.xcspec index 4b246b98..ba106172 100644 --- a/Sources/SWBWebAssemblyPlatform/Specs/WasmLd.xcspec +++ b/Sources/SWBWebAssemblyPlatform/Specs/WasmLd.xcspec @@ -64,6 +64,16 @@ NO = (); }; }, + { + Name = "DEAD_CODE_STRIPPING"; + Type = Boolean; + DefaultValue = NO; + Condition = "$(MACH_O_TYPE) != mh_object"; + CommandLineArgs = { + YES = ("-Xlinker", "--gc-sections"); + NO = (); + }; + }, { // Frameworks are Mac specific Name = "SYSTEM_FRAMEWORK_SEARCH_PATHS"; diff --git a/Sources/SWBWindowsPlatform/Specs/WindowsLd.xcspec b/Sources/SWBWindowsPlatform/Specs/WindowsLd.xcspec index 39f50000..fe66cb3f 100644 --- a/Sources/SWBWindowsPlatform/Specs/WindowsLd.xcspec +++ b/Sources/SWBWindowsPlatform/Specs/WindowsLd.xcspec @@ -102,6 +102,16 @@ Type = String; Condition = "NO"; }, + { + Name = "DEAD_CODE_STRIPPING"; + Type = Boolean; + DefaultValue = NO; + Condition = "$(MACH_O_TYPE) != mh_object"; + CommandLineArgs = { + YES = ("-Xlinker", "/OPT:REF"); + NO = (); + }; + }, { // No such concept Name = "LD_RUNPATH_SEARCH_PATHS"; diff --git a/Tests/SWBTaskConstructionTests/TaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/TaskConstructionTests.swift index a0edbbe6..3c55f3d9 100644 --- a/Tests/SWBTaskConstructionTests/TaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/TaskConstructionTests.swift @@ -10,7 +10,7 @@ // //===----------------------------------------------------------------------===// -import struct Foundation.Data +import Foundation import Testing @@ -8657,6 +8657,64 @@ fileprivate struct TaskConstructionTests: CoreBasedTests { } } + @Test(.requireSDKs(.host)) + func crossPlatformDeadCodeStripping() async throws { + try await withTemporaryDirectory { tmpDir in + let testProject = TestProject( + "aProject", + sourceRoot: tmpDir, + groupTree: TestGroup( + "SomeFiles", path: "Sources", + children: [ + TestFile("SourceFile.c"), + ]), + buildConfigurations: [ + TestBuildConfiguration("Debug", buildSettings: [ + "DEAD_CODE_STRIPPING": "YES", + "ONLY_ACTIVE_ARCH": "YES" + ]) + ], + targets: [ + TestStandardTarget( + "Library", + type: .dynamicLibrary, + buildConfigurations: [ + TestBuildConfiguration("Debug") + ], + buildPhases: [ + TestSourcesBuildPhase([ + "SourceFile.c", + ]) + ] + )] + ) + + let fs = PseudoFS() + + let core = try await getCore() + let tester = try TaskConstructionTester(core, testProject) + + try await tester.checkBuild(BuildParameters(configuration: "Debug", overrides: [:]), runDestination: .host, fs: fs) { results in + try results.checkTask(.matchRuleType("Ld")) { task in + switch try ProcessInfo.processInfo.hostOperatingSystem() { + case .macOS: + task.checkCommandLineContains(["-dead_strip"]) + task.checkCommandLineDoesNotContain("--gc-sections") + task.checkCommandLineDoesNotContain("/OPT:REF") + case .windows: + task.checkCommandLineDoesNotContain("-dead_strip") + task.checkCommandLineDoesNotContain("--gc-sections") + task.checkCommandLineContains(["/OPT:REF"]) + default: + task.checkCommandLineDoesNotContain("-dead_strip") + task.checkCommandLineContains(["--gc-sections"]) + task.checkCommandLineDoesNotContain("/OPT:REF") + } + } + } + } + } + @Test(.requireSDKs(.macOS)) func warningSuppression() async throws { try await withTemporaryDirectory { tmpDir in From 6c6309e4b37a0775db191f8bcc503f90c7c9edc7 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 12 Jun 2025 07:53:51 -0700 Subject: [PATCH 38/54] [Preview] Fix preview support for swift caching build (#562) Add support for swift caching build when swift compiler supports the new option that allows swift compiler performs an uncached build but loading module dependencies from CAS. It also does few adjustment to make sure preview build shares the same dependencies with regular build as caching build has a stricter rule for invalidation: * The VFS overlay used by preview is applied after swift driver invocation and onto the frontend invocation directly. This makes sure the VFS overlay doesn't invalidate all the module dependencies. * BridgingHeader PCH was disabled as a workaround when swift driver doesn't produce deterministic output path when planning for PCH jobs. Properly fix this issue by requesting the same path when planning both build and don't use this workaround in certain configurations. This should allow preview to build thunk correctly when swift caching is enabled for the regular build. rdar://152107465 --- .../Tools/SwiftCompiler.swift | 48 ++++++++++++++----- .../PreviewsBuildOperationTests.swift | 12 ++--- .../GeneratePreviewInfoTests.swift | 3 +- 3 files changed, 43 insertions(+), 20 deletions(-) diff --git a/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift b/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift index 2464e9c3..a8c6e2bc 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift @@ -337,6 +337,7 @@ public struct SwiftDriverPayload: Serializable, TaskPayload, Encodable { public let uniqueID: String public let compilerLocation: LibSwiftDriver.CompilerLocation public let moduleName: String + public let outputPrefix: String public let tempDirPath: Path public let explicitModulesTempDirPath: Path public let variant: String @@ -352,10 +353,11 @@ public struct SwiftDriverPayload: Serializable, TaskPayload, Encodable { public let dependencyFilteringRootPath: Path? public let verifyScannerDependencies: Bool - internal init(uniqueID: String, compilerLocation: LibSwiftDriver.CompilerLocation, moduleName: String, tempDirPath: Path, explicitModulesTempDirPath: Path, variant: String, architecture: String, eagerCompilationEnabled: Bool, explicitModulesEnabled: Bool, commandLine: [String], ruleInfo: [String], isUsingWholeModuleOptimization: Bool, casOptions: CASOptions?, reportRequiredTargetDependencies: BooleanWarningLevel, linkerResponseFilePath: Path?, dependencyFilteringRootPath: Path?, verifyScannerDependencies: Bool) { + internal init(uniqueID: String, compilerLocation: LibSwiftDriver.CompilerLocation, moduleName: String, outputPrefix: String, tempDirPath: Path, explicitModulesTempDirPath: Path, variant: String, architecture: String, eagerCompilationEnabled: Bool, explicitModulesEnabled: Bool, commandLine: [String], ruleInfo: [String], isUsingWholeModuleOptimization: Bool, casOptions: CASOptions?, reportRequiredTargetDependencies: BooleanWarningLevel, linkerResponseFilePath: Path?, dependencyFilteringRootPath: Path?, verifyScannerDependencies: Bool) { self.uniqueID = uniqueID self.compilerLocation = compilerLocation self.moduleName = moduleName + self.outputPrefix = outputPrefix self.tempDirPath = tempDirPath self.explicitModulesTempDirPath = explicitModulesTempDirPath self.variant = variant @@ -373,10 +375,11 @@ public struct SwiftDriverPayload: Serializable, TaskPayload, Encodable { } public init(from deserializer: any Deserializer) throws { - try deserializer.beginAggregate(17) + try deserializer.beginAggregate(18) self.uniqueID = try deserializer.deserialize() self.compilerLocation = try deserializer.deserialize() self.moduleName = try deserializer.deserialize() + self.outputPrefix = try deserializer.deserialize() self.tempDirPath = try deserializer.deserialize() self.explicitModulesTempDirPath = try deserializer.deserialize() self.variant = try deserializer.deserialize() @@ -394,10 +397,11 @@ public struct SwiftDriverPayload: Serializable, TaskPayload, Encodable { } public func serialize(to serializer: T) where T : Serializer { - serializer.serializeAggregate(17) { + serializer.serializeAggregate(18) { serializer.serialize(self.uniqueID) serializer.serialize(self.compilerLocation) serializer.serialize(self.moduleName) + serializer.serialize(self.outputPrefix) serializer.serialize(self.tempDirPath) serializer.serialize(self.explicitModulesTempDirPath) serializer.serialize(self.variant) @@ -2502,7 +2506,7 @@ public final class SwiftCompilerSpec : CompilerSpec, SpecIdentifierType, SwiftDi let explicitModuleBuildEnabled = await swiftExplicitModuleBuildEnabled(cbc.producer, cbc.scope, delegate) let verifyScannerDependencies = explicitModuleBuildEnabled && cbc.scope.evaluate(BuiltinMacros.SWIFT_DEPENDENCY_REGISTRATION_MODE) == .verifySwiftDependencyScanner - return SwiftDriverPayload(uniqueID: uniqueID, compilerLocation: compilerLocation, moduleName: scope.evaluate(BuiltinMacros.SWIFT_MODULE_NAME), tempDirPath: tempDirPath, explicitModulesTempDirPath: explicitModulesTempDirPath, variant: variant, architecture: arch, eagerCompilationEnabled: eagerCompilationEnabled(args: args, scope: scope, compilationMode: compilationMode, isUsingWholeModuleOptimization: isUsingWholeModuleOptimization), explicitModulesEnabled: explicitModuleBuildEnabled, commandLine: commandLine, ruleInfo: ruleInfo, isUsingWholeModuleOptimization: isUsingWholeModuleOptimization, casOptions: casOptions, reportRequiredTargetDependencies: scope.evaluate(BuiltinMacros.DIAGNOSE_MISSING_TARGET_DEPENDENCIES), linkerResponseFilePath: linkerResponseFilePath, dependencyFilteringRootPath: cbc.producer.sdk?.path, verifyScannerDependencies: verifyScannerDependencies) + return SwiftDriverPayload(uniqueID: uniqueID, compilerLocation: compilerLocation, moduleName: scope.evaluate(BuiltinMacros.SWIFT_MODULE_NAME), outputPrefix: scope.evaluate(BuiltinMacros.TARGET_NAME) + compilationMode.moduleBaseNameSuffix, tempDirPath: tempDirPath, explicitModulesTempDirPath: explicitModulesTempDirPath, variant: variant, architecture: arch, eagerCompilationEnabled: eagerCompilationEnabled(args: args, scope: scope, compilationMode: compilationMode, isUsingWholeModuleOptimization: isUsingWholeModuleOptimization), explicitModulesEnabled: explicitModuleBuildEnabled, commandLine: commandLine, ruleInfo: ruleInfo, isUsingWholeModuleOptimization: isUsingWholeModuleOptimization, casOptions: casOptions, reportRequiredTargetDependencies: scope.evaluate(BuiltinMacros.DIAGNOSE_MISSING_TARGET_DEPENDENCIES), linkerResponseFilePath: linkerResponseFilePath, dependencyFilteringRootPath: cbc.producer.sdk?.path, verifyScannerDependencies: verifyScannerDependencies) } func constructSwiftResponseFileTask(path: Path) { @@ -3397,10 +3401,12 @@ public final class SwiftCompilerSpec : CompilerSpec, SpecIdentifierType, SwiftDi removeWithParameter(arg) } - // We need to ignore precompiled headers due to: + // For some old version of swift driver, the output path for bridging header pch is not stable, + // we need to disable bridging header pch when caching or bridging header chaining is not enabled as a workaround: // rdar://126212044 ([JIT] iOS test Failures: Thunk build failure, unable to read PCH file) - removeWithPrefix("-cache-compile-job") - commandLine.append("-disable-bridging-pch") + if !commandLine.contains("-cache-compile-job") || !commandLine.contains("-auto-bridging-header-chaining") { + commandLine.append("-disable-bridging-pch") + } if payload.previewStyle == .dynamicReplacement { for (arg, newValue) in [ @@ -3440,6 +3446,7 @@ public final class SwiftCompilerSpec : CompilerSpec, SpecIdentifierType, SwiftDi } let selectedInputPath: Path + let newVFSOverlayPath: Path? if payload.previewStyle == .xojit { // Also pass the auxiliary Swift files. commandLine.append(contentsOf: originalInputs.map(\.str)) @@ -3448,7 +3455,8 @@ public final class SwiftCompilerSpec : CompilerSpec, SpecIdentifierType, SwiftDi if let driverPayload = payload.driverPayload { do { // Inject the thunk source into the output file map - let map = SwiftOutputFileMap(files: [sourceFile.str: .init(object: outputPath.str)]) + let pchPath = driverPayload.tempDirPath.join(driverPayload.outputPrefix + "-primary-Bridging-header.pch") + let map = SwiftOutputFileMap(files: [sourceFile.str: .init(object: outputPath.str), "": .init(pch: pchPath.str)]) let newOutputFileMap = driverPayload.tempDirPath.join(UUID().uuidString) try fs.createDirectory(newOutputFileMap.dirname, recursive: true) try fs.write(newOutputFileMap, contents: ByteString(JSONEncoder(outputFormatting: [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes]).encode(map))) @@ -3457,19 +3465,20 @@ public final class SwiftCompilerSpec : CompilerSpec, SpecIdentifierType, SwiftDi // rdar://127735418 ([JIT] Emit a vfsoverlay for JIT preview thunk compiler arguments so clients can specify the original file path when substituting contents) let vfs = VFS() vfs.addMapping(sourceFile, externalContents: inputPath) - let newVFSOverlayPath = driverPayload.tempDirPath.join("vfsoverlay-\(inputPath.basename).json") + newVFSOverlayPath = driverPayload.tempDirPath.join("vfsoverlay-\(inputPath.basename).json") try fs.createDirectory(newOutputFileMap.dirname, recursive: true) let overlay = try vfs.toVFSOverlay().propertyListItem.asJSONFragment().asString - try fs.write(newVFSOverlayPath, contents: ByteString(encodingAsUTF8: overlay)) - - commandLine.append(contentsOf: ["-vfsoverlay", newVFSOverlayPath.str]) + try fs.write(newVFSOverlayPath!, contents: ByteString(encodingAsUTF8: overlay)) } catch { return [] } + } else { + newVFSOverlayPath = nil } } else { selectedInputPath = inputPath + newVFSOverlayPath = nil commandLine.append(contentsOf: [inputPath.str]) } @@ -3516,6 +3525,21 @@ public final class SwiftCompilerSpec : CompilerSpec, SpecIdentifierType, SwiftDi casOptions: driverPayload.casOptions ) { commandLine = newCommandLine + // For swift caching jobs, add extra flags. + if commandLine.contains("-cache-compile-job") { + // Ideally, we should ask if swift-frontend supports the flag but we can only ask driver for an approximation. + if LibSwiftDriver.supportsDriverFlag(spelled: "-module-import-from-cas") { + commandLine.append("-module-import-from-cas") + } + // Then drop the cache build flag to do uncached preview compilation. + commandLine.removeAll { + $0 == "-cache-compile-job" + } + } + // Add vfsoverlay after the driver invocation as it can affect the module dependencies, causing modules from regular builds not being reused here. + if let vfsOverlay = newVFSOverlayPath { + commandLine.append(contentsOf: ["-vfsoverlay", vfsOverlay.str]) + } } else { commandLine = [] } diff --git a/Tests/SWBBuildSystemTests/PreviewsBuildOperationTests.swift b/Tests/SWBBuildSystemTests/PreviewsBuildOperationTests.swift index f27b8a59..f6627365 100644 --- a/Tests/SWBBuildSystemTests/PreviewsBuildOperationTests.swift +++ b/Tests/SWBBuildSystemTests/PreviewsBuildOperationTests.swift @@ -227,8 +227,6 @@ fileprivate struct PreviewsBuildOperationTests: CoreBasedTests { "\(srcRoot.str)/build/Debug-iphonesimulator", "-F", "\(srcRoot.str)/build/Debug-iphonesimulator", - "-vfsoverlay", - "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/vfsoverlay-main.selection.preview-thunk.swift.json", "-no-color-diagnostics", "-g", "-debug-info-format=dwarf", @@ -278,7 +276,9 @@ fileprivate struct PreviewsBuildOperationTests: CoreBasedTests { "-target-sdk-name", "iphonesimulator\(core.loadSDK(.iOSSimulator).defaultDeploymentTarget)", "-o", - "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/main.selection.preview-thunk.o" + "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/main.selection.preview-thunk.o", + "-vfsoverlay", + "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/vfsoverlay-main.selection.preview-thunk.swift.json", ] ) #expect(previewInfo.thunkInfo?.thunkSourceFile == Path("\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/main.selection.preview-thunk.swift")) @@ -531,8 +531,6 @@ fileprivate struct PreviewsBuildOperationTests: CoreBasedTests { "\(srcRoot.str)/build/Debug-iphonesimulator", "-F", "\(srcRoot.str)/build/Debug-iphonesimulator", - "-vfsoverlay", - "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/vfsoverlay-File1.selection.preview-thunk.swift.json", "-no-color-diagnostics", "-g", "-debug-info-format=dwarf", @@ -582,7 +580,9 @@ fileprivate struct PreviewsBuildOperationTests: CoreBasedTests { "-target-sdk-name", "iphonesimulator\(core.loadSDK(.iOSSimulator).defaultDeploymentTarget)", "-o", - "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/File1.selection.preview-thunk.o" + "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/File1.selection.preview-thunk.o", + "-vfsoverlay", + "\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/vfsoverlay-File1.selection.preview-thunk.swift.json", ] ) #expect(previewInfo.thunkInfo?.thunkSourceFile == Path("\(srcRoot.str)/build/ProjectName.build/Debug-iphonesimulator/AppTarget.build/Objects-normal/\(results.runDestinationTargetArchitecture)/File1.selection.preview-thunk.swift")) diff --git a/Tests/SwiftBuildTests/GeneratePreviewInfoTests.swift b/Tests/SwiftBuildTests/GeneratePreviewInfoTests.swift index aa2d667c..ab4b2b97 100644 --- a/Tests/SwiftBuildTests/GeneratePreviewInfoTests.swift +++ b/Tests/SwiftBuildTests/GeneratePreviewInfoTests.swift @@ -160,14 +160,13 @@ fileprivate struct GeneratePreviewInfoTests: CoreBasedTests { .anySequence, "-sdk", "\(sdkroot)", .anySequence, - "-vfsoverlay", "\(tmpDir.str)/Test/build/Test.build/Debug-iphoneos/App.build/Objects-normal/\(activeRunDestination.targetArchitecture)/vfsoverlay-TestFile4.canary.preview-thunk.swift.json", - .anySequence, "-Onone", .anySequence, "-module-name", "App", "-target-sdk-version", "\(deploymentTarget)", "-target-sdk-name", "iphoneos\(deploymentTarget)", "-o", "\(tmpDir.str)/Test/build/Test.build/Debug-iphoneos/App.build/Objects-normal/\(activeRunDestination.targetArchitecture)/TestFile4.canary.preview-thunk.o", + "-vfsoverlay", "\(tmpDir.str)/Test/build/Test.build/Debug-iphoneos/App.build/Objects-normal/\(activeRunDestination.targetArchitecture)/vfsoverlay-TestFile4.canary.preview-thunk.swift.json", .end ]) // Also spot-check that some options which were removed in SwiftCompilerSpec.generatePreviewInfo() for XOJIT are not present. From 02eb8cb5b90c8e8f9b3334e54f11ce4168a0fb76 Mon Sep 17 00:00:00 2001 From: Bob Wilson Date: Mon, 16 Jun 2025 11:38:02 -0700 Subject: [PATCH 39/54] Add a new VALIDATE_MODULE_DEPENDENCIES setting This will be used for validating the new MODULE_DEPENDENCIES setting. --- Sources/SWBCore/Settings/BuiltinMacros.swift | 2 ++ Sources/SWBCore/SpecImplementations/PropertyDomainSpec.swift | 1 + 2 files changed, 3 insertions(+) diff --git a/Sources/SWBCore/Settings/BuiltinMacros.swift b/Sources/SWBCore/Settings/BuiltinMacros.swift index 21be4140..31fbf53b 100644 --- a/Sources/SWBCore/Settings/BuiltinMacros.swift +++ b/Sources/SWBCore/Settings/BuiltinMacros.swift @@ -1135,6 +1135,7 @@ public final class BuiltinMacros { public static let VALIDATE_PRODUCT = BuiltinMacros.declareBooleanMacro("VALIDATE_PRODUCT") public static let VALIDATE_DEPENDENCIES = BuiltinMacros.declareEnumMacro("VALIDATE_DEPENDENCIES") as EnumMacroDeclaration public static let VALIDATE_DEVELOPMENT_ASSET_PATHS = BuiltinMacros.declareEnumMacro("VALIDATE_DEVELOPMENT_ASSET_PATHS") as EnumMacroDeclaration + public static let VALIDATE_MODULE_DEPENDENCIES = BuiltinMacros.declareEnumMacro("VALIDATE_MODULE_DEPENDENCIES") as EnumMacroDeclaration public static let VECTOR_SUFFIX = BuiltinMacros.declareStringMacro("VECTOR_SUFFIX") public static let VERBOSE_PBXCP = BuiltinMacros.declareBooleanMacro("VERBOSE_PBXCP") public static let VERSIONING_STUB = BuiltinMacros.declareStringMacro("VERSIONING_STUB") @@ -2342,6 +2343,7 @@ public final class BuiltinMacros { VALIDATE_PRODUCT, VALIDATE_DEPENDENCIES, VALIDATE_DEVELOPMENT_ASSET_PATHS, + VALIDATE_MODULE_DEPENDENCIES, VALID_ARCHS, VECTOR_SUFFIX, VERBOSE_PBXCP, diff --git a/Sources/SWBCore/SpecImplementations/PropertyDomainSpec.swift b/Sources/SWBCore/SpecImplementations/PropertyDomainSpec.swift index 7dec160d..67997f43 100644 --- a/Sources/SWBCore/SpecImplementations/PropertyDomainSpec.swift +++ b/Sources/SWBCore/SpecImplementations/PropertyDomainSpec.swift @@ -100,6 +100,7 @@ private final class EnumBuildOptionType : BuildOptionType { case "PACKAGE_RESOURCE_TARGET_KIND": return try namespace.declareEnumMacro(name) as EnumMacroDeclaration case "VALIDATE_DEPENDENCIES", + "VALIDATE_MODULE_DEPENDENCIES", "VALIDATE_DEVELOPMENT_ASSET_PATHS": return try namespace.declareEnumMacro(name) as EnumMacroDeclaration case "STRIP_STYLE": From c639a7f74587d3d8a8f0309a8535c1da4c6c4384 Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Sun, 15 Jun 2025 19:34:04 -0700 Subject: [PATCH 40/54] Lookup unix fallback toolchain relative to SWIFT_EXEC when set --- Sources/SWBGenericUnixPlatform/Plugin.swift | 69 +++++++++++---------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/Sources/SWBGenericUnixPlatform/Plugin.swift b/Sources/SWBGenericUnixPlatform/Plugin.swift index a3839068..0d6cd88c 100644 --- a/Sources/SWBGenericUnixPlatform/Plugin.swift +++ b/Sources/SWBGenericUnixPlatform/Plugin.swift @@ -129,39 +129,44 @@ struct GenericUnixToolchainRegistryExtension: ToolchainRegistryExtension { let fs = context.fs - if let swift = StackedSearchPath(environment: .current, fs: fs).lookup(Path("swift")), fs.exists(swift) { - let realSwiftPath = try fs.realpath(swift).dirname.normalize() - let hasUsrBin = realSwiftPath.str.hasSuffix("/usr/bin") - let hasUsrLocalBin = realSwiftPath.str.hasSuffix("/usr/local/bin") - let path: Path - switch (hasUsrBin, hasUsrLocalBin) { - case (true, false): - path = realSwiftPath.dirname.dirname - case (false, true): - path = realSwiftPath.dirname.dirname.dirname - case (false, false): - throw StubError.error("Unexpected toolchain layout for Swift installation path: \(realSwiftPath)") - case (true, true): - preconditionFailure() + for swift in [ + Environment.current["SWIFT_EXEC"].map(Path.init), + StackedSearchPath(environment: .current, fs: fs).lookup(Path("swift")) + ].compactMap(\.self) { + if fs.exists(swift) { + let realSwiftPath = try fs.realpath(swift).dirname.normalize() + let hasUsrBin = realSwiftPath.str.hasSuffix("/usr/bin") + let hasUsrLocalBin = realSwiftPath.str.hasSuffix("/usr/local/bin") + let path: Path + switch (hasUsrBin, hasUsrLocalBin) { + case (true, false): + path = realSwiftPath.dirname.dirname + case (false, true): + path = realSwiftPath.dirname.dirname.dirname + case (false, false): + throw StubError.error("Unexpected toolchain layout for Swift installation path: \(realSwiftPath)") + case (true, true): + preconditionFailure() + } + let llvmDirectories = try Array(fs.listdir(Path("/usr/lib")).filter { $0.hasPrefix("llvm-") }.sorted().reversed()) + let llvmDirectoriesLocal = try Array(fs.listdir(Path("/usr/local")).filter { $0.hasPrefix("llvm") }.sorted().reversed()) + return [ + Toolchain( + identifier: ToolchainRegistry.defaultToolchainIdentifier, + displayName: "Default", + version: Version(), + aliases: ["default"], + path: path, + frameworkPaths: [], + libraryPaths: llvmDirectories.map { "/usr/lib/\($0)/lib" } + llvmDirectoriesLocal.map { "/usr/local/\($0)/lib" } + ["/usr/lib64"], + defaultSettings: [:], + overrideSettings: [:], + defaultSettingsWhenPrimary: [:], + executableSearchPaths: realSwiftPath.dirname.relativeSubpath(from: path).map { [path.join($0).join("bin")] } ?? [], + testingLibraryPlatformNames: [], + fs: fs) + ] } - let llvmDirectories = try Array(fs.listdir(Path("/usr/lib")).filter { $0.hasPrefix("llvm-") }.sorted().reversed()) - let llvmDirectoriesLocal = try Array(fs.listdir(Path("/usr/local")).filter { $0.hasPrefix("llvm") }.sorted().reversed()) - return [ - Toolchain( - identifier: ToolchainRegistry.defaultToolchainIdentifier, - displayName: "Default", - version: Version(), - aliases: ["default"], - path: path, - frameworkPaths: [], - libraryPaths: llvmDirectories.map { "/usr/lib/\($0)/lib" } + llvmDirectoriesLocal.map { "/usr/local/\($0)/lib" } + ["/usr/lib64"], - defaultSettings: [:], - overrideSettings: [:], - defaultSettingsWhenPrimary: [:], - executableSearchPaths: realSwiftPath.dirname.relativeSubpath(from: path).map { [path.join($0).join("bin")] } ?? [], - testingLibraryPlatformNames: [], - fs: fs) - ] } return [] From 2957dd1f8a7f46fb4a89644063b28822d248b49d Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Tue, 17 Jun 2025 21:24:33 -0700 Subject: [PATCH 41/54] Skip Windows tests triggering hangs in CI --- Tests/SWBBuildServiceTests/BuildServiceTests.swift | 3 ++- Tests/SwiftBuildTests/BuildOperationTests.swift | 2 +- .../ConsoleCommands/CreateXCFrameworkCommandTests.swift | 2 +- .../ConsoleCommands/GeneralCommandsTests.swift | 2 +- .../SwiftBuildTests/ConsoleCommands/ServiceConsoleTests.swift | 2 +- .../ConsoleCommands/SessionCommandsTests.swift | 2 +- .../SwiftBuildTests/ConsoleCommands/XcodeCommandsTests.swift | 2 +- Tests/SwiftBuildTests/MacroEvaluationTests.swift | 4 ++-- Tests/SwiftBuildTests/PIFTests.swift | 4 ++-- 9 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Tests/SWBBuildServiceTests/BuildServiceTests.swift b/Tests/SWBBuildServiceTests/BuildServiceTests.swift index ddd64936..6178220d 100644 --- a/Tests/SWBBuildServiceTests/BuildServiceTests.swift +++ b/Tests/SWBBuildServiceTests/BuildServiceTests.swift @@ -16,7 +16,8 @@ import SwiftBuild import SWBBuildService import SWBTestSupport -@Suite fileprivate struct BuildServiceTests: CoreBasedTests { +@Suite(.skipHostOS(.windows)) +fileprivate struct BuildServiceTests: CoreBasedTests { @Test func createXCFramework() async throws { do { let (result, message) = try await withBuildService { await $0.createXCFramework([], currentWorkingDirectory: Path.root.str, developerPath: nil) } diff --git a/Tests/SwiftBuildTests/BuildOperationTests.swift b/Tests/SwiftBuildTests/BuildOperationTests.swift index 5191e372..077da084 100644 --- a/Tests/SwiftBuildTests/BuildOperationTests.swift +++ b/Tests/SwiftBuildTests/BuildOperationTests.swift @@ -22,7 +22,7 @@ import SWBUtil import SWBTestSupport import SWBProtocol -@Suite +@Suite(.skipHostOS(.windows)) fileprivate struct BuildOperationTests: CoreBasedTests { /// Check the basic behavior of an empty build. @Test diff --git a/Tests/SwiftBuildTests/ConsoleCommands/CreateXCFrameworkCommandTests.swift b/Tests/SwiftBuildTests/ConsoleCommands/CreateXCFrameworkCommandTests.swift index 47d19412..5cd01bb7 100644 --- a/Tests/SwiftBuildTests/ConsoleCommands/CreateXCFrameworkCommandTests.swift +++ b/Tests/SwiftBuildTests/ConsoleCommands/CreateXCFrameworkCommandTests.swift @@ -17,7 +17,7 @@ import SWBTestSupport // Note: The functionality of this class is heavily unit tested in `XCFrameworkTests.swift`. These tests are only to ensure that the command is indeed hooked up and registered properly. -@Suite(.skipInGitHubActions("failing in the GitHub actions runner environment")) +@Suite(.skipInGitHubActions("failing in the GitHub actions runner environment"), .skipHostOS(.windows)) fileprivate struct CreateXCFrameworkCommandTests { @Test func commandInvocation() async throws { diff --git a/Tests/SwiftBuildTests/ConsoleCommands/GeneralCommandsTests.swift b/Tests/SwiftBuildTests/ConsoleCommands/GeneralCommandsTests.swift index 09f3fffa..0fdc8694 100644 --- a/Tests/SwiftBuildTests/ConsoleCommands/GeneralCommandsTests.swift +++ b/Tests/SwiftBuildTests/ConsoleCommands/GeneralCommandsTests.swift @@ -15,7 +15,7 @@ import SwiftBuild import SWBTestSupport import SWBUtil -@Suite(.skipInGitHubActions("failing in the GitHub actions runner environment")) +@Suite(.skipInGitHubActions("failing in the GitHub actions runner environment"), .skipHostOS(.windows)) fileprivate struct GeneralCommandsTests { @Test(.skipHostOS(.windows), // PTY not supported on Windows .requireHostOS(.macOS)) // something with terminal echo is different on macOS vs Linux diff --git a/Tests/SwiftBuildTests/ConsoleCommands/ServiceConsoleTests.swift b/Tests/SwiftBuildTests/ConsoleCommands/ServiceConsoleTests.swift index 26985a79..fa107c6b 100644 --- a/Tests/SwiftBuildTests/ConsoleCommands/ServiceConsoleTests.swift +++ b/Tests/SwiftBuildTests/ConsoleCommands/ServiceConsoleTests.swift @@ -19,7 +19,7 @@ import SWBUtil import WinSDK #endif -@Suite(.skipInGitHubActions("failing in the GitHub actions runner environment")) +@Suite(.skipInGitHubActions("failing in the GitHub actions runner environment"), .skipHostOS(.windows)) fileprivate struct ServiceConsoleTests { @Test func emptyInput() async throws { diff --git a/Tests/SwiftBuildTests/ConsoleCommands/SessionCommandsTests.swift b/Tests/SwiftBuildTests/ConsoleCommands/SessionCommandsTests.swift index 5e8c5bb3..327e7d38 100644 --- a/Tests/SwiftBuildTests/ConsoleCommands/SessionCommandsTests.swift +++ b/Tests/SwiftBuildTests/ConsoleCommands/SessionCommandsTests.swift @@ -14,7 +14,7 @@ import SWBTestSupport import Testing import SWBUtil -@Suite(.skipInGitHubActions("failing in the GitHub actions runner environment")) +@Suite(.skipInGitHubActions("failing in the GitHub actions runner environment"), .skipHostOS(.windows)) fileprivate struct SessionCommandsTests { @Test(.skipHostOS(.windows), // PTY not supported on Windows .requireHostOS(.macOS)) // something with terminal echo is different on macOS vs Linux diff --git a/Tests/SwiftBuildTests/ConsoleCommands/XcodeCommandsTests.swift b/Tests/SwiftBuildTests/ConsoleCommands/XcodeCommandsTests.swift index b824666a..6aaf3805 100644 --- a/Tests/SwiftBuildTests/ConsoleCommands/XcodeCommandsTests.swift +++ b/Tests/SwiftBuildTests/ConsoleCommands/XcodeCommandsTests.swift @@ -14,7 +14,7 @@ import SWBTestSupport import Testing import SWBUtil -@Suite(.skipInGitHubActions("failing in the GitHub actions runner environment")) +@Suite(.skipInGitHubActions("failing in the GitHub actions runner environment"), .skipHostOS(.windows)) fileprivate struct XcodeCommandsTests { @Test(.skipHostOS(.windows), // PTY not supported on Windows .requireHostOS(.macOS)) // something with terminal echo is different on macOS vs Linux diff --git a/Tests/SwiftBuildTests/MacroEvaluationTests.swift b/Tests/SwiftBuildTests/MacroEvaluationTests.swift index ac58f7ce..e399b30e 100644 --- a/Tests/SwiftBuildTests/MacroEvaluationTests.swift +++ b/Tests/SwiftBuildTests/MacroEvaluationTests.swift @@ -18,7 +18,7 @@ import SWBTestSupport @_spi(Testing) import SWBUtil /// Test evaluating both using a scope, and directly against the model objects. -@Suite +@Suite(.skipHostOS(.windows)) fileprivate struct MacroEvaluationTests { @Test func macroEvaluationBasics() async throws { @@ -83,7 +83,7 @@ fileprivate struct MacroEvaluationTests { } } - @Test(.requireSDKs(.host), .userDefaults(["EnablePluginManagerLogging": "0"])) + @Test(.requireSDKs(.host), .skipHostOS(.windows), .userDefaults(["EnablePluginManagerLogging": "0"])) func macroEvaluationAdvanced() async throws { try await withTemporaryDirectory { tmpDir in try await withAsyncDeferrable { deferrable in diff --git a/Tests/SwiftBuildTests/PIFTests.swift b/Tests/SwiftBuildTests/PIFTests.swift index 1bf165cc..333b74c5 100644 --- a/Tests/SwiftBuildTests/PIFTests.swift +++ b/Tests/SwiftBuildTests/PIFTests.swift @@ -226,7 +226,7 @@ fileprivate struct PIFTests { typealias LookupObject = (@Sendable (SwiftBuildServicePIFObjectType, String) async throws -> SWBPropertyListItem) /// Check incremental PIF transfer. - @Test(.userDefaults(["EnablePluginManagerLogging": "0"])) + @Test(.skipHostOS(.windows), .userDefaults(["EnablePluginManagerLogging": "0"])) func sessionPIFLoading() async throws { let service = try await SWBBuildService() @@ -327,7 +327,7 @@ fileprivate struct PIFTests { } /// Check PIF incremental cache. - @Test(.userDefaults(["EnablePluginManagerLogging": "0"])) + @Test(.skipHostOS(.windows), .userDefaults(["EnablePluginManagerLogging": "0"])) func sessionPIFCache() async throws { try await withTemporaryDirectory { tmpDir in try await withAsyncDeferrable { deferrable in From a4b018693d2e0e7614aeb23d0a054e439aac18b3 Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Mon, 16 Jun 2025 12:59:27 -0700 Subject: [PATCH 42/54] Cleanup compilation caching feature availability flags --- .../Extensions/FeatureAvailabilityExtension.swift | 1 - Sources/SWBCore/Settings/Settings.swift | 9 --------- .../SWBCore/SpecImplementations/Tools/CCompiler.swift | 2 -- .../SpecImplementations/Tools/SwiftCompiler.swift | 2 -- Sources/SWBCore/TaskGeneration.swift | 2 -- .../SWBTaskConstruction/TaskProducers/TaskProducer.swift | 4 ---- Sources/SWBTaskExecution/BuildDescriptionManager.swift | 2 +- Sources/SWBTestSupport/CoreBasedTests.swift | 8 -------- Sources/SWBTestSupport/DummyCommandProducer.swift | 4 ---- Sources/SWBTestSupport/SkippedTestSupport.swift | 8 +------- .../ClangCompilationCachingTests.swift | 2 +- Tests/SWBBuildSystemTests/ClangModuleVerifierTests.swift | 2 +- .../SwiftCompilationCachingTests.swift | 2 +- Tests/SWBBuildSystemTests/SwiftDriverTests.swift | 2 +- .../CompilationCachingTaskConstructionTests.swift | 2 +- 15 files changed, 7 insertions(+), 45 deletions(-) diff --git a/Sources/SWBCore/Extensions/FeatureAvailabilityExtension.swift b/Sources/SWBCore/Extensions/FeatureAvailabilityExtension.swift index 76b4d38e..a8012305 100644 --- a/Sources/SWBCore/Extensions/FeatureAvailabilityExtension.swift +++ b/Sources/SWBCore/Extensions/FeatureAvailabilityExtension.swift @@ -21,5 +21,4 @@ public struct FeatureAvailabilityExtensionPoint: ExtensionPoint { } public protocol FeatureAvailabilityExtension: Sendable { - var supportsCompilationCaching: Bool { get } } diff --git a/Sources/SWBCore/Settings/Settings.swift b/Sources/SWBCore/Settings/Settings.swift index 048a1f6d..70191a15 100644 --- a/Sources/SWBCore/Settings/Settings.swift +++ b/Sources/SWBCore/Settings/Settings.swift @@ -764,15 +764,6 @@ public final class Settings: PlatformBuildContext, Sendable { (scope.evaluate(BuiltinMacros.IS_ZIPPERED) && scope.evaluate(BuiltinMacros.INDEX_ENABLE_BUILD_ARENA)) } - public static func supportsCompilationCaching(_ core: Core) -> Bool { - @preconcurrency @PluginExtensionSystemActor func featureAvailabilityExtensions() -> [any FeatureAvailabilityExtensionPoint.ExtensionProtocol] { - core.pluginManager.extensions(of: FeatureAvailabilityExtensionPoint.self) - } - return featureAvailabilityExtensions().contains { - $0.supportsCompilationCaching - } - } - public var enableTargetPlatformSpecialization: Bool { return Settings.targetPlatformSpecializationEnabled(scope: globalScope) } diff --git a/Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift b/Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift index 313eba1e..152c391d 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift @@ -824,8 +824,6 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible language: GCCCompatibleLanguageDialect, clangInfo: DiscoveredClangToolSpecInfo? ) -> Bool { - guard cbc.producer.supportsCompilationCaching else { return false } - // Disabling compilation caching for index build, for now. guard !cbc.scope.evaluate(BuiltinMacros.INDEX_ENABLE_BUILD_ARENA) else { return false diff --git a/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift b/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift index a8c6e2bc..acb7d266 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift @@ -1414,8 +1414,6 @@ public final class SwiftCompilerSpec : CompilerSpec, SpecIdentifierType, SwiftDi } private func swiftCachingEnabled(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, _ moduleName: String, _ useIntegratedDriver: Bool, _ explicitModuleBuildEnabled: Bool, _ disabledPCHCompile: Bool) async -> Bool { - guard cbc.producer.supportsCompilationCaching else { return false } - guard cbc.scope.evaluate(BuiltinMacros.SWIFT_ENABLE_COMPILE_CACHE) else { return false } diff --git a/Sources/SWBCore/TaskGeneration.swift b/Sources/SWBCore/TaskGeneration.swift index 511db51e..dfcb5c9f 100644 --- a/Sources/SWBCore/TaskGeneration.swift +++ b/Sources/SWBCore/TaskGeneration.swift @@ -264,8 +264,6 @@ public protocol CommandProducer: PlatformBuildContext, SpecLookupContext, Refere var targetShouldBuildModuleForInstallAPI: Bool { get } - var supportsCompilationCaching: Bool { get } - func lookupLibclang(path: Path) -> (libclang: Libclang?, version: Version?) var userPreferences: UserPreferences { get } diff --git a/Sources/SWBTaskConstruction/TaskProducers/TaskProducer.swift b/Sources/SWBTaskConstruction/TaskProducers/TaskProducer.swift index 38501ae7..01cda675 100644 --- a/Sources/SWBTaskConstruction/TaskProducers/TaskProducer.swift +++ b/Sources/SWBTaskConstruction/TaskProducers/TaskProducer.swift @@ -1390,10 +1390,6 @@ extension TaskProducerContext: CommandProducer { return globalProductPlan.targetsWhichShouldBuildModulesDuringInstallAPI?.contains(configuredTarget) ?? false } - public var supportsCompilationCaching: Bool { - return Settings.supportsCompilationCaching(workspaceContext.core) - } - public var systemInfo: SystemInfo? { return workspaceContext.systemInfo } diff --git a/Sources/SWBTaskExecution/BuildDescriptionManager.swift b/Sources/SWBTaskExecution/BuildDescriptionManager.swift index fc351510..c6a57ead 100644 --- a/Sources/SWBTaskExecution/BuildDescriptionManager.swift +++ b/Sources/SWBTaskExecution/BuildDescriptionManager.swift @@ -183,7 +183,7 @@ package final class BuildDescriptionManager: Sendable { var casValidationInfos: OrderedSet = [] let buildGraph = planRequest.buildGraph - let shouldValidateCAS = Settings.supportsCompilationCaching(plan.workspaceContext.core) && UserDefaults.enableCASValidation + let shouldValidateCAS = UserDefaults.enableCASValidation // Add the SFR identifier for target-independent tasks. staleFileRemovalIdentifierPerTarget[nil] = plan.staleFileRemovalTaskIdentifier(for: nil) diff --git a/Sources/SWBTestSupport/CoreBasedTests.swift b/Sources/SWBTestSupport/CoreBasedTests.swift index 7c3e6f42..4606074a 100644 --- a/Sources/SWBTestSupport/CoreBasedTests.swift +++ b/Sources/SWBTestSupport/CoreBasedTests.swift @@ -243,14 +243,6 @@ extension CoreBasedTests { } } - /// If compilation caching is supported. - package var supportsCompilationCaching: Bool { - get async throws { - let core = try await getCore() - return Settings.supportsCompilationCaching(core) - } - } - package var supportsSDKImports: Bool { get async throws { #if os(macOS) diff --git a/Sources/SWBTestSupport/DummyCommandProducer.swift b/Sources/SWBTestSupport/DummyCommandProducer.swift index 021ee8fd..83a7e342 100644 --- a/Sources/SWBTestSupport/DummyCommandProducer.swift +++ b/Sources/SWBTestSupport/DummyCommandProducer.swift @@ -229,10 +229,6 @@ package struct MockCommandProducer: CommandProducer, Sendable { false } - package var supportsCompilationCaching: Bool { - false - } - package var systemInfo: SystemInfo? { return nil } diff --git a/Sources/SWBTestSupport/SkippedTestSupport.swift b/Sources/SWBTestSupport/SkippedTestSupport.swift index bef3a66e..ca3eed21 100644 --- a/Sources/SWBTestSupport/SkippedTestSupport.swift +++ b/Sources/SWBTestSupport/SkippedTestSupport.swift @@ -367,12 +367,6 @@ extension Trait where Self == Testing.ConditionTrait { } } - package static var requireCompilationCaching: Self { - enabled("compilation caching is not supported") { - try await ConditionTraitContext.shared.supportsCompilationCaching - } - } - package static var requireDependencyScannerPlusCaching: Self { disabled { let libclang = try #require(try await ConditionTraitContext.shared.libclang) @@ -397,7 +391,7 @@ extension Trait where Self == Testing.ConditionTrait { package static var requireCASValidation: Self { enabled { - guard try await ConditionTraitContext.shared.supportsCompilationCaching, UserDefaults.enableCASValidation else { + guard UserDefaults.enableCASValidation else { return false } guard let path = try? await ConditionTraitContext.shared.llvmCasToolPath else { diff --git a/Tests/SWBBuildSystemTests/ClangCompilationCachingTests.swift b/Tests/SWBBuildSystemTests/ClangCompilationCachingTests.swift index f1dd82bc..b9066b22 100644 --- a/Tests/SWBBuildSystemTests/ClangCompilationCachingTests.swift +++ b/Tests/SWBBuildSystemTests/ClangCompilationCachingTests.swift @@ -19,7 +19,7 @@ import SWBTestSupport import SWBUtil @Suite(.skipHostOS(.windows, "Windows platform has no CAS support yet"), - .requireCompilationCaching, .requireDependencyScannerPlusCaching, + .requireDependencyScannerPlusCaching, .flaky("A handful of Swift Build CAS tests fail when running the entire test suite"), .bug("rdar://146781403")) fileprivate struct ClangCompilationCachingTests: CoreBasedTests { let canUseCASPlugin: Bool diff --git a/Tests/SWBBuildSystemTests/ClangModuleVerifierTests.swift b/Tests/SWBBuildSystemTests/ClangModuleVerifierTests.swift index 850a64cd..a05d0662 100644 --- a/Tests/SWBBuildSystemTests/ClangModuleVerifierTests.swift +++ b/Tests/SWBBuildSystemTests/ClangModuleVerifierTests.swift @@ -191,7 +191,7 @@ fileprivate struct ClangModuleVerifierTests: CoreBasedTests { } } - @Test(.requireSDKs(.macOS), .requireClangFeatures(.wSystemHeadersInModule), .requireDependencyScannerPlusCaching, .requireCompilationCaching, + @Test(.requireSDKs(.macOS), .requireClangFeatures(.wSystemHeadersInModule), .requireDependencyScannerPlusCaching, .flaky("A handful of Swift Build CAS tests fail when running the entire test suite"), .bug("rdar://146781403")) func cachedBuild() async throws { try await withTemporaryDirectory { (tmpDirPath: Path) in diff --git a/Tests/SWBBuildSystemTests/SwiftCompilationCachingTests.swift b/Tests/SWBBuildSystemTests/SwiftCompilationCachingTests.swift index da5864bc..7b81a154 100644 --- a/Tests/SWBBuildSystemTests/SwiftCompilationCachingTests.swift +++ b/Tests/SWBBuildSystemTests/SwiftCompilationCachingTests.swift @@ -19,7 +19,7 @@ import SWBUtil import SWBTaskExecution import SWBProtocol -@Suite(.requireSwiftFeatures(.compilationCaching), .requireCompilationCaching, +@Suite(.requireSwiftFeatures(.compilationCaching), .flaky("A handful of Swift Build CAS tests fail when running the entire test suite"), .bug("rdar://146781403")) fileprivate struct SwiftCompilationCachingTests: CoreBasedTests { @Test(.requireSDKs(.iOS)) diff --git a/Tests/SWBBuildSystemTests/SwiftDriverTests.swift b/Tests/SWBBuildSystemTests/SwiftDriverTests.swift index e6209702..7cc31253 100644 --- a/Tests/SWBBuildSystemTests/SwiftDriverTests.swift +++ b/Tests/SWBBuildSystemTests/SwiftDriverTests.swift @@ -5146,7 +5146,7 @@ fileprivate struct SwiftDriverTests: CoreBasedTests { } } - @Test(.requireSDKs(.macOS)) + @Test(.requireSDKs(.macOS), .requireSwiftFeatures(.compilationCaching), .skipSwiftPackage) func ensureIdenticalCommandLinesWithDifferentDependenciesAreNotDeduplicated() async throws { try await withTemporaryDirectory { tmpDir in let testWorkspace = try await TestWorkspace( diff --git a/Tests/SWBTaskConstructionTests/CompilationCachingTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/CompilationCachingTaskConstructionTests.swift index c55ef757..863cd6ea 100644 --- a/Tests/SWBTaskConstructionTests/CompilationCachingTaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/CompilationCachingTaskConstructionTests.swift @@ -17,7 +17,7 @@ import SWBCore @Suite fileprivate struct CompilationCachingTaskConstructionTests: CoreBasedTests { - @Test(.requireSDKs(.macOS, comment: "Caching requires explicit modules, which requires libclang and is only available on macOS"), .requireCompilationCaching) + @Test(.requireSDKs(.macOS, comment: "Caching requires explicit modules, which requires libclang and is only available on macOS")) func settingRemoteCacheSupportedLanguages() async throws { let testProject = try await TestProject( "aProject", From 46faa85cf20dbb29ac05dafd4420819e6aa1a117 Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Fri, 20 Jun 2025 16:51:29 -0700 Subject: [PATCH 43/54] Support invoking the API digester in API mode in addition to ABI mode --- Sources/SWBCore/Settings/BuiltinMacros.swift | 9 ++ .../SpecImplementations/ProductTypes.swift | 4 + .../PropertyDomainSpec.swift | 2 + ...SwiftFrameworkABICheckerTaskProducer.swift | 24 +++- .../SWBUniversalPlatform/Specs/Swift.xcspec | 44 ++++++- ...SwiftABICheckerTaskConstructionTests.swift | 113 +++++++++++++++++- 6 files changed, 185 insertions(+), 11 deletions(-) diff --git a/Sources/SWBCore/Settings/BuiltinMacros.swift b/Sources/SWBCore/Settings/BuiltinMacros.swift index 31fbf53b..ea94fc70 100644 --- a/Sources/SWBCore/Settings/BuiltinMacros.swift +++ b/Sources/SWBCore/Settings/BuiltinMacros.swift @@ -974,6 +974,7 @@ public final class BuiltinMacros { public static let REZ_PREFIX_FILE = BuiltinMacros.declarePathMacro("REZ_PREFIX_FILE") public static let REZ_SEARCH_PATHS = BuiltinMacros.declarePathListMacro("REZ_SEARCH_PATHS") public static let RUN_CLANG_STATIC_ANALYZER = BuiltinMacros.declareBooleanMacro("RUN_CLANG_STATIC_ANALYZER") + public static let SWIFT_API_DIGESTER_MODE = BuiltinMacros.declareEnumMacro("SWIFT_API_DIGESTER_MODE") as EnumMacroDeclaration public static let RUN_SWIFT_ABI_CHECKER_TOOL = BuiltinMacros.declareBooleanMacro("RUN_SWIFT_ABI_CHECKER_TOOL") public static let RUN_SWIFT_ABI_CHECKER_TOOL_DRIVER = BuiltinMacros.declareBooleanMacro("RUN_SWIFT_ABI_CHECKER_TOOL_DRIVER") public static let RUN_SWIFT_ABI_GENERATION_TOOL = BuiltinMacros.declareBooleanMacro("RUN_SWIFT_ABI_GENERATION_TOOL") @@ -2107,6 +2108,7 @@ public final class BuiltinMacros { SKIP_BUILDING_DOCUMENTATION, RUN_SYMBOL_GRAPH_EXTRACT, SYSTEM_EXTENSIONS_FOLDER_PATH, + SWIFT_API_DIGESTER_MODE, RUN_SWIFT_ABI_CHECKER_TOOL, RUN_SWIFT_ABI_CHECKER_TOOL_DRIVER, RUN_SWIFT_ABI_GENERATION_TOOL, @@ -2640,6 +2642,13 @@ public enum SwiftEnableExplicitModulesSetting: String, Equatable, Hashable, Enum case disabled = "NO" } +public enum SwiftAPIDigesterMode: String, Equatable, Hashable, EnumerationMacroType { + public static let defaultValue: SwiftAPIDigesterMode = .abi + + case abi = "abi" + case api = "api" +} + public enum SwiftDependencyRegistrationMode: String, Equatable, Hashable, EnumerationMacroType { public static let defaultValue: SwiftDependencyRegistrationMode = .makeStyleDependenciesSupplementedByScanner diff --git a/Sources/SWBCore/SpecImplementations/ProductTypes.swift b/Sources/SWBCore/SpecImplementations/ProductTypes.swift index 28415e4b..7f26983b 100644 --- a/Sources/SWBCore/SpecImplementations/ProductTypes.swift +++ b/Sources/SWBCore/SpecImplementations/ProductTypes.swift @@ -772,6 +772,10 @@ public class StandaloneExecutableProductTypeSpec : ProductTypeSpec, SpecClassTyp public class var className: String { return "XCStandaloneExecutableProductType" } + + public override var supportsSwiftABIChecker: Bool { + true + } } public class LibraryProductTypeSpec: StandaloneExecutableProductTypeSpec, @unchecked Sendable { diff --git a/Sources/SWBCore/SpecImplementations/PropertyDomainSpec.swift b/Sources/SWBCore/SpecImplementations/PropertyDomainSpec.swift index 67997f43..eed3f5e2 100644 --- a/Sources/SWBCore/SpecImplementations/PropertyDomainSpec.swift +++ b/Sources/SWBCore/SpecImplementations/PropertyDomainSpec.swift @@ -113,6 +113,8 @@ private final class EnumBuildOptionType : BuildOptionType { return try namespace.declareEnumMacro(name) as EnumMacroDeclaration case "LINKER_DRIVER": return try namespace.declareEnumMacro(name) as EnumMacroDeclaration + case "SWIFT_API_DIGESTER_MODE": + return try namespace.declareEnumMacro(name) as EnumMacroDeclaration default: return try namespace.declareStringMacro(name) } diff --git a/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/SwiftFrameworkABICheckerTaskProducer.swift b/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/SwiftFrameworkABICheckerTaskProducer.swift index dd829325..6b52df82 100644 --- a/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/SwiftFrameworkABICheckerTaskProducer.swift +++ b/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/SwiftFrameworkABICheckerTaskProducer.swift @@ -20,8 +20,8 @@ fileprivate func supportSwiftABIChecking(_ context: TaskProducerContext) -> Bool // swift-api-digester is run only when the "build" component is present. guard scope.evaluate(BuiltinMacros.BUILD_COMPONENTS).contains("build") else { return false } - guard scope.evaluate(BuiltinMacros.SWIFT_EMIT_MODULE_INTERFACE) && - scope.evaluate(BuiltinMacros.SWIFT_ENABLE_LIBRARY_EVOLUTION) else { + guard scope.evaluate(BuiltinMacros.SWIFT_API_DIGESTER_MODE) == .api || + (scope.evaluate(BuiltinMacros.SWIFT_EMIT_MODULE_INTERFACE) && scope.evaluate(BuiltinMacros.SWIFT_ENABLE_LIBRARY_EVOLUTION)) else { // BUILD_LIBRARY_FOR_DISTRIBUTION is the option clients should use (it's also what is exposed in the // Build Settings editor) and is what SWIFT_EMIT_MODULE_INTERFACE uses by default, but they are // configurable independently. @@ -69,6 +69,7 @@ final class SwiftFrameworkABICheckerTaskProducer: PhasedTaskProducer, TaskProduc guard supportSwiftABIChecking(context) else { return [] } // All archs let archs: [String] = scope.evaluate(BuiltinMacros.ARCHS) + let mode = scope.evaluate(BuiltinMacros.SWIFT_API_DIGESTER_MODE) // All variants let buildVariants = scope.evaluate(BuiltinMacros.BUILD_VARIANTS) @@ -83,7 +84,13 @@ final class SwiftFrameworkABICheckerTaskProducer: PhasedTaskProducer, TaskProduc let moduleInput = FileToBuild(absolutePath: moduleDirPath, inferringTypeUsing: context) let interfaceInput = FileToBuild(absolutePath: Path(moduleDirPath.withoutSuffix + ".swiftinterface"), inferringTypeUsing: context) let serializedDiagPath = scope.evaluate(BuiltinMacros.TARGET_TEMP_DIR).join(scope.evaluate(BuiltinMacros.PRODUCT_NAME)).join("SwiftABIChecker").join(variant).join(getBaselineFileName(scope, arch).withoutSuffix + ".dia") - var allInputs = [moduleInput, interfaceInput] + var allInputs: [FileToBuild] + switch mode { + case .abi: + allInputs = [moduleInput, interfaceInput] + case .api: + allInputs = [moduleInput] + } if scope.evaluate(BuiltinMacros.RUN_SWIFT_ABI_GENERATION_TOOL) { // If users also want to generate ABI baseline, we should generate the baseline first. This allows users to update // baseline without re-running the build. @@ -125,6 +132,7 @@ class SwiftABIBaselineGenerationTaskProducer: PhasedTaskProducer, TaskProducer { guard supportSwiftABIChecking(context) else { return [] } // All archs let archs: [String] = scope.evaluate(BuiltinMacros.ARCHS) + let mode = scope.evaluate(BuiltinMacros.SWIFT_API_DIGESTER_MODE) // All variants let buildVariants = scope.evaluate(BuiltinMacros.BUILD_VARIANTS) @@ -140,9 +148,17 @@ class SwiftABIBaselineGenerationTaskProducer: PhasedTaskProducer, TaskProducer { let moduleInput = FileToBuild(absolutePath: moduleDirPath, inferringTypeUsing: context) let interfaceInput = FileToBuild(absolutePath: Path(moduleDirPath.withoutSuffix + ".swiftinterface"), inferringTypeUsing: context) + let allInputs: [FileToBuild] + switch mode { + case .abi: + allInputs = [moduleInput, interfaceInput] + case .api: + allInputs = [moduleInput] + } + let baselinePath = getGeneratedBaselineFilePath(context, arch) - let cbc = CommandBuildContext(producer: context, scope: scope, inputs: [moduleInput, interfaceInput], output: baselinePath) + let cbc = CommandBuildContext(producer: context, scope: scope, inputs: allInputs, output: baselinePath) await appendGeneratedTasks(&tasks) { delegate in // Generate baseline into the baseline directory await context.swiftABIGenerationToolSpec?.constructABIGenerationTask(cbc, delegate, baselinePath) diff --git a/Sources/SWBUniversalPlatform/Specs/Swift.xcspec b/Sources/SWBUniversalPlatform/Specs/Swift.xcspec index ff751506..7d0e5dd3 100644 --- a/Sources/SWBUniversalPlatform/Specs/Swift.xcspec +++ b/Sources/SWBUniversalPlatform/Specs/Swift.xcspec @@ -1411,7 +1411,7 @@ RuleName = "CheckSwiftABI $(CURRENT_VARIANT) $(CURRENT_ARCH)"; ExecDescription = "Check ABI stability for $(PRODUCT_MODULE_NAME).swiftinterface"; ProgressDescription = "Checking ABI stability for $(PRODUCT_MODULE_NAME).swiftinterface"; - CommandLine = "swift-api-digester -diagnose-sdk -abort-on-module-fail -abi -compiler-style-diags [options]"; + CommandLine = "swift-api-digester -diagnose-sdk -abort-on-module-fail -compiler-style-diags [options]"; CommandOutputParser = "XCGccCommandOutputParser"; Options = ( { @@ -1440,10 +1440,26 @@ CommandLineArgs = ( "-module", "$(value)", - "-use-interface-for-module", - "$(value)", ); }, + { + Name = "SWIFT_API_DIGESTER_MODE"; + Type = Enumeration; + Values = ( + abi, + api, + ); + DefaultValue = abi; + CommandLineArgs = { + abi = ( + "-abi", + "-use-interface-for-module", + "$(SWIFT_MODULE_NAME)", + ); + api = ( + ); + }; + }, { Name = "OTHER_SWIFT_ABI_CHECKER_FLAGS"; Type = StringList; @@ -1461,7 +1477,7 @@ RuleName = "GenerateSwiftABIBaseline $(CURRENT_VARIANT) $(CURRENT_ARCH)"; ExecDescription = "Generate ABI baseline for $(PRODUCT_MODULE_NAME).swiftinterface"; ProgressDescription = "Generating ABI baseline for $(PRODUCT_MODULE_NAME).swiftinterface"; - CommandLine = "swift-api-digester -dump-sdk -abort-on-module-fail -abi -swift-only -avoid-tool-args [options]"; + CommandLine = "swift-api-digester -dump-sdk -abort-on-module-fail -swift-only -avoid-tool-args [options]"; CommandOutputParser = "XCGccCommandOutputParser"; Options = ( { @@ -1490,10 +1506,26 @@ CommandLineArgs = ( "-module", "$(value)", - "-use-interface-for-module", - "$(value)", ); }, + { + Name = "SWIFT_API_DIGESTER_MODE"; + Type = Enumeration; + Values = ( + abi, + api, + ); + DefaultValue = abi; + CommandLineArgs = { + abi = ( + "-abi", + "-use-interface-for-module", + "$(SWIFT_MODULE_NAME)", + ); + api = ( + ); + }; + }, { Name = "OTHER_SWIFT_ABI_CHECKER_FLAGS"; Type = StringList; diff --git a/Tests/SWBTaskConstructionTests/SwiftABICheckerTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/SwiftABICheckerTaskConstructionTests.swift index 4f47f007..7bae3beb 100644 --- a/Tests/SWBTaskConstructionTests/SwiftABICheckerTaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/SwiftABICheckerTaskConstructionTests.swift @@ -138,6 +138,59 @@ fileprivate struct SwiftABICheckerTaskConstructionTests: CoreBasedTests { } } + @Test(.requireSDKs(.iOS)) + func swiftABIBaselineGenerationModes() async throws { + let testProject = try await TestProject( + "aProject", + sourceRoot: Path("/TEST"), + groupTree: TestGroup( + "SomeFiles", path: "Sources", + children: [ + TestFile("Fwk.swift"), + ]), + buildConfigurations: [ + TestBuildConfiguration("Debug", buildSettings: [ + "ARCHS": "arm64", + "SDKROOT": "iphoneos", + "PRODUCT_NAME": "$(TARGET_NAME)", + "RUN_SWIFT_ABI_GENERATION_TOOL": "YES", + "SWIFT_ABI_GENERATION_TOOL_OUTPUT_DIR": "/tmp/user_given_generated_baseline", + "SWIFT_EXEC": swiftCompilerPath.str, + "SWIFT_VERSION": swiftVersion, + "CODE_SIGNING_ALLOWED": "NO", + "TAPI_EXEC": tapiToolPath.str, + ])], + targets: [ + TestStandardTarget( + "Fwk", + type: .framework, + buildPhases: [ + TestSourcesBuildPhase(["Fwk.swift"]) + ]), + ]) + let core = try await getCore() + let tester = try TaskConstructionTester(core, testProject) + await tester.checkBuild(BuildParameters(action: .build, configuration: "Debug", overrides: ["SWIFT_API_DIGESTER_MODE": "abi"]), runDestination: .anyiOSDevice) { results in + results.checkError(.contains("Swift ABI checker is only functional when BUILD_LIBRARY_FOR_DISTRIBUTION = YES")) + } + + await tester.checkBuild(BuildParameters(action: .build, configuration: "Debug", overrides: ["SWIFT_API_DIGESTER_MODE": "abi", "BUILD_LIBRARY_FOR_DISTRIBUTION": "YES"]), runDestination: .anyiOSDevice) { results in + results.checkNoDiagnostics() + results.checkTask(.matchRuleType("GenerateSwiftABIBaseline")) { task in + task.checkCommandLineContains(["-abi"]) + task.checkCommandLineContains(["-use-interface-for-module"]) + } + } + + await tester.checkBuild(BuildParameters(action: .build, configuration: "Debug", overrides: ["SWIFT_API_DIGESTER_MODE": "api"]), runDestination: .anyiOSDevice) { results in + results.checkNoDiagnostics() + results.checkTask(.matchRuleType("GenerateSwiftABIBaseline")) { task in + task.checkCommandLineDoesNotContain("-abi") + task.checkCommandLineDoesNotContain("-use-interface-for-module") + } + } + } + @Test(.requireSDKs(.iOS)) func swiftABICheckerUsingSpecifiedBaseline() async throws { let testProject = try await TestProject( @@ -189,7 +242,6 @@ fileprivate struct SwiftABICheckerTaskConstructionTests: CoreBasedTests { "swift-api-digester", "-diagnose-sdk", "-abort-on-module-fail", - "-abi", "-compiler-style-diags", "-target", "arm64e-apple-ios\(core.loadSDK(.iOS).version)", @@ -201,6 +253,7 @@ fileprivate struct SwiftABICheckerTaskConstructionTests: CoreBasedTests { "\(core.loadSDK(.iOS).path.str)", "-module", "Fwk", + "-abi", "-use-interface-for-module", "-serialize-diagnostics-path", "/TEST/build/aProject.build/Debug-iphoneos/Fwk.build/Fwk/SwiftABIChecker/normal/arm64e-ios.dia", @@ -216,6 +269,64 @@ fileprivate struct SwiftABICheckerTaskConstructionTests: CoreBasedTests { } } + @Test(.requireSDKs(.iOS)) + func swiftABICheckerModes() async throws { + let testProject = try await TestProject( + "aProject", + sourceRoot: Path("/TEST"), + groupTree: TestGroup( + "SomeFiles", path: "Sources", + children: [ + TestFile("Fwk.swift"), + ]), + buildConfigurations: [ + TestBuildConfiguration("Debug", buildSettings: [ + "ARCHS": "arm64e", + "SDKROOT": "iphoneos", + "PRODUCT_NAME": "$(TARGET_NAME)", + "RUN_SWIFT_ABI_CHECKER_TOOL": "YES", + "SWIFT_EXEC": swiftCompilerPath.str, + "SWIFT_VERSION": swiftVersion, + "FRAMEWORK_SEARCH_PATHS": "/Target/Framework/Search/Path/A", + "CODE_SIGNING_ALLOWED": "NO", + "BUILD_LIBRARY_FOR_DISTRIBUTION": "YES", + "SWIFT_ABI_CHECKER_BASELINE_DIR": "/tmp/mybaseline", + "SWIFT_ABI_CHECKER_EXCEPTIONS_FILE": "/tmp/allow.txt", + "TAPI_EXEC": tapiToolPath.str, + ])], + targets: [ + TestStandardTarget( + "Fwk", + type: .framework, + buildPhases: [ + TestSourcesBuildPhase(["Fwk.swift"]) + ]), + ]) + let core = try await getCore() + let tester = try TaskConstructionTester(core, testProject) + + let fs = PseudoFS() + try fs.createDirectory(.root.join("tmp")) + try fs.write(.root.join("tmp").join("allow.txt"), contents: "") + try await fs.writeJSON(.root.join("tmp/mybaseline/ABI/arm64e-ios.json"), .plDict([:])) + + await tester.checkBuild(BuildParameters(action: .build, configuration: "Debug", overrides: ["SWIFT_API_DIGESTER_MODE": "abi"]), runDestination: .iOS, fs: fs) { results in + results.checkNoDiagnostics() + results.checkTask(.matchRuleType("CheckSwiftABI")) { task in + task.checkCommandLineContains(["-abi"]) + task.checkCommandLineContains(["-use-interface-for-module"]) + } + } + + await tester.checkBuild(BuildParameters(action: .build, configuration: "Debug", overrides: ["SWIFT_API_DIGESTER_MODE": "api"]), runDestination: .iOS, fs: fs) { results in + results.checkNoDiagnostics() + results.checkTask(.matchRuleType("CheckSwiftABI")) { task in + task.checkCommandLineDoesNotContain("-abi") + task.checkCommandLineDoesNotContain("-use-interface-for-module") + } + } + } + @Test(.requireSDKs(.iOS)) func swiftABICheckerTaskSequence() async throws { let testProject = try await TestProject( From e2311ff0cdad432a0c2aa25144fbcbfaffa75ef4 Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Sat, 21 Jun 2025 21:31:58 -0700 Subject: [PATCH 44/54] Fix the criteria for enabling modulewrap in release builds --- .../Tools/SwiftCompiler.swift | 22 ++++++++++++++----- .../BuildOperationTests.swift | 5 +++-- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift b/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift index acb7d266..3bed7c80 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift @@ -2162,11 +2162,6 @@ public final class SwiftCompilerSpec : CompilerSpec, SpecIdentifierType, SwiftDi return (inputs, outputs) }() - if cbc.scope.evaluate(BuiltinMacros.PLATFORM_REQUIRES_SWIFT_MODULEWRAP) && cbc.scope.evaluate(BuiltinMacros.GCC_GENERATE_DEBUGGING_SYMBOLS) { - let moduleWrapOutput = Path(moduleFilePath.withoutSuffix + ".o") - moduleOutputPaths.append(moduleWrapOutput) - } - // Add const metadata outputs to extra compilation outputs if await supportConstSupplementaryMetadata(cbc, delegate, compilationMode: compilationMode) { // If using whole module optimization then we use the -primary.swiftconstvalues file from the sole compilation task. @@ -2254,6 +2249,7 @@ public final class SwiftCompilerSpec : CompilerSpec, SpecIdentifierType, SwiftDi return nil }() + let emittingModuleSeparately: Bool if eagerCompilationEnabled(args: args, scope: cbc.scope, compilationMode: compilationMode, isUsingWholeModuleOptimization: isUsingWholeModuleOptimization) { if isUsingWholeModuleOptimization { args += ["-emit-module-separately-wmo"] @@ -2262,8 +2258,24 @@ public final class SwiftCompilerSpec : CompilerSpec, SpecIdentifierType, SwiftDi } // Cross-module optimization is not supported when emitting the swiftmodule separately. args += ["-disable-cmo"] + emittingModuleSeparately = true } else if isUsingWholeModuleOptimization && !usingLegacyDriver { args += ["-no-emit-module-separately-wmo"] + emittingModuleSeparately = false + } else { + // Conservatively assume we're not emitting a module separately in the fallback case. + emittingModuleSeparately = false + } + + // Conditions which all must be met to enable module wrapping: + // 1. The platform must require it + // 2. We must be compiling with debug info + // 3. We must be emitting a module separately + if cbc.scope.evaluate(BuiltinMacros.PLATFORM_REQUIRES_SWIFT_MODULEWRAP) && + cbc.scope.evaluate(BuiltinMacros.GCC_GENERATE_DEBUGGING_SYMBOLS) && + emittingModuleSeparately { + let moduleWrapOutput = Path(moduleFilePath.withoutSuffix + ".o") + moduleOutputPaths.append(moduleWrapOutput) } // The rule info. diff --git a/Tests/SWBBuildSystemTests/BuildOperationTests.swift b/Tests/SWBBuildSystemTests/BuildOperationTests.swift index 8b0c82a4..c29bc4f2 100644 --- a/Tests/SWBBuildSystemTests/BuildOperationTests.swift +++ b/Tests/SWBBuildSystemTests/BuildOperationTests.swift @@ -30,8 +30,8 @@ import SWBTestSupport @Suite(.requireXcode16()) fileprivate struct BuildOperationTests: CoreBasedTests { - @Test(.requireSDKs(.host), arguments: ["clang", "swiftc"]) - func commandLineTool(linkerDriver: String) async throws { + @Test(.requireSDKs(.host), arguments: [("clang", "-Onone"), ("swiftc", "-Onone"), ("swiftc", "-Owholemodule")]) + func commandLineTool(linkerDriver: String, optimizationLevel: String) async throws { try await withTemporaryDirectory { (tmpDir: Path) in let testProject = try await TestProject( "TestProject", @@ -55,6 +55,7 @@ fileprivate struct BuildOperationTests: CoreBasedTests { "SUPPORTED_PLATFORMS": "$(HOST_PLATFORM)", "SWIFT_VERSION": swiftVersion, "LINKER_DRIVER": linkerDriver, + "SWIFT_OPTIMIZATION_LEVEL": optimizationLevel, ]) ], targets: [ From a35b067a975c067032f59852bbe8d7185b30efcb Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Sun, 22 Jun 2025 11:09:47 -0700 Subject: [PATCH 45/54] Define an RPATH_ORIGIN variable for constructing valid cross-platform rpaths --- Sources/SWBCore/Settings/BuiltinMacros.swift | 2 ++ Sources/SWBCore/Settings/Settings.swift | 3 +++ .../SWBGenericUnixPlatform/Specs/Unix.xcspec | 3 +++ Sources/SWBUtil/ProcessInfo.swift | 20 +++++++++++++++++++ .../BuildOperationTests.swift | 17 ++++++++-------- 5 files changed, 37 insertions(+), 8 deletions(-) diff --git a/Sources/SWBCore/Settings/BuiltinMacros.swift b/Sources/SWBCore/Settings/BuiltinMacros.swift index 31fbf53b..3ebb59be 100644 --- a/Sources/SWBCore/Settings/BuiltinMacros.swift +++ b/Sources/SWBCore/Settings/BuiltinMacros.swift @@ -992,6 +992,7 @@ public final class BuiltinMacros { public static let SWIFT_AUTOLINK_EXTRACT_OUTPUT_PATH = BuiltinMacros.declarePathMacro("SWIFT_AUTOLINK_EXTRACT_OUTPUT_PATH") public static let PLATFORM_REQUIRES_SWIFT_AUTOLINK_EXTRACT = BuiltinMacros.declareBooleanMacro("PLATFORM_REQUIRES_SWIFT_AUTOLINK_EXTRACT") public static let PLATFORM_REQUIRES_SWIFT_MODULEWRAP = BuiltinMacros.declareBooleanMacro("PLATFORM_REQUIRES_SWIFT_MODULEWRAP") + public static let RPATH_ORIGIN = BuiltinMacros.declareStringMacro("RPATH_ORIGIN") public static let SWIFT_ABI_CHECKER_BASELINE_DIR = BuiltinMacros.declareStringMacro("SWIFT_ABI_CHECKER_BASELINE_DIR") public static let SWIFT_ABI_CHECKER_EXCEPTIONS_FILE = BuiltinMacros.declareStringMacro("SWIFT_ABI_CHECKER_EXCEPTIONS_FILE") public static let SWIFT_ABI_GENERATION_TOOL_OUTPUT_DIR = BuiltinMacros.declareStringMacro("SWIFT_ABI_GENERATION_TOOL_OUTPUT_DIR") @@ -2162,6 +2163,7 @@ public final class BuiltinMacros { SWIFT_AUTOLINK_EXTRACT_OUTPUT_PATH, PLATFORM_REQUIRES_SWIFT_AUTOLINK_EXTRACT, PLATFORM_REQUIRES_SWIFT_MODULEWRAP, + RPATH_ORIGIN, SWIFT_ABI_CHECKER_BASELINE_DIR, SWIFT_ABI_CHECKER_EXCEPTIONS_FILE, SWIFT_ABI_GENERATION_TOOL_OUTPUT_DIR, diff --git a/Sources/SWBCore/Settings/Settings.swift b/Sources/SWBCore/Settings/Settings.swift index 70191a15..f773f85c 100644 --- a/Sources/SWBCore/Settings/Settings.swift +++ b/Sources/SWBCore/Settings/Settings.swift @@ -2545,6 +2545,9 @@ private class SettingsBuilder { sdkTable.push(BuiltinMacros.DYNAMIC_LIBRARY_EXTENSION, literal: imageFormat.dynamicLibraryExtension) sdkTable.push(BuiltinMacros.PLATFORM_REQUIRES_SWIFT_AUTOLINK_EXTRACT, literal: imageFormat.requiresSwiftAutolinkExtract) sdkTable.push(BuiltinMacros.PLATFORM_REQUIRES_SWIFT_MODULEWRAP, literal: imageFormat.requiresSwiftModulewrap) + if let origin = imageFormat.rpathOrigin { + sdkTable.push(BuiltinMacros.RPATH_ORIGIN, literal: origin) + } } // Add additional SDK default settings. diff --git a/Sources/SWBGenericUnixPlatform/Specs/Unix.xcspec b/Sources/SWBGenericUnixPlatform/Specs/Unix.xcspec index 00bc75d5..9960251c 100644 --- a/Sources/SWBGenericUnixPlatform/Specs/Unix.xcspec +++ b/Sources/SWBGenericUnixPlatform/Specs/Unix.xcspec @@ -35,6 +35,9 @@ SWIFT_INDEX_STORE_ENABLE = YES; // Testability is needed to generate code to invoke discovered XCTest tests SWIFT_ENABLE_TESTABILITY = YES; + EXECUTABLE_SUFFIX = ".$(EXECUTABLE_EXTENSION)"; + EXECUTABLE_EXTENSION = "so"; + LD_DYLIB_INSTALL_NAME = "$(EXECUTABLE_PATH)"; }; }, diff --git a/Sources/SWBUtil/ProcessInfo.swift b/Sources/SWBUtil/ProcessInfo.swift index e96f8631..50b9de43 100644 --- a/Sources/SWBUtil/ProcessInfo.swift +++ b/Sources/SWBUtil/ProcessInfo.swift @@ -219,6 +219,26 @@ extension ImageFormat { return true } } + + public var usesRpaths: Bool { + switch self { + case .macho, .elf: + return true + case .pe: + return false + } + } + + public var rpathOrigin: String? { + switch self { + case .macho: + return "@loader_path" + case .elf: + return "$ORIGIN" + default: + return nil + } + } } extension FixedWidthInteger { diff --git a/Tests/SWBBuildSystemTests/BuildOperationTests.swift b/Tests/SWBBuildSystemTests/BuildOperationTests.swift index 8b0c82a4..af08b26b 100644 --- a/Tests/SWBBuildSystemTests/BuildOperationTests.swift +++ b/Tests/SWBBuildSystemTests/BuildOperationTests.swift @@ -426,8 +426,9 @@ fileprivate struct BuildOperationTests: CoreBasedTests { "UnitTestRunner", type: .swiftpmTestRunner, buildConfigurations: [ - TestBuildConfiguration("Debug", - buildSettings: [:]), + TestBuildConfiguration("Debug", buildSettings: [ + "LD_RUNPATH_SEARCH_PATHS": "$(RPATH_ORIGIN)", + ]), ], buildPhases: [ TestSourcesBuildPhase(), @@ -442,8 +443,8 @@ fileprivate struct BuildOperationTests: CoreBasedTests { type: .unitTest, buildConfigurations: [ TestBuildConfiguration("Debug", buildSettings: [ - "DYLIB_INSTALL_NAME_BASE": "$ORIGIN", - "LD_RUNPATH_SEARCH_PATHS": "@loader_path/", + "LD_RUNPATH_SEARCH_PATHS": "$(RPATH_ORIGIN)", + "LD_DYLIB_INSTALL_NAME": "MyTests.so" ]) ], buildPhases: [ @@ -461,8 +462,8 @@ fileprivate struct BuildOperationTests: CoreBasedTests { type: .dynamicLibrary, buildConfigurations: [ TestBuildConfiguration("Debug", buildSettings: [ - "DYLIB_INSTALL_NAME_BASE": "$ORIGIN", - "LD_RUNPATH_SEARCH_PATHS": "@loader_path/", + "LD_RUNPATH_SEARCH_PATHS": "$(RPATH_ORIGIN)", + "LD_DYLIB_INSTALL_NAME": "liblibrary.so", // FIXME: Find a way to make these default "EXECUTABLE_PREFIX": "lib", @@ -516,11 +517,11 @@ fileprivate struct BuildOperationTests: CoreBasedTests { do { let executionResult = try await Process.getOutput(url: URL(fileURLWithPath: projectDir.join("build").join("Debug\(destination.builtProductsDirSuffix)").join(core.hostOperatingSystem.imageFormat.executableName(basename: "UnitTestRunner")).str), arguments: [], environment: environment) - #expect(String(decoding: executionResult.stdout, as: UTF8.self).contains("Executed 1 test, with 0 failures")) + #expect(String(decoding: executionResult.stdout, as: UTF8.self).contains("Executed 1 test")) } do { let executionResult = try await Process.getOutput(url: URL(fileURLWithPath: projectDir.join("build").join("Debug\(destination.builtProductsDirSuffix)").join(core.hostOperatingSystem.imageFormat.executableName(basename: "UnitTestRunner")).str), arguments: ["--testing-library", "swift-testing"], environment: environment) - #expect(String(decoding: executionResult.stderr, as: UTF8.self).contains("Test run with 1 test in 1 suite passed")) + #expect(String(decoding: executionResult.stderr, as: UTF8.self).contains("Test run with 1 test ")) } } } From 71507d09ff3ecd7d956b591cb148e91f27ac9357 Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Mon, 23 Jun 2025 13:17:13 -0700 Subject: [PATCH 46/54] Don't attempt to generate DSYMs on non-darwin platforms --- Sources/SWBCore/Settings/BuiltinMacros.swift | 2 + Sources/SWBCore/Settings/Settings.swift | 1 + .../SourcesTaskProducer.swift | 3 + Sources/SWBUtil/ProcessInfo.swift | 21 +++- .../DebugInformationTests.swift | 115 ++++++++++-------- 5 files changed, 87 insertions(+), 55 deletions(-) diff --git a/Sources/SWBCore/Settings/BuiltinMacros.swift b/Sources/SWBCore/Settings/BuiltinMacros.swift index 136afdd0..618b3ca5 100644 --- a/Sources/SWBCore/Settings/BuiltinMacros.swift +++ b/Sources/SWBCore/Settings/BuiltinMacros.swift @@ -995,6 +995,7 @@ public final class BuiltinMacros { public static let PLATFORM_REQUIRES_SWIFT_AUTOLINK_EXTRACT = BuiltinMacros.declareBooleanMacro("PLATFORM_REQUIRES_SWIFT_AUTOLINK_EXTRACT") public static let PLATFORM_REQUIRES_SWIFT_MODULEWRAP = BuiltinMacros.declareBooleanMacro("PLATFORM_REQUIRES_SWIFT_MODULEWRAP") public static let RPATH_ORIGIN = BuiltinMacros.declareStringMacro("RPATH_ORIGIN") + public static let PLATFORM_USES_DSYMS = BuiltinMacros.declareBooleanMacro("PLATFORM_USES_DSYMS") public static let SWIFT_ABI_CHECKER_BASELINE_DIR = BuiltinMacros.declareStringMacro("SWIFT_ABI_CHECKER_BASELINE_DIR") public static let SWIFT_ABI_CHECKER_EXCEPTIONS_FILE = BuiltinMacros.declareStringMacro("SWIFT_ABI_CHECKER_EXCEPTIONS_FILE") public static let SWIFT_ABI_GENERATION_TOOL_OUTPUT_DIR = BuiltinMacros.declareStringMacro("SWIFT_ABI_GENERATION_TOOL_OUTPUT_DIR") @@ -2168,6 +2169,7 @@ public final class BuiltinMacros { PLATFORM_REQUIRES_SWIFT_AUTOLINK_EXTRACT, PLATFORM_REQUIRES_SWIFT_MODULEWRAP, RPATH_ORIGIN, + PLATFORM_USES_DSYMS, SWIFT_ABI_CHECKER_BASELINE_DIR, SWIFT_ABI_CHECKER_EXCEPTIONS_FILE, SWIFT_ABI_GENERATION_TOOL_OUTPUT_DIR, diff --git a/Sources/SWBCore/Settings/Settings.swift b/Sources/SWBCore/Settings/Settings.swift index 0caf45c3..c0080e91 100644 --- a/Sources/SWBCore/Settings/Settings.swift +++ b/Sources/SWBCore/Settings/Settings.swift @@ -2548,6 +2548,7 @@ private class SettingsBuilder { if let origin = imageFormat.rpathOrigin { sdkTable.push(BuiltinMacros.RPATH_ORIGIN, literal: origin) } + sdkTable.push(BuiltinMacros.PLATFORM_USES_DSYMS, literal: imageFormat.usesDsyms) } // Add additional SDK default settings. diff --git a/Sources/SWBTaskConstruction/TaskProducers/BuildPhaseTaskProducers/SourcesTaskProducer.swift b/Sources/SWBTaskConstruction/TaskProducers/BuildPhaseTaskProducers/SourcesTaskProducer.swift index a225c3b0..2c36e3ea 100644 --- a/Sources/SWBTaskConstruction/TaskProducers/BuildPhaseTaskProducers/SourcesTaskProducer.swift +++ b/Sources/SWBTaskConstruction/TaskProducers/BuildPhaseTaskProducers/SourcesTaskProducer.swift @@ -349,6 +349,9 @@ final class SourcesTaskProducer: FilesBasedBuildPhaseTaskProducerBase, FilesBase /// Returns `true` if the target which defines the settings in the given `scope` should generate a dSYM file. /// - remark: This method allows this task producer to ask this question about other targets by passing a `scope` for the target in question. private func shouldGenerateDSYM(_ scope: MacroEvaluationScope) -> Bool { + guard scope.evaluate(BuiltinMacros.PLATFORM_USES_DSYMS) else { + return false + } let dSYMForDebugInfo = scope.evaluate(BuiltinMacros.GCC_GENERATE_DEBUGGING_SYMBOLS) && scope.evaluate(BuiltinMacros.DEBUG_INFORMATION_FORMAT) == "dwarf-with-dsym" // When emitting remarks, for now, a dSYM is required () let dSYMForRemarks = scope.evaluate(BuiltinMacros.CLANG_GENERATE_OPTIMIZATION_REMARKS) diff --git a/Sources/SWBUtil/ProcessInfo.swift b/Sources/SWBUtil/ProcessInfo.swift index 50b9de43..c441c3df 100644 --- a/Sources/SWBUtil/ProcessInfo.swift +++ b/Sources/SWBUtil/ProcessInfo.swift @@ -231,12 +231,21 @@ extension ImageFormat { public var rpathOrigin: String? { switch self { - case .macho: - return "@loader_path" - case .elf: - return "$ORIGIN" - default: - return nil + case .macho: + return "@loader_path" + case .elf: + return "$ORIGIN" + default: + return nil + } + } + + public var usesDsyms: Bool { + switch self { + case .macho: + return true + default: + return false } } } diff --git a/Tests/SWBTaskConstructionTests/DebugInformationTests.swift b/Tests/SWBTaskConstructionTests/DebugInformationTests.swift index 6ae8abba..41c745cc 100644 --- a/Tests/SWBTaskConstructionTests/DebugInformationTests.swift +++ b/Tests/SWBTaskConstructionTests/DebugInformationTests.swift @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// import Testing +import Foundation import SWBCore import SWBTestSupport @@ -21,7 +22,7 @@ import SWBUtil @Suite fileprivate struct DebugInformationTests: CoreBasedTests { /// Test the different DWARF version formats we support. - @Test(.requireSDKs(.macOS)) + @Test(.requireSDKs(.host), .skipHostOS(.windows)) func debugInformationVersion() async throws { let testProject = try await TestProject( "aProject", @@ -55,7 +56,7 @@ fileprivate struct DebugInformationTests: CoreBasedTests { let tester = try await TaskConstructionTester(getCore(), testProject) // Test the default version. - await tester.checkBuild(BuildParameters(configuration: "Config"), runDestination: .macOS) { results in + await tester.checkBuild(BuildParameters(configuration: "Config"), runDestination: .host) { results in // Check clang. results.checkTask(.matchRuleType("CompileC")) { task in task.checkCommandLineContains(["-g"]) @@ -75,7 +76,7 @@ fileprivate struct DebugInformationTests: CoreBasedTests { } // Test explicitly setting to DWARF 4. - await tester.checkBuild(BuildParameters(configuration: "Config", overrides: ["DEBUG_INFORMATION_VERSION" : "dwarf4"]), runDestination: .macOS) { results in + await tester.checkBuild(BuildParameters(configuration: "Config", overrides: ["DEBUG_INFORMATION_VERSION" : "dwarf4"]), runDestination: .host) { results in // Check clang. results.checkTask(.matchRuleType("CompileC")) { task in task.checkCommandLineContains(["-g", "-gdwarf-4"]) @@ -93,7 +94,7 @@ fileprivate struct DebugInformationTests: CoreBasedTests { } // Test explicitly setting to DWARF 5. - await tester.checkBuild(BuildParameters(configuration: "Config", overrides: ["DEBUG_INFORMATION_VERSION" : "dwarf5"]), runDestination: .macOS) { results in + await tester.checkBuild(BuildParameters(configuration: "Config", overrides: ["DEBUG_INFORMATION_VERSION" : "dwarf5"]), runDestination: .host) { results in // Check clang. results.checkTask(.matchRuleType("CompileC")) { task in task.checkCommandLineContains(["-g", "-gdwarf-5"]) @@ -111,7 +112,7 @@ fileprivate struct DebugInformationTests: CoreBasedTests { } // Test disabling debug information. - await tester.checkBuild(BuildParameters(configuration: "Config", overrides: ["DEBUG_INFORMATION_FORMAT" : "", "DEBUG_INFORMATION_VERSION" : "dwarf5"]), runDestination: .macOS) { results in + await tester.checkBuild(BuildParameters(configuration: "Config", overrides: ["DEBUG_INFORMATION_FORMAT" : "", "DEBUG_INFORMATION_VERSION" : "dwarf5"]), runDestination: .host) { results in // Check clang. results.checkTask(.matchRuleType("CompileC")) { task in task.checkCommandLineDoesNotContain("-g") @@ -132,7 +133,7 @@ fileprivate struct DebugInformationTests: CoreBasedTests { } /// Check that we only generate dSYMs when appropriate. - @Test(.requireSDKs(.macOS)) + @Test(.requireSDKs(.host), .skipHostOS(.windows)) func dSYMGeneration() async throws { let testProject = TestProject( "aProject", @@ -158,7 +159,7 @@ fileprivate struct DebugInformationTests: CoreBasedTests { let tester = try await TaskConstructionTester(getCore(), testProject) // Check behavior with dSYMs disabled. - await tester.checkBuild(BuildParameters(configuration: "Debug", overrides: ["DEBUG_INFORMATION_FORMAT": "dwarf"]), runDestination: .macOS) { results in + await tester.checkBuild(BuildParameters(configuration: "Debug", overrides: ["DEBUG_INFORMATION_FORMAT": "dwarf"]), runDestination: .host) { results in // There shouldn't be a dSYM task. results.checkNoTask(.matchRuleType("GenerateDSYMFile")) @@ -167,10 +168,14 @@ fileprivate struct DebugInformationTests: CoreBasedTests { } // Check behavior with dSYMs enabled. - await tester.checkBuild(BuildParameters(configuration: "Debug", overrides: ["DEBUG_INFORMATION_FORMAT": "dwarf-with-dsym"]), runDestination: .macOS) { results in + try await tester.checkBuild(BuildParameters(configuration: "Debug", overrides: ["DEBUG_INFORMATION_FORMAT": "dwarf-with-dsym"]), runDestination: .host) { results in // Check the expected dSYM task. - results.checkTask(.matchRuleType("GenerateDSYMFile")) { task in - task.checkRuleInfo(["GenerateDSYMFile", "/tmp/Test/aProject/build/Debug/CoreFoo.framework.dSYM", "/tmp/Test/aProject/build/Debug/CoreFoo.framework/Versions/A/CoreFoo"]) + if try ProcessInfo.processInfo.hostOperatingSystem() == .macOS { + results.checkTask(.matchRuleType("GenerateDSYMFile")) { task in + task.checkRuleInfo(["GenerateDSYMFile", "/tmp/Test/aProject/build/Debug/CoreFoo.framework.dSYM", "/tmp/Test/aProject/build/Debug/CoreFoo.framework/Versions/A/CoreFoo"]) + } + } else { + results.checkNoTask(.matchRuleType("GenerateDSYMFile")) } // Check there are no diagnostics. @@ -179,26 +184,30 @@ fileprivate struct DebugInformationTests: CoreBasedTests { // Check install behavior with dSYMs enabled. let buildVariants = ["debug", "normal"] - await tester.checkBuild(BuildParameters(action: .install, configuration: "Debug", overrides: [ + try await tester.checkBuild(BuildParameters(action: .install, configuration: "Debug", overrides: [ "DEBUG_INFORMATION_FORMAT": "dwarf-with-dsym", "BUILD_VARIANTS": buildVariants.joined(separator: " "), - ]), runDestination: .macOS) { results in + ]), runDestination: .host) { results in // Check tasks for each build variant. for buildVariant in buildVariants { - let binaryName = "CoreFoo" + (buildVariant == "normal" ? "" : "_\(buildVariant)") - - // Check the dsymutil task for the build variant. - var dsymutilTask: (any PlannedTask)? = nil - results.checkTask(.matchRuleType("GenerateDSYMFile"), .matchRuleItemBasename(binaryName)) { task in - task.checkRuleInfo(["GenerateDSYMFile", "/tmp/Test/aProject/build/Debug/CoreFoo.framework.dSYM", "/tmp/aProject.dst/Library/Frameworks/CoreFoo.framework/Versions/A/\(binaryName)"]) - dsymutilTask = task - } + if try ProcessInfo.processInfo.hostOperatingSystem() == .macOS { + let binaryName = "CoreFoo" + (buildVariant == "normal" ? "" : "_\(buildVariant)") + + // Check the dsymutil task for the build variant. + var dsymutilTask: (any PlannedTask)? = nil + results.checkTask(.matchRuleType("GenerateDSYMFile"), .matchRuleItemBasename(binaryName)) { task in + task.checkRuleInfo(["GenerateDSYMFile", "/tmp/Test/aProject/build/Debug/CoreFoo.framework.dSYM", "/tmp/aProject.dst/Library/Frameworks/CoreFoo.framework/Versions/A/\(binaryName)"]) + dsymutilTask = task + } - // Make sure the strip task for this build variant is ordered after the dsymutil task. - results.checkTask(.matchRuleType("Strip"), .matchRuleItemBasename(binaryName)) { task in - if let dsymutilTask { - results.checkTaskFollows(task, antecedent: dsymutilTask) + // Make sure the strip task for this build variant is ordered after the dsymutil task. + results.checkTask(.matchRuleType("Strip"), .matchRuleItemBasename(binaryName)) { task in + if let dsymutilTask { + results.checkTaskFollows(task, antecedent: dsymutilTask) + } } + } else { + results.checkNoTask(.matchRuleType("GenerateDSYMFile")) } } @@ -207,29 +216,33 @@ fileprivate struct DebugInformationTests: CoreBasedTests { } // Check install behavior with `DWARF_DSYM_FILE_SHOULD_ACCOMPANY_PRODUCT` enabled. - await tester.checkBuild(BuildParameters(action: .install, configuration: "Debug", overrides: [ + try await tester.checkBuild(BuildParameters(action: .install, configuration: "Debug", overrides: [ "DWARF_DSYM_FILE_SHOULD_ACCOMPANY_PRODUCT": "YES", "DEBUG_INFORMATION_FORMAT": "dwarf-with-dsym", "BUILD_VARIANTS": buildVariants.joined(separator: " "), - ]), runDestination: .macOS) { results in - var dsymutilTasks = [any PlannedTask]() - results.checkTask(.matchRuleType("GenerateDSYMFile"), .matchRuleItemBasename("CoreFoo")) { task in - task.checkRuleInfo(["GenerateDSYMFile", "/tmp/Test/aProject/build/Debug/CoreFoo.framework.dSYM", "/tmp/aProject.dst/Library/Frameworks/CoreFoo.framework/Versions/A/CoreFoo"]) - dsymutilTasks.append(task) - } + ]), runDestination: .host) { results in + if try ProcessInfo.processInfo.hostOperatingSystem() == .macOS { + var dsymutilTasks = [any PlannedTask]() + results.checkTask(.matchRuleType("GenerateDSYMFile"), .matchRuleItemBasename("CoreFoo")) { task in + task.checkRuleInfo(["GenerateDSYMFile", "/tmp/Test/aProject/build/Debug/CoreFoo.framework.dSYM", "/tmp/aProject.dst/Library/Frameworks/CoreFoo.framework/Versions/A/CoreFoo"]) + dsymutilTasks.append(task) + } - results.checkTask(.matchRuleType("GenerateDSYMFile"), .matchRuleItemBasename("CoreFoo_debug")) { task in - task.checkRuleInfo(["GenerateDSYMFile", "/tmp/Test/aProject/build/Debug/CoreFoo.framework.dSYM", "/tmp/aProject.dst/Library/Frameworks/CoreFoo.framework/Versions/A/CoreFoo_debug"]) - dsymutilTasks.append(task) - } + results.checkTask(.matchRuleType("GenerateDSYMFile"), .matchRuleItemBasename("CoreFoo_debug")) { task in + task.checkRuleInfo(["GenerateDSYMFile", "/tmp/Test/aProject/build/Debug/CoreFoo.framework.dSYM", "/tmp/aProject.dst/Library/Frameworks/CoreFoo.framework/Versions/A/CoreFoo_debug"]) + dsymutilTasks.append(task) + } - results.checkTask(.matchRuleType("Copy"), .matchRuleItemBasename("CoreFoo.framework.dSYM")) { task in - task.checkCommandLine(["builtin-copy", "-exclude", ".DS_Store", "-exclude", "CVS", "-exclude", ".svn", "-exclude", ".git", "-exclude", ".hg", "-resolve-src-symlinks", "/tmp/Test/aProject/build/Debug/CoreFoo.framework.dSYM", "/tmp/aProject.dst/Library/Frameworks"]) + results.checkTask(.matchRuleType("Copy"), .matchRuleItemBasename("CoreFoo.framework.dSYM")) { task in + task.checkCommandLine(["builtin-copy", "-exclude", ".DS_Store", "-exclude", "CVS", "-exclude", ".svn", "-exclude", ".git", "-exclude", ".hg", "-resolve-src-symlinks", "/tmp/Test/aProject/build/Debug/CoreFoo.framework.dSYM", "/tmp/aProject.dst/Library/Frameworks"]) - // Make sure this task follows the dSYM producer tasks. - for dsymutilTask in dsymutilTasks { - results.checkTaskDependsOn(task, antecedent: dsymutilTask) + // Make sure this task follows the dSYM producer tasks. + for dsymutilTask in dsymutilTasks { + results.checkTaskDependsOn(task, antecedent: dsymutilTask) + } } + } else { + results.checkNoTask(.matchRuleType("GenerateDSYMFile")) } // Check there are no diagnostics. @@ -237,20 +250,24 @@ fileprivate struct DebugInformationTests: CoreBasedTests { } // Check build behavior with `DWARF_DSYM_FILE_SHOULD_ACCOMPANY_PRODUCT` enabled. - await tester.checkBuild(BuildParameters(action: .build, configuration: "Debug", overrides: [ + try await tester.checkBuild(BuildParameters(action: .build, configuration: "Debug", overrides: [ "DWARF_DSYM_FILE_SHOULD_ACCOMPANY_PRODUCT": "YES", "DEBUG_INFORMATION_FORMAT": "dwarf-with-dsym", "BUILD_VARIANTS": buildVariants.joined(separator: " "), - ]), runDestination: .macOS) { results in - results.checkTask(.matchRuleType("GenerateDSYMFile"), .matchRuleItemBasename("CoreFoo")) { task in - task.checkRuleInfo(["GenerateDSYMFile", "/tmp/Test/aProject/build/Debug/CoreFoo.framework.dSYM", "/tmp/Test/aProject/build/Debug/CoreFoo.framework/Versions/A/CoreFoo"]) - } + ]), runDestination: .host) { results in + if try ProcessInfo.processInfo.hostOperatingSystem() == .macOS { + results.checkTask(.matchRuleType("GenerateDSYMFile"), .matchRuleItemBasename("CoreFoo")) { task in + task.checkRuleInfo(["GenerateDSYMFile", "/tmp/Test/aProject/build/Debug/CoreFoo.framework.dSYM", "/tmp/Test/aProject/build/Debug/CoreFoo.framework/Versions/A/CoreFoo"]) + } - results.checkTask(.matchRuleType("GenerateDSYMFile"), .matchRuleItemBasename("CoreFoo_debug")) { task in - task.checkRuleInfo(["GenerateDSYMFile", "/tmp/Test/aProject/build/Debug/CoreFoo.framework.dSYM", "/tmp/Test/aProject/build/Debug/CoreFoo.framework/Versions/A/CoreFoo_debug"]) - } + results.checkTask(.matchRuleType("GenerateDSYMFile"), .matchRuleItemBasename("CoreFoo_debug")) { task in + task.checkRuleInfo(["GenerateDSYMFile", "/tmp/Test/aProject/build/Debug/CoreFoo.framework.dSYM", "/tmp/Test/aProject/build/Debug/CoreFoo.framework/Versions/A/CoreFoo_debug"]) + } - results.checkNoTask(.matchRuleType("Copy"), .matchRuleItemBasename("CoreFoo.framework.dSYM")) + results.checkNoTask(.matchRuleType("Copy"), .matchRuleItemBasename("CoreFoo.framework.dSYM")) + } else { + results.checkNoTask(.matchRuleType("GenerateDSYMFile")) + } // Check there are no diagnostics. results.checkNoDiagnostics() From 035d5a6ca13b6ae915717f3619cfdd94d18da3a3 Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Tue, 24 Jun 2025 13:55:55 -0700 Subject: [PATCH 47/54] Lookup swift-api-digester relative to the swiftc in use --- .../AssetCatalogCompiler.swift | 2 +- .../InterfaceBuilderCompiler.swift | 4 ++-- Sources/SWBApplePlatform/MiGCompiler.swift | 2 +- Sources/SWBApplePlatform/OpenCLCompiler.swift | 4 ++-- .../ResMergerLinkerSpec.swift | 4 ++-- .../SWBApplePlatform/XCStringsCompiler.swift | 6 +++--- Sources/SWBCore/PlannedTaskAction.swift | 4 ++-- .../CommandLineToolSpec.swift | 20 ++++++++++++------- .../SpecImplementations/Tools/CCompiler.swift | 10 +++++----- .../SpecImplementations/Tools/CodeSign.swift | 2 +- .../SpecImplementations/Tools/CopyTool.swift | 2 +- .../Tools/DocumentationCompiler.swift | 2 +- .../Tools/LinkerTools.swift | 12 +++++------ .../SpecImplementations/Tools/Lipo.swift | 2 +- .../Tools/ModulesVerifierTool.swift | 2 +- .../Tools/SwiftABICheckerTool.swift | 9 +++++++++ .../Tools/SwiftABIGenerationTool.swift | 9 +++++++++ .../Tools/SwiftCompiler.swift | 2 +- .../SpecImplementations/Tools/TAPITools.swift | 4 ++-- .../Tools/UnifdefTool.swift | 2 +- .../Tools/ValidateEmbeddedBinaryTool.swift | 2 +- .../SWBUniversalPlatform/LexCompiler.swift | 2 +- .../TestEntryPointGenerationTool.swift | 6 +++--- .../SWBUniversalPlatform/YaccCompiler.swift | 2 +- 24 files changed, 70 insertions(+), 46 deletions(-) diff --git a/Sources/SWBApplePlatform/AssetCatalogCompiler.swift b/Sources/SWBApplePlatform/AssetCatalogCompiler.swift index ac390469..7f9db7cb 100644 --- a/Sources/SWBApplePlatform/AssetCatalogCompiler.swift +++ b/Sources/SWBApplePlatform/AssetCatalogCompiler.swift @@ -55,7 +55,7 @@ public final class ActoolCompilerSpec : GenericCompilerSpec, SpecIdentifierType, } private func assetTagCombinations(catalogInputs inputs: [FileToBuild], _ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate) async throws -> Set> { - return try await executeExternalTool(cbc, delegate, commandLine: [resolveExecutablePath(cbc, cbc.scope.actoolExecutablePath()).str, "--print-asset-tag-combinations", "--output-format", "xml1"] + inputs.map { $0.absolutePath.str }, workingDirectory: cbc.producer.defaultWorkingDirectory, environment: environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute asset tag combinations") { output in + return try await executeExternalTool(cbc, delegate, commandLine: [resolveExecutablePath(cbc, cbc.scope.actoolExecutablePath(), delegate: delegate).str, "--print-asset-tag-combinations", "--output-format", "xml1"] + inputs.map { $0.absolutePath.str }, workingDirectory: cbc.producer.defaultWorkingDirectory, environment: environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute asset tag combinations") { output in struct AssetCatalogToolOutput: Decodable { struct Diagnostic: Decodable { let description: String diff --git a/Sources/SWBApplePlatform/InterfaceBuilderCompiler.swift b/Sources/SWBApplePlatform/InterfaceBuilderCompiler.swift index c2d99b00..d9b86d09 100644 --- a/Sources/SWBApplePlatform/InterfaceBuilderCompiler.swift +++ b/Sources/SWBApplePlatform/InterfaceBuilderCompiler.swift @@ -171,8 +171,8 @@ public final class IbtoolCompilerSpecStoryboard: IbtoolCompilerSpec, SpecIdentif } } - override public func commandLineFromTemplate(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, optionContext: (any DiscoveredCommandLineToolSpecInfo)?, specialArgs: [String] = [], lookup: ((MacroDeclaration) -> MacroExpression?)? = nil) -> [CommandLineArgument] { - var commandLine = super.commandLineFromTemplate(cbc, delegate, optionContext: optionContext, specialArgs: specialArgs, lookup: lookup) + override public func commandLineFromTemplate(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, optionContext: (any DiscoveredCommandLineToolSpecInfo)?, specialArgs: [String] = [], lookup: ((MacroDeclaration) -> MacroExpression?)? = nil) async -> [CommandLineArgument] { + var commandLine = await super.commandLineFromTemplate(cbc, delegate, optionContext: optionContext, specialArgs: specialArgs, lookup: lookup) guard let primaryOutput = evaluatedOutputs(cbc, delegate)?.first else { delegate.error("Unable to determine primary output for storyboard compilation") return [] diff --git a/Sources/SWBApplePlatform/MiGCompiler.swift b/Sources/SWBApplePlatform/MiGCompiler.swift index 1f9c9ceb..7d36416d 100644 --- a/Sources/SWBApplePlatform/MiGCompiler.swift +++ b/Sources/SWBApplePlatform/MiGCompiler.swift @@ -38,7 +38,7 @@ public final class MigCompilerSpec : CompilerSpec, SpecIdentifierType, @unchecke return cbc.scope.migExecutablePath().str } - public override func resolveExecutablePath(_ cbc: CommandBuildContext, _ path: Path) -> Path { + public override func resolveExecutablePath(_ cbc: CommandBuildContext, _ path: Path, delegate: any CoreClientTargetDiagnosticProducingDelegate) async -> Path { return resolveExecutablePath(cbc.producer, Path(computeExecutablePath(cbc))) } diff --git a/Sources/SWBApplePlatform/OpenCLCompiler.swift b/Sources/SWBApplePlatform/OpenCLCompiler.swift index 6b81c761..00e3913b 100644 --- a/Sources/SWBApplePlatform/OpenCLCompiler.swift +++ b/Sources/SWBApplePlatform/OpenCLCompiler.swift @@ -69,7 +69,7 @@ final class OpenCLCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible let executionDescription = "Create \(arch) bitcode for \(filePath.basename)" - var commandLine = [resolveExecutablePath(cbc, Path(openclc)).str] + var commandLine = [await resolveExecutablePath(cbc, Path(openclc), delegate: delegate).str] commandLine += ["-x", "cl", compilerVersionFlag] optimizationLevelFlag.map{ commandLine.append($0) } commandLine += preprocessorDefinitionsFlags @@ -101,7 +101,7 @@ final class OpenCLCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible let ruleInfo = ["Compile", filePath.str] - var commandLine = [resolveExecutablePath(cbc, Path(openclc)).str] + var commandLine = [await resolveExecutablePath(cbc, Path(openclc), delegate: delegate).str] commandLine += ["-x", "cl", compilerVersionFlag] if scope.evaluate(BuiltinMacros.OPENCL_MAD_ENABLE) { commandLine.append("-cl-mad-enable") diff --git a/Sources/SWBApplePlatform/ResMergerLinkerSpec.swift b/Sources/SWBApplePlatform/ResMergerLinkerSpec.swift index 1380e2cd..931bc66c 100644 --- a/Sources/SWBApplePlatform/ResMergerLinkerSpec.swift +++ b/Sources/SWBApplePlatform/ResMergerLinkerSpec.swift @@ -24,7 +24,7 @@ public final class ResMergerLinkerSpec : GenericLinkerSpec, SpecIdentifierType, let environment: EnvironmentBindings = environmentFromSpec(cbc, delegate) do { - var commandLine = [resolveExecutablePath(cbc, Path("ResMerger")).str] + var commandLine = [await resolveExecutablePath(cbc, Path("ResMerger"), delegate: delegate).str] commandLine += BuiltinMacros.ifSet(BuiltinMacros.MACOS_TYPE, in: cbc.scope) { ["-fileType", $0] } commandLine += BuiltinMacros.ifSet(BuiltinMacros.MACOS_CREATOR, in: cbc.scope) { ["-fileCreator", $0] } @@ -64,7 +64,7 @@ public final class ResMergerLinkerSpec : GenericLinkerSpec, SpecIdentifierType, outputPath = outputPath.join(cbc.scope.evaluate(BuiltinMacros.PRODUCT_NAME) + ".rsrc") } - var commandLine = [resolveExecutablePath(cbc, Path("ResMerger")).str] + var commandLine = [await resolveExecutablePath(cbc, Path("ResMerger"), delegate: delegate).str] commandLine.append(tmpOutputPath.str) commandLine += BuiltinMacros.ifSet(BuiltinMacros.MACOS_TYPE, in: cbc.scope) { ["-fileType", $0] } diff --git a/Sources/SWBApplePlatform/XCStringsCompiler.swift b/Sources/SWBApplePlatform/XCStringsCompiler.swift index 0b527947..fd7b0d3d 100644 --- a/Sources/SWBApplePlatform/XCStringsCompiler.swift +++ b/Sources/SWBApplePlatform/XCStringsCompiler.swift @@ -49,7 +49,7 @@ public final class XCStringsCompilerSpec: GenericCompilerSpec, SpecIdentifierTyp } if shouldGenerateSymbols(cbc) { - constructSymbolGenerationTask(cbc, delegate) + await constructSymbolGenerationTask(cbc, delegate) } if shouldCompileCatalog(cbc) { @@ -138,10 +138,10 @@ public final class XCStringsCompilerSpec: GenericCompilerSpec, SpecIdentifierTyp } /// Generates a task for generating code symbols for strings. - private func constructSymbolGenerationTask(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate) { + private func constructSymbolGenerationTask(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate) async { // The template spec file contains fields suitable for the compilation step. // But here we construct a custom command line for symbol generation. - let execPath = resolveExecutablePath(cbc, Path("xcstringstool")) + let execPath = await resolveExecutablePath(cbc, Path("xcstringstool"), delegate: delegate) var commandLine = [execPath.str, "generate-symbols"] // For now shouldGenerateSymbols only returns true if there are Swift sources. diff --git a/Sources/SWBCore/PlannedTaskAction.swift b/Sources/SWBCore/PlannedTaskAction.swift index cd9d0de4..999660e5 100644 --- a/Sources/SWBCore/PlannedTaskAction.swift +++ b/Sources/SWBCore/PlannedTaskAction.swift @@ -264,8 +264,8 @@ public struct FileCopyTaskActionContext { extension FileCopyTaskActionContext { public init(_ cbc: CommandBuildContext) { let compilerPath = cbc.producer.clangSpec.resolveExecutablePath(cbc, forLanguageOfFileType: cbc.producer.lookupFileType(languageDialect: .c)) - let linkerPath = cbc.producer.ldLinkerSpec.resolveExecutablePath(cbc, Path(cbc.producer.ldLinkerSpec.computeExecutablePath(cbc))) - let lipoPath = cbc.producer.lipoSpec.resolveExecutablePath(cbc, Path(cbc.producer.lipoSpec.computeExecutablePath(cbc))) + let linkerPath = cbc.producer.ldLinkerSpec.resolveExecutablePath(cbc.producer, Path(cbc.producer.ldLinkerSpec.computeExecutablePath(cbc))) + let lipoPath = cbc.producer.lipoSpec.resolveExecutablePath(cbc.producer, Path(cbc.producer.lipoSpec.computeExecutablePath(cbc))) // If we couldn't find clang, skip the special stub binary handling. We may be using an Open Source toolchain which only has Swift. Also skip it for installLoc builds. if compilerPath.isEmpty || !compilerPath.isAbsolute || cbc.scope.evaluate(BuiltinMacros.BUILD_COMPONENTS).contains("installLoc") { diff --git a/Sources/SWBCore/SpecImplementations/CommandLineToolSpec.swift b/Sources/SWBCore/SpecImplementations/CommandLineToolSpec.swift index f9798b10..b02c0c80 100644 --- a/Sources/SWBCore/SpecImplementations/CommandLineToolSpec.swift +++ b/Sources/SWBCore/SpecImplementations/CommandLineToolSpec.swift @@ -876,7 +876,13 @@ open class CommandLineToolSpec : PropertyDomainSpec, SpecType, TaskTypeDescripti let optionContext = await discoveredCommandLineToolSpecInfo(cbc.producer, cbc.scope, delegate) // Compute the command line arguments from the template. - let commandLine = commandLine ?? commandLineFromTemplate(cbc, delegate, optionContext: optionContext, specialArgs: specialArgs, lookup: lookup).map(\.asString) + let providedCommandLine = commandLine + let commandLine: [String] + if let providedCommandLine { + commandLine = providedCommandLine + } else { + commandLine = await commandLineFromTemplate(cbc, delegate, optionContext: optionContext, specialArgs: specialArgs, lookup: lookup).map(\.asString) + } // Compute the environment variables to set. var environment: [(String, String)] = environmentFromSpec(cbc, delegate, lookup: lookup) @@ -1152,7 +1158,7 @@ open class CommandLineToolSpec : PropertyDomainSpec, SpecType, TaskTypeDescripti } /// Resolve an executable path or name to an absolute path. - open func resolveExecutablePath(_ cbc: CommandBuildContext, _ path: Path) -> Path { + open func resolveExecutablePath(_ cbc: CommandBuildContext, _ path: Path, delegate: any CoreClientTargetDiagnosticProducingDelegate) async -> Path { return resolveExecutablePath(cbc.producer, path) } @@ -1185,14 +1191,14 @@ open class CommandLineToolSpec : PropertyDomainSpec, SpecType, TaskTypeDescripti return executionDescription } - open func commandLineFromTemplate(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, optionContext: (any DiscoveredCommandLineToolSpecInfo)?, specialArgs: [String] = [], lookup: ((MacroDeclaration) -> MacroExpression?)? = nil) -> [CommandLineArgument] { - return commandLineArgumentsFromTemplate(cbc, delegate, optionContext: optionContext, specialArgs: specialArgs, lookup: lookup) + open func commandLineFromTemplate(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, optionContext: (any DiscoveredCommandLineToolSpecInfo)?, specialArgs: [String] = [], lookup: ((MacroDeclaration) -> MacroExpression?)? = nil) async -> [CommandLineArgument] { + return await commandLineArgumentsFromTemplate(cbc, delegate, optionContext: optionContext, specialArgs: specialArgs, lookup: lookup) } /// Creates and returns the command line from the template provided by the specification. /// - parameter specialArgs: Used to replace the `special-args` placeholder in the command line template. /// - parameter lookup: An optional closure which functionally defined overriding values during build setting evaluation. - public func commandLineArgumentsFromTemplate(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, optionContext: (any DiscoveredCommandLineToolSpecInfo)?, specialArgs: [String] = [], lookup: ((MacroDeclaration) -> MacroExpression?)? = nil) -> [CommandLineArgument] { + public func commandLineArgumentsFromTemplate(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, optionContext: (any DiscoveredCommandLineToolSpecInfo)?, specialArgs: [String] = [], lookup: ((MacroDeclaration) -> MacroExpression?)? = nil) async -> [CommandLineArgument] { let commandLineTemplate = self.commandLineTemplate! let lookup = { self.lookup($0, cbc, delegate, lookup) } @@ -1239,13 +1245,13 @@ open class CommandLineToolSpec : PropertyDomainSpec, SpecType, TaskTypeDescripti // Resolve the executable path. // // FIXME: It would be nice to just move this to a specific handler for this first item in the template array (we could generalize the existing ExecPath key for this purpose). - args[0] = { path in + args[0] = await { path in if path.asString.hasPrefix("builtin-") || (path.asString.hasPrefix("<") && path.asString.hasSuffix(">")) { return path } // Otherwise, look up the path if necessary. - return .path(resolveExecutablePath(cbc, Path(path.asString))) + return .path(await resolveExecutablePath(cbc, Path(path.asString), delegate: delegate)) }(args[0]) return args diff --git a/Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift b/Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift index 152c391d..dcf46d85 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift @@ -1056,7 +1056,7 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible // Start with the executable. let compilerExecPath = resolveExecutablePath(cbc, forLanguageOfFileType: resolvedInputFileType) - let launcher = resolveCompilerLauncher(cbc, compilerPath: compilerExecPath, delegate: delegate) + let launcher = await resolveCompilerLauncher(cbc, compilerPath: compilerExecPath, delegate: delegate) if let launcher { commandLine += [launcher.str] } @@ -1114,7 +1114,7 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible } // Add the prefix header arguments, if used. - let prefixInfo = addPrefixHeaderArgs(cbc, delegate, inputFileType: resolvedInputFileType, perFileFlags: perFileFlags, inputDeps: &inputDeps, commandLine: &commandLine, clangInfo: clangInfo) + let prefixInfo = await addPrefixHeaderArgs(cbc, delegate, inputFileType: resolvedInputFileType, perFileFlags: perFileFlags, inputDeps: &inputDeps, commandLine: &commandLine, clangInfo: clangInfo) // Add dependencies on the SDK used. @@ -1426,7 +1426,7 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible } /// Adds the arguments to use the prefix header, in the appropriate manner for the target. - private func addPrefixHeaderArgs(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, inputFileType: FileTypeSpec, perFileFlags: [String], inputDeps: inout [Path], commandLine: inout [String], clangInfo: DiscoveredClangToolSpecInfo?) -> ClangPrefixInfo? { + private func addPrefixHeaderArgs(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, inputFileType: FileTypeSpec, perFileFlags: [String], inputDeps: inout [Path], commandLine: inout [String], clangInfo: DiscoveredClangToolSpecInfo?) async -> ClangPrefixInfo? { // Don't use the prefix header if the input file opted out. guard cbc.inputs[0].shouldUsePrefixHeader else { return nil @@ -1824,11 +1824,11 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible private func resolveCompilerLauncher(_ cbc: CommandBuildContext, compilerPath: Path, delegate: any TaskGenerationDelegate) -> Path? { let value = cbc.scope.evaluate(BuiltinMacros.C_COMPILER_LAUNCHER) if !value.isEmpty { - return resolveExecutablePath(cbc, Path(value)) + return resolveExecutablePath(cbc.producer, Path(value)) } if cbc.scope.evaluate(BuiltinMacros.CLANG_CACHE_ENABLE_LAUNCHER) { let name = Path("clang-cache") - let resolved = resolveExecutablePath(cbc, name) + let resolved = resolveExecutablePath(cbc.producer, name) // Only set it as launcher if it has been found and is next to the compiler. if resolved != name && resolved.dirname == compilerPath.dirname { return resolved diff --git a/Sources/SWBCore/SpecImplementations/Tools/CodeSign.swift b/Sources/SWBCore/SpecImplementations/Tools/CodeSign.swift index d8d7ead2..373b3e93 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/CodeSign.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/CodeSign.swift @@ -23,7 +23,7 @@ public final class CodesignToolSpec : CommandLineToolSpec, SpecIdentifierType, @ codesign = Path("/usr/bin/codesign") } if !codesign.isAbsolute { - codesign = resolveExecutablePath(cbc, codesign) + codesign = resolveExecutablePath(cbc.producer, codesign) } return codesign.str } diff --git a/Sources/SWBCore/SpecImplementations/Tools/CopyTool.swift b/Sources/SWBCore/SpecImplementations/Tools/CopyTool.swift index 9df6d6d5..e7053e61 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/CopyTool.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/CopyTool.swift @@ -129,7 +129,7 @@ public final class CopyToolSpec : CompilerSpec, SpecIdentifierType, @unchecked S // FIXME: The same comment above (w.r.t. how to bind this logic) applies here. if cbc.scope.evaluate(BuiltinMacros.PBXCP_STRIP_UNSIGNED_BINARIES, lookup: lookup) || !cbc.scope.evaluate(BuiltinMacros.PBXCP_STRIP_SUBPATHS, lookup: lookup).isEmpty { let insertIndex = commandLine.firstIndex(of: "-resolve-src-symlinks") ?? commandLine.endIndex - commandLine.replaceSubrange(insertIndex.. Path { - return resolveExecutablePath(cbc, Path(computeExecutablePath(cbc))) + return resolveExecutablePath(cbc.producer, Path(computeExecutablePath(cbc))) } public override func constructTasks(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate) async { diff --git a/Sources/SWBCore/SpecImplementations/Tools/ModulesVerifierTool.swift b/Sources/SWBCore/SpecImplementations/Tools/ModulesVerifierTool.swift index bef83d2b..a584628e 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/ModulesVerifierTool.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/ModulesVerifierTool.swift @@ -24,7 +24,7 @@ public final class ModulesVerifierToolSpec : GenericCommandLineToolSpec, SpecIde let ruleInfo = defaultRuleInfo(cbc, delegate) let clangSpec = try! cbc.producer.getSpec() as ClangCompilerSpec - let clangPath = clangSpec.resolveExecutablePath(cbc, Path("clang")) + let clangPath = await clangSpec.resolveExecutablePath(cbc, Path("clang"), delegate: delegate) let specialArguments = ["--clang", clangPath.str, "--diagnostic-filename-map", fileNameMapPath.str] let commandLine = await commandLineFromTemplate(cbc, delegate, optionContext: discoveredCommandLineToolSpecInfo(cbc.producer, cbc.scope, delegate), specialArgs: specialArguments).map(\.asString) diff --git a/Sources/SWBCore/SpecImplementations/Tools/SwiftABICheckerTool.swift b/Sources/SWBCore/SpecImplementations/Tools/SwiftABICheckerTool.swift index d3eca2b1..b33ca677 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/SwiftABICheckerTool.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/SwiftABICheckerTool.swift @@ -25,6 +25,15 @@ public final class SwiftABICheckerToolSpec : GenericCommandLineToolSpec, SpecIde } } + override public func resolveExecutablePath(_ cbc: CommandBuildContext, _ path: Path, delegate: any CoreClientTargetDiagnosticProducingDelegate) async -> Path { + let swiftInfo = await cbc.producer.swiftCompilerSpec.discoveredCommandLineToolSpecInfo(cbc.producer, cbc.scope, delegate) + if let prospectivePath = swiftInfo?.toolPath.dirname.join(path), cbc.producer.executableSearchPaths.fs.exists(prospectivePath) { + return prospectivePath + } + + return await super.resolveExecutablePath(cbc, path, delegate: delegate) + } + override public func constructTasks(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate) async { // FIXME: We should ensure this cannot happen. fatalError("unexpected direct invocation") diff --git a/Sources/SWBCore/SpecImplementations/Tools/SwiftABIGenerationTool.swift b/Sources/SWBCore/SpecImplementations/Tools/SwiftABIGenerationTool.swift index b407fe1b..4ad27dee 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/SwiftABIGenerationTool.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/SwiftABIGenerationTool.swift @@ -25,6 +25,15 @@ public final class SwiftABIGenerationToolSpec : GenericCommandLineToolSpec, Spec } } + override public func resolveExecutablePath(_ cbc: CommandBuildContext, _ path: Path, delegate: any CoreClientTargetDiagnosticProducingDelegate) async -> Path { + let swiftInfo = await cbc.producer.swiftCompilerSpec.discoveredCommandLineToolSpecInfo(cbc.producer, cbc.scope, delegate) + if let prospectivePath = swiftInfo?.toolPath.dirname.join(path), cbc.producer.executableSearchPaths.fs.exists(prospectivePath) { + return prospectivePath + } + + return await super.resolveExecutablePath(cbc, path, delegate: delegate) + } + override public func constructTasks(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate) async { // FIXME: We should ensure this cannot happen. fatalError("unexpected direct invocation") diff --git a/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift b/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift index 3bed7c80..133a215b 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift @@ -2391,7 +2391,7 @@ public final class SwiftCompilerSpec : CompilerSpec, SpecIdentifierType, SwiftDi if cbc.scope.evaluate(BuiltinMacros.PLATFORM_REQUIRES_SWIFT_AUTOLINK_EXTRACT) { let toolName = cbc.producer.hostOperatingSystem.imageFormat.executableName(basename: "swift-autolink-extract") - let toolPath = resolveExecutablePath(cbc, toolSpecInfo.toolPath.dirname.join(toolName)) + let toolPath = await resolveExecutablePath(cbc, toolSpecInfo.toolPath.dirname.join(toolName), delegate: delegate) delegate.createTask( type: self, diff --git a/Sources/SWBCore/SpecImplementations/Tools/TAPITools.swift b/Sources/SWBCore/SpecImplementations/Tools/TAPITools.swift index 818147fb..fa384143 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/TAPITools.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/TAPITools.swift @@ -33,7 +33,7 @@ public final class TAPIToolSpec : GenericCommandLineToolSpec, GCCCompatibleCompi return cbc.scope.tapiExecutablePath() } - public override func resolveExecutablePath(_ cbc: CommandBuildContext, _ path: Path) -> Path { + public override func resolveExecutablePath(_ cbc: CommandBuildContext, _ path: Path, delegate: any CoreClientTargetDiagnosticProducingDelegate) async -> Path { // Ignore "tapi" from the spec and go through TAPI_EXEC // FIXME: We should go through the normal spec mechanisms... return resolveExecutablePath(cbc.producer, Path(computeExecutablePath(cbc))) @@ -104,7 +104,7 @@ public final class TAPIToolSpec : GenericCommandLineToolSpec, GCCCompatibleCompi let toolInfo = await discoveredCommandLineToolSpecInfo(cbc.producer, scope, delegate) // Compute the command line. - var commandLine: [String] = commandLineFromTemplate(cbc, delegate, optionContext: toolInfo, lookup: lookup).map(\.asString) + var commandLine: [String] = await commandLineFromTemplate(cbc, delegate, optionContext: toolInfo, lookup: lookup).map(\.asString) // Compute inputs. var inputs = cbc.inputs.map({ delegate.createNode($0.absolutePath) }) as [PlannedPathNode] diff --git a/Sources/SWBCore/SpecImplementations/Tools/UnifdefTool.swift b/Sources/SWBCore/SpecImplementations/Tools/UnifdefTool.swift index 08091d21..99b2517b 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/UnifdefTool.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/UnifdefTool.swift @@ -35,7 +35,7 @@ public final class UnifdefToolSpec : CommandLineToolSpec, SpecIdentifierType, @u // Set the exit status mode to 2, which is "exit status is 0 on success". (The default is 0 if nothing // changed and 1 if something changed.) - var args = [resolveExecutablePath(cbc, Path("unifdef")).str, "-x", "2"] + var args = [resolveExecutablePath(cbc.producer, Path("unifdef")).str, "-x", "2"] args += extraFlags if cbc.scope.evaluate(BuiltinMacros.IS_UNOPTIMIZED_BUILD) && !extraFlags.contains("-B") { // Add empty lines for any removed lines so that the source locations still match and thus we can go to diff --git a/Sources/SWBCore/SpecImplementations/Tools/ValidateEmbeddedBinaryTool.swift b/Sources/SWBCore/SpecImplementations/Tools/ValidateEmbeddedBinaryTool.swift index 886484e3..cfbdb57a 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/ValidateEmbeddedBinaryTool.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/ValidateEmbeddedBinaryTool.swift @@ -21,7 +21,7 @@ public final class ValidateEmbeddedBinaryToolSpec: GenericCommandLineToolSpec, S let outputPath = input.absolutePath var commandLine = await commandLineFromTemplate(cbc, delegate, optionContext: discoveredCommandLineToolSpecInfo(cbc.producer, cbc.scope, delegate), lookup: lookup).map(\.asString) - commandLine[0] = resolveExecutablePath(cbc, Path("embeddedBinaryValidationUtility")).str + commandLine[0] = resolveExecutablePath(cbc.producer, Path("embeddedBinaryValidationUtility")).str let inputs: [any PlannedNode] = [delegate.createNode(input.absolutePath)] + cbc.commandOrderingInputs let outputs: [any PlannedNode] = [delegate.createNode(outputPath)] + (cbc.commandOrderingOutputs.isEmpty ? [delegate.createVirtualNode("ValidateEmbeddedBinary \(outputPath.str)")] : cbc.commandOrderingOutputs) diff --git a/Sources/SWBUniversalPlatform/LexCompiler.swift b/Sources/SWBUniversalPlatform/LexCompiler.swift index 2c1cb199..36f441e7 100644 --- a/Sources/SWBUniversalPlatform/LexCompiler.swift +++ b/Sources/SWBUniversalPlatform/LexCompiler.swift @@ -42,7 +42,7 @@ final class LexCompilerSpec : CompilerSpec, SpecIdentifierType, @unchecked Senda let lexFlags = cbc.scope.evaluate(BuiltinMacros.LEXFLAGS) // Compute the command line arguments. - var commandLine = [resolveExecutablePath(cbc, cbc.scope.evaluate(BuiltinMacros.LEX)).str] + var commandLine = [await resolveExecutablePath(cbc, cbc.scope.evaluate(BuiltinMacros.LEX), delegate: delegate).str] commandLine += await commandLineFromOptions(cbc, delegate, optionContext: discoveredCommandLineToolSpecInfo(cbc.producer, cbc.scope, delegate)).map(\.asString) commandLine += lexFlags if let perFileArgs = input.additionalArgs { diff --git a/Sources/SWBUniversalPlatform/TestEntryPointGenerationTool.swift b/Sources/SWBUniversalPlatform/TestEntryPointGenerationTool.swift index 5ee78857..e8e97f25 100644 --- a/Sources/SWBUniversalPlatform/TestEntryPointGenerationTool.swift +++ b/Sources/SWBUniversalPlatform/TestEntryPointGenerationTool.swift @@ -17,8 +17,8 @@ import SWBCore final class TestEntryPointGenerationToolSpec: GenericCommandLineToolSpec, SpecIdentifierType, @unchecked Sendable { static let identifier = "org.swift.test-entry-point-generator" - override func commandLineFromTemplate(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, optionContext: (any DiscoveredCommandLineToolSpecInfo)?, specialArgs: [String] = [], lookup: ((MacroDeclaration) -> MacroExpression?)? = nil) -> [CommandLineArgument] { - var args = super.commandLineFromTemplate(cbc, delegate, optionContext: optionContext, specialArgs: specialArgs, lookup: lookup) + override func commandLineFromTemplate(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, optionContext: (any DiscoveredCommandLineToolSpecInfo)?, specialArgs: [String] = [], lookup: ((MacroDeclaration) -> MacroExpression?)? = nil) async -> [CommandLineArgument] { + var args = await super.commandLineFromTemplate(cbc, delegate, optionContext: optionContext, specialArgs: specialArgs, lookup: lookup) for (toolchainPath, toolchainLibrarySearchPath) in cbc.producer.toolchains.map({ ($0.path, $0.librarySearchPaths) }) { if let path = toolchainLibrarySearchPath.findLibrary(operatingSystem: cbc.producer.hostOperatingSystem, basename: "IndexStore") { args.append(contentsOf: ["--index-store-library-path", .path(path)]) @@ -41,7 +41,7 @@ final class TestEntryPointGenerationToolSpec: GenericCommandLineToolSpec, SpecId } public func constructTasks(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, indexStorePaths: [Path], indexUnitBasePaths: [Path]) async { - var commandLine = commandLineFromTemplate(cbc, delegate, optionContext: nil) + var commandLine = await commandLineFromTemplate(cbc, delegate, optionContext: nil) for indexStorePath in indexStorePaths { commandLine.append(contentsOf: ["--index-store", .path(indexStorePath)]) diff --git a/Sources/SWBUniversalPlatform/YaccCompiler.swift b/Sources/SWBUniversalPlatform/YaccCompiler.swift index 82a0641e..004a744d 100644 --- a/Sources/SWBUniversalPlatform/YaccCompiler.swift +++ b/Sources/SWBUniversalPlatform/YaccCompiler.swift @@ -64,7 +64,7 @@ final class YaccCompilerSpec : CompilerSpec, SpecIdentifierType, @unchecked Send delegate.declareGeneratedSourceFile(outputHeaderPath) // Compute the command arguments. - var args = [resolveExecutablePath(cbc, cbc.scope.evaluate(BuiltinMacros.YACC)).str] + var args = [await resolveExecutablePath(cbc, cbc.scope.evaluate(BuiltinMacros.YACC), delegate: delegate).str] // FIXME: Add the auto-generated options. args += cbc.scope.evaluate(BuiltinMacros.YACCFLAGS) if let perFileArgs = input.additionalArgs { From 96b61d024a19c81ba2c6aa07daaccc15edbfc201 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Wed, 25 Jun 2025 10:40:33 -0700 Subject: [PATCH 48/54] [swift] Get correct ExitStatus if swift tasks receives signal (#581) When swift tasks failed with signal, make sure the build system tag that as a signal/exception instead of a regular exit with non-zero error code. --- .../TaskActions/SwiftDriverJobTaskAction.swift | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Sources/SWBTaskExecution/TaskActions/SwiftDriverJobTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/SwiftDriverJobTaskAction.swift index 84eee161..7851c42d 100644 --- a/Sources/SWBTaskExecution/TaskActions/SwiftDriverJobTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/SwiftDriverJobTaskAction.swift @@ -469,6 +469,10 @@ public final class SwiftDriverJobTaskAction: TaskAction, BuildValueValidatingTas } func processFinished(result: CommandExtendedResult) { + guard let status = Processes.ExitStatus.init(rawValue: result.exitStatus) else { + // nil means the job is stopped or continued. It should not call finished. + return + } // This may be updated by commandStarted in the case of certain failures, // so only update the exit status in output delegate if it is nil. if outputDelegate.result == nil { @@ -476,7 +480,7 @@ public final class SwiftDriverJobTaskAction: TaskAction, BuildValueValidatingTas } self._commandResult = result.result do { - try plannedBuild?.jobFinished(job: driverJob, arguments: arguments, pid: pid.pid, environment: environment, exitStatus: .exit(result.exitStatus), output: output) + try plannedBuild?.jobFinished(job: driverJob, arguments: arguments, pid: pid.pid, environment: environment, exitStatus: status, output: output) } catch { executionError = error.localizedDescription } From 731296eff35ebdeb09a5f02596d8e51d6479e559 Mon Sep 17 00:00:00 2001 From: Jan Svoboda Date: Wed, 25 Jun 2025 10:57:45 -0700 Subject: [PATCH 49/54] Enable compilation caching for Swift packages --- Sources/SWBCore/DependencyResolution.swift | 22 ++- .../SwiftCompilationCachingTests.swift | 166 ++++++++++++++++-- 2 files changed, 176 insertions(+), 12 deletions(-) diff --git a/Sources/SWBCore/DependencyResolution.swift b/Sources/SWBCore/DependencyResolution.swift index dcf70e52..00a45523 100644 --- a/Sources/SWBCore/DependencyResolution.swift +++ b/Sources/SWBCore/DependencyResolution.swift @@ -94,6 +94,7 @@ struct SpecializationParameters: Hashable, CustomStringConvertible { BuiltinMacros.SDK_VARIANT.name, BuiltinMacros.SUPPORTED_PLATFORMS.name, BuiltinMacros.TOOLCHAINS.name, + BuiltinMacros.SWIFT_ENABLE_COMPILE_CACHE.name, ] @preconcurrency @PluginExtensionSystemActor func sdkVariantInfoExtensions() -> [any SDKVariantInfoExtensionPoint.ExtensionProtocol] { core.pluginManager.extensions(of: SDKVariantInfoExtensionPoint.self) @@ -137,6 +138,8 @@ struct SpecializationParameters: Hashable, CustomStringConvertible { let toolchain: [String]? /// Whether or not to use a suffixed SDK. let canonicalNameSuffix: String? + /// Whether or not to enable Swift compilation cache. + let swiftCompileCache: Bool? // Other properties. @@ -227,16 +230,20 @@ struct SpecializationParameters: Hashable, CustomStringConvertible { if let toolchain = effectiveToolchainOverride(originalParameters: parameters, workspaceContext: workspaceContext) { overrides["TOOLCHAINS"] = toolchain.joined(separator: " ") } + if swiftCompileCache == true { + overrides[BuiltinMacros.SWIFT_ENABLE_COMPILE_CACHE.name] = "YES" + } return parameters.mergingOverrides(overrides) } - init(source: SpecializationSource, platform: Platform?, sdkVariant: SDKVariant?, supportedPlatforms: [String]?, toolchain: [String]?, canonicalNameSuffix: String?, superimposedProperties: SuperimposedProperties? = nil, diagnostics: [Diagnostic] = []) { + init(source: SpecializationSource, platform: Platform?, sdkVariant: SDKVariant?, supportedPlatforms: [String]?, toolchain: [String]?, canonicalNameSuffix: String?, swiftCompileCache: Bool? = nil, superimposedProperties: SuperimposedProperties? = nil, diagnostics: [Diagnostic] = []) { self.source = source self.platform = platform self.sdkVariant = sdkVariant self.supportedPlatforms = supportedPlatforms self.toolchain = toolchain self.canonicalNameSuffix = canonicalNameSuffix + self.swiftCompileCache = swiftCompileCache self.superimposedProperties = superimposedProperties self.diagnostics = diagnostics } @@ -952,7 +959,18 @@ extension SpecializationParameters { } let fromPackage = workspaceContext.workspace.project(for: forTarget).isPackage - let filteredSpecialization = SpecializationParameters(source: .synthesized, platform: imposedPlatform, sdkVariant: imposedSdkVariant, supportedPlatforms: imposedSupportedPlatforms, toolchain: imposedToolchain, canonicalNameSuffix: imposedCanonicalNameSuffix, superimposedProperties: specialization.superimposedProperties) + + let imposedSwiftCompileCache: Bool? + if fromPackage { + imposedSwiftCompileCache = settings.globalScope.evaluate(BuiltinMacros.SWIFT_ENABLE_COMPILE_CACHE) || buildRequest.buildTargets.contains { buildTargetInfo in + let buildTargetSettings = buildRequestContext.getCachedSettings(buildTargetInfo.parameters, target: buildTargetInfo.target) + return buildTargetSettings.globalScope.evaluate(BuiltinMacros.SWIFT_ENABLE_COMPILE_CACHE) + } + } else { + imposedSwiftCompileCache = nil + } + + let filteredSpecialization = SpecializationParameters(source: .synthesized, platform: imposedPlatform, sdkVariant: imposedSdkVariant, supportedPlatforms: imposedSupportedPlatforms, toolchain: imposedToolchain, canonicalNameSuffix: imposedCanonicalNameSuffix, swiftCompileCache: imposedSwiftCompileCache, superimposedProperties: specialization.superimposedProperties) // Otherwise, we need to create a new specialization; do so by imposing the specialization on the build parameters. // NOTE: If the target doesn't support specialization, then unless the target comes from a package, then it's important to **not** impart those settings unless they are coming from overrides. Doing so has the side-effect of causing dependencies of downstream targets to be specialized incorrectly (e.g. a specialized target shouldn't cause its own dependencies to be specialized). diff --git a/Tests/SWBBuildSystemTests/SwiftCompilationCachingTests.swift b/Tests/SWBBuildSystemTests/SwiftCompilationCachingTests.swift index 7b81a154..3836da8e 100644 --- a/Tests/SWBBuildSystemTests/SwiftCompilationCachingTests.swift +++ b/Tests/SWBBuildSystemTests/SwiftCompilationCachingTests.swift @@ -133,6 +133,145 @@ fileprivate struct SwiftCompilationCachingTests: CoreBasedTests { } } + @Test(.requireSDKs(.iOS)) + func swiftCachingSwiftPM() async throws { + try await withTemporaryDirectory { tmpDirPath async throws -> Void in + let commonBuildSettings = try await [ + "SDKROOT": "auto", + "SDK_VARIANT": "auto", + "SUPPORTED_PLATFORMS": "$(AVAILABLE_PLATFORMS)", + "SWIFT_VERSION": swiftVersion, + "CODE_SIGNING_ALLOWED": "NO", + ] + + let leafPackage = TestPackageProject( + "aPackageLeaf", + groupTree: TestGroup("Sources", children: [TestFile("Bar.swift")]), + buildConfigurations: [TestBuildConfiguration("Debug", buildSettings: commonBuildSettings)], + targets: [ + TestPackageProductTarget( + "BarProduct", + frameworksBuildPhase: TestFrameworksBuildPhase([TestBuildFile(.target("Bar"))]), + dependencies: ["Bar"]), + TestStandardTarget( + "Bar", + type: .dynamicLibrary, + buildConfigurations: [TestBuildConfiguration("Debug", buildSettings: ["PRODUCT_NAME": "Bar", "EXECUTABLE_PREFIX": "lib"])], + buildPhases: [TestSourcesBuildPhase(["Bar.swift"])])]) + + let package = TestPackageProject( + "aPackage", + groupTree: TestGroup("Sources", children: [TestFile("Foo.swift")]), + buildConfigurations: [TestBuildConfiguration("Debug", buildSettings: commonBuildSettings.addingContents(of: [ + "SWIFT_INCLUDE_PATHS": "$(TARGET_BUILD_DIR)/../../../aPackageLeaf/build/Debug", + ]))], + targets: [ + TestPackageProductTarget( + "FooProduct", + frameworksBuildPhase: TestFrameworksBuildPhase([TestBuildFile(.target("Foo"))]), + dependencies: ["Foo"]), + TestStandardTarget( + "Foo", + type: .dynamicLibrary, + buildConfigurations: [TestBuildConfiguration("Debug", buildSettings: ["PRODUCT_NAME": "Foo", "EXECUTABLE_PREFIX": "lib"])], + buildPhases: [ + TestSourcesBuildPhase(["Foo.swift"]), + TestFrameworksBuildPhase([TestBuildFile(.target("BarProduct"))])], + dependencies: ["BarProduct"])]) + + let project = TestProject( + "aProject", + groupTree: TestGroup("Sources", children: [TestFile("App1.swift"), TestFile("App2.swift")]), + buildConfigurations: [TestBuildConfiguration("Debug", buildSettings: commonBuildSettings.addingContents(of: [ + "SWIFT_INCLUDE_PATHS": "$(TARGET_BUILD_DIR)/../../../aPackage/build/Debug $(TARGET_BUILD_DIR)/../../../aPackageLeaf/build/Debug"]))], + targets: [ + TestStandardTarget( + "App1", + type: .framework, + buildConfigurations: [TestBuildConfiguration("Debug", buildSettings: [ + "PRODUCT_NAME": "$(TARGET_NAME)", + "SWIFT_ENABLE_COMPILE_CACHE": "YES", + "COMPILATION_CACHE_ENABLE_DIAGNOSTIC_REMARKS": "YES", + "COMPILATION_CACHE_CAS_PATH": "$(DSTROOT)/CompilationCache"])], + buildPhases: [ + TestSourcesBuildPhase(["App1.swift"]), + TestFrameworksBuildPhase([TestBuildFile(.target("FooProduct"))])], + dependencies: ["FooProduct"]), + TestStandardTarget( + "App2", + type: .framework, + buildConfigurations: [TestBuildConfiguration("Debug", buildSettings: [ + "PRODUCT_NAME": "$(TARGET_NAME)"])], + buildPhases: [ + TestSourcesBuildPhase(["App2.swift"]), + TestFrameworksBuildPhase([TestBuildFile(.target("FooProduct"))])], + dependencies: ["FooProduct"])]) + + let workspace = TestWorkspace("aWorkspace", sourceRoot: tmpDirPath.join("Test"), projects: [project, package, leafPackage]) + + let tester = try await BuildOperationTester(getCore(), workspace, simulated: false) + + try await tester.fs.writeFileContents(workspace.sourceRoot.join("aPackageLeaf/Bar.swift")) { stream in + stream <<< + """ + public func baz() {} + """ + } + + try await tester.fs.writeFileContents(workspace.sourceRoot.join("aPackage/Foo.swift")) { stream in + stream <<< + """ + import Bar + public func foo() { baz() } + """ + } + + try await tester.fs.writeFileContents(workspace.sourceRoot.join("aProject/App1.swift")) { stream in + stream <<< + """ + import Foo + func app() { foo() } + """ + } + + try await tester.fs.writeFileContents(workspace.sourceRoot.join("aProject/App2.swift")) { stream in + stream <<< + """ + import Foo + func app() { foo() } + """ + } + + let parameters = BuildParameters(configuration: "Debug", overrides: ["ARCHS": "arm64"]) + let buildApp1Target = BuildRequest.BuildTargetInfo(parameters: parameters, target: tester.workspace.projects[0].targets[0]) + let buildApp2Target = BuildRequest.BuildTargetInfo(parameters: parameters, target: tester.workspace.projects[0].targets[1]) + let buildRequest = BuildRequest(parameters: parameters, buildTargets: [buildApp2Target, buildApp1Target], continueBuildingAfterErrors: false, useParallelTargets: false, useImplicitDependencies: false, useDryRun: false) + + try await tester.checkBuild(runDestination: .macOS, buildRequest: buildRequest, persistent: true) { results in + results.checkNoDiagnostics() + + results.checkTasks(.matchRule(["SwiftCompile", "normal", "arm64", "Compiling Bar.swift", tmpDirPath.join("Test/aPackageLeaf/Bar.swift").str])) { tasks in + #expect(tasks.count == 1) + for task in tasks { + results.checkKeyQueryCacheMiss(task) + } + } + + results.checkTask(.matchRule(["SwiftCompile", "normal", "arm64", "Compiling Foo.swift", tmpDirPath.join("Test/aPackage/Foo.swift").str])) { task in + results.checkKeyQueryCacheMiss(task) + } + + results.checkTask(.matchRule(["SwiftCompile", "normal", "arm64", "Compiling App1.swift", tmpDirPath.join("Test/aProject/App1.swift").str])) { task in + results.checkKeyQueryCacheMiss(task) + } + + results.checkTask(.matchRule(["SwiftCompile", "normal", "arm64", "Compiling App2.swift", "\(tmpDirPath.str)/Test/aProject/App2.swift"])) { task in + results.checkNotCached(task) + } + } + } + } + @Test(.requireSDKs(.macOS)) func swiftCASLimiting() async throws { try await withTemporaryDirectory { (tmpDirPath: Path) async throws -> Void in @@ -273,21 +412,28 @@ fileprivate struct SwiftCompilationCachingTests: CoreBasedTests { } extension BuildOperationTester.BuildResults { + fileprivate func checkNotCached(_ task: Task, sourceLocation: SourceLocation = #_sourceLocation) { + check(notContains: .taskHadEvent(task, event: .hadOutput(contents: "Cache miss\n")), sourceLocation: sourceLocation) + check(notContains: .taskHadEvent(task, event: .hadOutput(contents: "Cache hit\n")), sourceLocation: sourceLocation) + } + fileprivate func checkKeyQueryCacheMiss(_ task: Task, sourceLocation: SourceLocation = #_sourceLocation) { - let found = (getDiagnosticMessageForTask(.contains("cache miss"), kind: .note, task: task) != nil) - guard found else { - Issue.record("Unable to find cache miss diagnostic for task \(task)", sourceLocation: sourceLocation) - return - } + // FIXME: This doesn't work as expected (at least for Swift package targets). + // let found = (getDiagnosticMessageForTask(.contains("cache miss"), kind: .note, task: task) != nil) + // guard found else { + // Issue.record("Unable to find cache miss diagnostic for task \(task)", sourceLocation: sourceLocation) + // return + // } check(contains: .taskHadEvent(task, event: .hadOutput(contents: "Cache miss\n")), sourceLocation: sourceLocation) } fileprivate func checkKeyQueryCacheHit(_ task: Task, sourceLocation: SourceLocation = #_sourceLocation) { - let found = (getDiagnosticMessageForTask(.contains("cache found for key"), kind: .note, task: task) != nil) - guard found else { - Issue.record("Unable to find cache hit diagnostic for task \(task)", sourceLocation: sourceLocation) - return - } + // FIXME: This doesn't work as expected (at least for Swift package targets). + // let found = (getDiagnosticMessageForTask(.contains("cache found for key"), kind: .note, task: task) != nil) + // guard found else { + // Issue.record("Unable to find cache hit diagnostic for task \(task)", sourceLocation: sourceLocation) + // return + // } check(contains: .taskHadEvent(task, event: .hadOutput(contents: "Cache hit\n")), sourceLocation: sourceLocation) } } From c4d6059e25dbd6ed8b26efb248331452aee1c360 Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Thu, 26 Jun 2025 11:23:55 -0700 Subject: [PATCH 50/54] Fix toolchain stack equality checks when checking if specialized targets are compatible --- Sources/SWBCore/DependencyResolution.swift | 4 +- .../TargetDependencyResolverTests.swift | 52 +++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/Sources/SWBCore/DependencyResolution.swift b/Sources/SWBCore/DependencyResolution.swift index dcf70e52..e3bcbe3f 100644 --- a/Sources/SWBCore/DependencyResolution.swift +++ b/Sources/SWBCore/DependencyResolution.swift @@ -189,8 +189,8 @@ struct SpecializationParameters: Hashable, CustomStringConvertible { func isCompatible(with configuredTarget: ConfiguredTarget, settings: Settings, workspaceContext: WorkspaceContext) -> Bool { let toolchain = effectiveToolchainOverride(originalParameters: configuredTarget.parameters, workspaceContext: workspaceContext) return (platform == nil || platform === settings.platform) && - (sdkVariant == nil || sdkVariant?.name == settings.sdkVariant?.name) && - (toolchain == nil || toolchain == settings.globalScope.evaluate(BuiltinMacros.TOOLCHAINS)) && + (sdkVariant == nil || sdkVariant?.name == settings.sdkVariant?.name) && + (toolchain == nil || toolchain == settings.toolchains.map(\.identifier)) && (canonicalNameSuffix == nil || canonicalNameSuffix?.nilIfEmpty == settings.sdk?.canonicalNameSuffix) } diff --git a/Tests/SWBCoreTests/TargetDependencyResolverTests.swift b/Tests/SWBCoreTests/TargetDependencyResolverTests.swift index 0b43a0dc..ca0b1e04 100644 --- a/Tests/SWBCoreTests/TargetDependencyResolverTests.swift +++ b/Tests/SWBCoreTests/TargetDependencyResolverTests.swift @@ -922,6 +922,58 @@ fileprivate enum TargetPlatformSpecializationMode { } } + @Test + func toolchainOverlaysViaOverridesDoNotConflictWithSpecialization() async throws { + let core = try await getCore() + let workspace = try TestWorkspace("Workspace", + projects: [TestPackageProject("aProject", + groupTree: TestGroup("SomeFiles"), + targets: [ + TestAggregateTarget("ALL", dependencies: ["iOSFwk", "PackageLibProduct"]), + TestStandardTarget( + "iOSFwk", + type: .framework, + buildConfigurations: [ + TestBuildConfiguration("Debug", buildSettings: ["SDKROOT": "macosx"]), + ], + dependencies: ["PackageLibProduct"] + ), + TestPackageProductTarget( + "PackageLibProduct", + frameworksBuildPhase: TestFrameworksBuildPhase([ + TestBuildFile(.target("PackageLib"))]), + buildConfigurations: [ + // Targets need to opt-in to specialization. + TestBuildConfiguration("Debug", buildSettings: [ + "SDKROOT": "auto", + "SDK_VARIANT": "auto", + "SUPPORTED_PLATFORMS": "macosx iphoneos iphonesimulator appletvos appletvsimulator watchos watchsimulator", + ]), + ], + dependencies: ["PackageLib"] + ), + TestStandardTarget("PackageLib", type: .staticLibrary), + ] + )] + ).load(core) + let workspaceContext = WorkspaceContext(core: core, workspace: workspace, processExecutionCache: .sharedForTesting) + let project = workspace.projects[0] + + // Configure the targets and create a BuildRequest. + let buildParameters = BuildParameters(configuration: "Debug", activeRunDestination: RunDestinationInfo.macOS, overrides: ["TOOLCHAINS": "com.fake-toolchain-identifier"]) + let allTarget = BuildRequest.BuildTargetInfo(parameters: buildParameters, target: project.targets[0]) + let packageTarget = BuildRequest.BuildTargetInfo(parameters: buildParameters, target: project.targets[2]) + let buildRequest = BuildRequest(parameters: buildParameters, buildTargets: [allTarget, packageTarget], continueBuildingAfterErrors: true, useParallelTargets: false, useImplicitDependencies: false, useDryRun: false) + let buildRequestContext = BuildRequestContext(workspaceContext: workspaceContext) + + for type in TargetGraphFactory.GraphType.allCases { + // Get the dependency closure for the build request and examine it. + let delegate = EmptyTargetDependencyResolverDelegate(workspace: workspaceContext.workspace) + _ = await TargetGraphFactory(workspaceContext: workspaceContext, buildRequest: buildRequest, buildRequestContext: buildRequestContext, delegate: delegate).graph(type: type) + delegate.checkNoDiagnostics() + } + } + @Test(.requireSDKs(.macOS)) func macCatalystSpecialization() async throws { let core = try await getCore() From 01a7c3d87b236609e21259c4097f5a8f427d9424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Laferrie=CC=80re?= Date: Thu, 26 Jun 2025 13:43:55 -0700 Subject: [PATCH 51/54] Don't require objc in modulemaps generated for Swift compatibility headers Let's remove the `requires objc` from the module map file generated for the compatibility header. This condition triggered an error when importing the compatibility header from a C source file. This declaration is superfluous, the compatibility header is already printed in a way where the Objective-C code is protected behind a language check. C clients can safely import the current compatibility header even if they may not see any content. Let's lift this restriction. It isn't currently necessary and we're adding C content to the compatibility header with the official support for `@cdecl` that is independent of Objective-C. --- .../OtherTaskProducers/ModuleMapTaskProducer.swift | 1 - .../ModuleMapTaskConstructionTests.swift | 3 --- .../SWBTaskConstructionTests/SwiftTaskConstructionTests.swift | 2 -- 3 files changed, 6 deletions(-) diff --git a/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/ModuleMapTaskProducer.swift b/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/ModuleMapTaskProducer.swift index 3de99336..26124e5c 100644 --- a/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/ModuleMapTaskProducer.swift +++ b/Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/ModuleMapTaskProducer.swift @@ -404,7 +404,6 @@ final class ModuleMapTaskProducer: PhasedTaskProducer, TaskProducer { outputStream <<< "module \(try moduleName.asModuleIdentifierString()).Swift {\n" } outputStream <<< " header \"\(interfaceHeaderName.asCStringLiteralContent)\"\n" - outputStream <<< " requires objc\n" outputStream <<< "}\n" return outputStream.bytes diff --git a/Tests/SWBTaskConstructionTests/ModuleMapTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/ModuleMapTaskConstructionTests.swift index 02438a96..0c67d4ab 100644 --- a/Tests/SWBTaskConstructionTests/ModuleMapTaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/ModuleMapTaskConstructionTests.swift @@ -584,7 +584,6 @@ fileprivate struct ModuleMapTaskConstructionTests: CoreBasedTests { #expect(contents == (OutputByteStream() <<< "framework module SwiftOnly {\n" <<< " header \"SwiftOnly-Swift.h\"\n" - <<< " requires objc\n" <<< "}\n").bytes) } @@ -629,7 +628,6 @@ fileprivate struct ModuleMapTaskConstructionTests: CoreBasedTests { <<< "\n" <<< "module ObjCCompatibilityHeader.Swift {\n" <<< " header \"ObjCCompatibilityHeader-Swift.h\"\n" - <<< " requires objc\n" <<< "}\n").bytes) } @@ -978,7 +976,6 @@ fileprivate struct ModuleMapTaskConstructionTests: CoreBasedTests { <<< "\n" <<< "module \(targetName).Swift {\n" <<< " header \"\(targetName)-Swift.h\"\n" - <<< " requires objc\n" <<< "}\n").bytes) } diff --git a/Tests/SWBTaskConstructionTests/SwiftTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/SwiftTaskConstructionTests.swift index 42904bd0..76180b0c 100644 --- a/Tests/SWBTaskConstructionTests/SwiftTaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/SwiftTaskConstructionTests.swift @@ -606,7 +606,6 @@ fileprivate struct SwiftTaskConstructionTests: CoreBasedTests { #expect(contents == (OutputByteStream() <<< "framework module CoreFoo {\n" <<< " header \"CoreFoo-Swift.h\"\n" - <<< " requires objc\n" <<< "}\n").bytes) } } @@ -1086,7 +1085,6 @@ fileprivate struct SwiftTaskConstructionTests: CoreBasedTests { stream <<< "\n" stream <<< "module CoreFoo.Swift {\n" stream <<< " header \"CoreFoo-Swift.h\"\n" - stream <<< " requires objc\n" stream <<< "}\n" #expect(contents == stream.bytes) From 675cd2b19f617cd11fbb30296b2ca519c76d8031 Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Fri, 27 Jun 2025 22:22:45 -0700 Subject: [PATCH 52/54] Allow downgrading API breakage errors to warnings --- Sources/SWBCore/Settings/BuiltinMacros.swift | 2 + .../Tools/SwiftABICheckerTool.swift | 64 +++++++++++-- .../APIDigesterBuildOperationTests.swift | 90 +++++++++++++++++++ 3 files changed, 151 insertions(+), 5 deletions(-) create mode 100644 Tests/SWBBuildSystemTests/APIDigesterBuildOperationTests.swift diff --git a/Sources/SWBCore/Settings/BuiltinMacros.swift b/Sources/SWBCore/Settings/BuiltinMacros.swift index 618b3ca5..ba164dd5 100644 --- a/Sources/SWBCore/Settings/BuiltinMacros.swift +++ b/Sources/SWBCore/Settings/BuiltinMacros.swift @@ -997,6 +997,7 @@ public final class BuiltinMacros { public static let RPATH_ORIGIN = BuiltinMacros.declareStringMacro("RPATH_ORIGIN") public static let PLATFORM_USES_DSYMS = BuiltinMacros.declareBooleanMacro("PLATFORM_USES_DSYMS") public static let SWIFT_ABI_CHECKER_BASELINE_DIR = BuiltinMacros.declareStringMacro("SWIFT_ABI_CHECKER_BASELINE_DIR") + public static let SWIFT_ABI_CHECKER_DOWNGRADE_ERRORS = BuiltinMacros.declareBooleanMacro("SWIFT_ABI_CHECKER_DOWNGRADE_ERRORS") public static let SWIFT_ABI_CHECKER_EXCEPTIONS_FILE = BuiltinMacros.declareStringMacro("SWIFT_ABI_CHECKER_EXCEPTIONS_FILE") public static let SWIFT_ABI_GENERATION_TOOL_OUTPUT_DIR = BuiltinMacros.declareStringMacro("SWIFT_ABI_GENERATION_TOOL_OUTPUT_DIR") public static let SWIFT_ACCESS_NOTES_PATH = BuiltinMacros.declareStringMacro("SWIFT_ACCESS_NOTES_PATH") @@ -2171,6 +2172,7 @@ public final class BuiltinMacros { RPATH_ORIGIN, PLATFORM_USES_DSYMS, SWIFT_ABI_CHECKER_BASELINE_DIR, + SWIFT_ABI_CHECKER_DOWNGRADE_ERRORS, SWIFT_ABI_CHECKER_EXCEPTIONS_FILE, SWIFT_ABI_GENERATION_TOOL_OUTPUT_DIR, SWIFT_ACCESS_NOTES_PATH, diff --git a/Sources/SWBCore/SpecImplementations/Tools/SwiftABICheckerTool.swift b/Sources/SWBCore/SpecImplementations/Tools/SwiftABICheckerTool.swift index b33ca677..4cb16d6b 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/SwiftABICheckerTool.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/SwiftABICheckerTool.swift @@ -42,17 +42,22 @@ public final class SwiftABICheckerToolSpec : GenericCommandLineToolSpec, SpecIde /// The path to the serialized diagnostic output. Every clang task must provide this path. let serializedDiagnosticsPath: Path - init(serializedDiagnosticsPath: Path) { + let downgradeErrors: Bool + + init(serializedDiagnosticsPath: Path, downgradeErrors: Bool) { self.serializedDiagnosticsPath = serializedDiagnosticsPath + self.downgradeErrors = downgradeErrors } public func serialize(to serializer: T) { - serializer.serializeAggregate(1) { + serializer.serializeAggregate(2) { serializer.serialize(serializedDiagnosticsPath) + serializer.serialize(downgradeErrors) } } public init(from deserializer: any Deserializer) throws { - try deserializer.beginAggregate(1) + try deserializer.beginAggregate(2) self.serializedDiagnosticsPath = try deserializer.deserialize() + self.downgradeErrors = try deserializer.deserialize() } } @@ -67,7 +72,12 @@ public final class SwiftABICheckerToolSpec : GenericCommandLineToolSpec, SpecIde // Override this func to ensure we can see these diagnostics in unit tests. public override func customOutputParserType(for task: any ExecutableTask) -> (any TaskOutputParser.Type)? { - return SerializedDiagnosticsOutputParser.self + let payload = task.payload! as! ABICheckerPayload + if payload.downgradeErrors { + return APIDigesterDowngradingSerializedDiagnosticsOutputParser.self + } else { + return SerializedDiagnosticsOutputParser.self + } } public func constructABICheckingTask(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, _ serializedDiagsPath: Path, _ baselinePath: Path?, _ allowlistPath: Path?) async { let toolSpecInfo: DiscoveredSwiftCompilerToolSpecInfo @@ -86,6 +96,10 @@ public final class SwiftABICheckerToolSpec : GenericCommandLineToolSpec, SpecIde if let allowlistPath { commandLine += ["-breakage-allowlist-path", allowlistPath.normalize().str] } + let downgradeErrors = cbc.scope.evaluate(BuiltinMacros.SWIFT_ABI_CHECKER_DOWNGRADE_ERRORS) + if downgradeErrors { + commandLine += ["-disable-fail-on-error"] + } let allInputs = cbc.inputs.map { delegate.createNode($0.absolutePath) } + [baselinePath, allowlistPath].compactMap { $0 }.map { delegate.createNode($0.normalize()) } // Add import search paths for searchPath in SwiftCompilerSpec.collectInputSearchPaths(cbc, toolInfo: toolSpecInfo) { @@ -95,7 +109,10 @@ public final class SwiftABICheckerToolSpec : GenericCommandLineToolSpec, SpecIde commandLine += cbc.scope.evaluate(BuiltinMacros.SWIFT_SYSTEM_INCLUDE_PATHS).flatMap { ["-I", $0] } commandLine += cbc.scope.evaluate(BuiltinMacros.SYSTEM_FRAMEWORK_SEARCH_PATHS).flatMap { ["-F", $0] } delegate.createTask(type: self, - payload: ABICheckerPayload(serializedDiagnosticsPath: serializedDiagsPath), + payload: ABICheckerPayload( + serializedDiagnosticsPath: serializedDiagsPath, + downgradeErrors: downgradeErrors + ), ruleInfo: defaultRuleInfo(cbc, delegate), commandLine: commandLine, environment: environmentFromSpec(cbc, delegate), @@ -105,3 +122,40 @@ public final class SwiftABICheckerToolSpec : GenericCommandLineToolSpec, SpecIde enableSandboxing: enableSandboxing) } } + +public final class APIDigesterDowngradingSerializedDiagnosticsOutputParser: TaskOutputParser { + private let task: any ExecutableTask + + public let workspaceContext: WorkspaceContext + public let buildRequestContext: BuildRequestContext + public let delegate: any TaskOutputParserDelegate + + required public init(for task: any ExecutableTask, workspaceContext: WorkspaceContext, buildRequestContext: BuildRequestContext, delegate: any TaskOutputParserDelegate, progressReporter: (any SubtaskProgressReporter)?) { + self.task = task + self.workspaceContext = workspaceContext + self.buildRequestContext = buildRequestContext + self.delegate = delegate + } + + public func write(bytes: ByteString) { + // Forward the unparsed bytes immediately (without line buffering). + delegate.emitOutput(bytes) + + // Disable diagnostic scraping, since we use serialized diagnostics. + } + + public func close(result: TaskResult?) { + defer { + delegate.close() + } + // Don't try to read diagnostics if the process crashed or got cancelled as they were almost certainly not written in this case. + if result.shouldSkipParsingDiagnostics { return } + + for path in task.type.serializedDiagnosticsPaths(task, workspaceContext.fs) { + let diagnostics = delegate.readSerializedDiagnostics(at: path, workingDirectory: task.workingDirectory, workspaceContext: workspaceContext) + for diagnostic in diagnostics { + delegate.diagnosticsEngine.emit(diagnostic.with(behavior: diagnostic.behavior == .error ? .warning : diagnostic.behavior)) + } + } + } +} diff --git a/Tests/SWBBuildSystemTests/APIDigesterBuildOperationTests.swift b/Tests/SWBBuildSystemTests/APIDigesterBuildOperationTests.swift new file mode 100644 index 00000000..afd19b47 --- /dev/null +++ b/Tests/SWBBuildSystemTests/APIDigesterBuildOperationTests.swift @@ -0,0 +1,90 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Testing +import Foundation + +import SWBBuildSystem +import SWBCore +import SWBTestSupport +import SWBTaskExecution +import SWBUtil +import SWBProtocol + +@Suite +fileprivate struct APIDigesterBuildOperationTests: CoreBasedTests { + @Test(.requireSDKs(.host), .skipHostOS(.windows, "Windows toolchains are missing swift-api-digester")) + func apiDigesterDisableFailOnError() async throws { + try await withTemporaryDirectory { (tmpDir: Path) in + let testProject = try await TestProject( + "TestProject", + sourceRoot: tmpDir, + groupTree: TestGroup( + "SomeFiles", + children: [ + TestFile("foo.swift"), + ]), + buildConfigurations: [ + TestBuildConfiguration("Debug", buildSettings: [ + "ARCHS": "$(ARCHS_STANDARD)", + "PRODUCT_NAME": "$(TARGET_NAME)", + "SDKROOT": "$(HOST_PLATFORM)", + "SUPPORTED_PLATFORMS": "$(HOST_PLATFORM)", + "SWIFT_VERSION": swiftVersion, + "CODE_SIGNING_ALLOWED": "NO", + ]) + ], + targets: [ + TestStandardTarget( + "foo", + type: .dynamicLibrary, + buildConfigurations: [ + TestBuildConfiguration("Debug", buildSettings: [:]) + ], + buildPhases: [ + TestSourcesBuildPhase(["foo.swift"]), + ] + ), + ]) + let core = try await getCore() + let tester = try await BuildOperationTester(core, testProject, simulated: false) + + let projectDir = tester.workspace.projects[0].sourceRoot + + try await tester.fs.writeFileContents(projectDir.join("foo.swift")) { stream in + stream <<< "public func foo() -> Int { 42 }" + } + + try await tester.checkBuild(parameters: BuildParameters(configuration: "Debug", overrides: [ + "RUN_SWIFT_ABI_GENERATION_TOOL": "YES", + "SWIFT_API_DIGESTER_MODE": "api", + "SWIFT_ABI_GENERATION_TOOL_OUTPUT_DIR": tmpDir.join("baseline").join("ABI").str, + ]), runDestination: .host) { results in + results.checkNoErrors() + } + + try await tester.fs.writeFileContents(projectDir.join("foo.swift")) { stream in + stream <<< "public func foo() -> String { \"hello, world!\" }" + } + + try await tester.checkBuild(parameters: BuildParameters(configuration: "Debug", overrides: [ + "RUN_SWIFT_ABI_CHECKER_TOOL": "YES", + "SWIFT_API_DIGESTER_MODE": "api", + "SWIFT_ABI_CHECKER_BASELINE_DIR": tmpDir.join("baseline").str, + "SWIFT_ABI_CHECKER_DOWNGRADE_ERRORS": "YES", + ]), runDestination: .host) { results in + results.checkWarning(.contains("func foo() has return type change from Swift.Int to Swift.String")) + results.checkNoDiagnostics() + } + } + } +} From b860b49a99f7b2a91777bd7cfffb6d71f3a463a4 Mon Sep 17 00:00:00 2001 From: Owen Voorhees Date: Sun, 29 Jun 2025 13:42:32 -0700 Subject: [PATCH 53/54] Allow setting LINKER_DRIVER=auto to choose an appropriate value for the given target's sources --- Sources/SWBCore/PlannedTaskAction.swift | 9 +- Sources/SWBCore/Settings/BuiltinMacros.swift | 1 + .../SpecImplementations/ProductTypes.swift | 24 +++-- .../Tools/LinkerTools.swift | 102 +++++++++++++----- Tests/SWBCoreTests/CommandLineSpecTests.swift | 6 +- .../LinkerTaskConstructionTests.swift | 97 +++++++++++++++++ 6 files changed, 196 insertions(+), 43 deletions(-) create mode 100644 Tests/SWBTaskConstructionTests/LinkerTaskConstructionTests.swift diff --git a/Sources/SWBCore/PlannedTaskAction.swift b/Sources/SWBCore/PlannedTaskAction.swift index 999660e5..3518ddc3 100644 --- a/Sources/SWBCore/PlannedTaskAction.swift +++ b/Sources/SWBCore/PlannedTaskAction.swift @@ -264,7 +264,14 @@ public struct FileCopyTaskActionContext { extension FileCopyTaskActionContext { public init(_ cbc: CommandBuildContext) { let compilerPath = cbc.producer.clangSpec.resolveExecutablePath(cbc, forLanguageOfFileType: cbc.producer.lookupFileType(languageDialect: .c)) - let linkerPath = cbc.producer.ldLinkerSpec.resolveExecutablePath(cbc.producer, Path(cbc.producer.ldLinkerSpec.computeExecutablePath(cbc))) + let linkerPath = cbc.producer.ldLinkerSpec.resolveExecutablePath(cbc.producer, cbc.producer.ldLinkerSpec.computeLinkerPath(cbc, usedCXX: false, lookup: { macro in + switch macro { + case BuiltinMacros.LINKER_DRIVER: + return cbc.scope.namespace.parseString("clang") + default: + return nil + } + })) let lipoPath = cbc.producer.lipoSpec.resolveExecutablePath(cbc.producer, Path(cbc.producer.lipoSpec.computeExecutablePath(cbc))) // If we couldn't find clang, skip the special stub binary handling. We may be using an Open Source toolchain which only has Swift. Also skip it for installLoc builds. diff --git a/Sources/SWBCore/Settings/BuiltinMacros.swift b/Sources/SWBCore/Settings/BuiltinMacros.swift index 618b3ca5..541cbcad 100644 --- a/Sources/SWBCore/Settings/BuiltinMacros.swift +++ b/Sources/SWBCore/Settings/BuiltinMacros.swift @@ -2701,6 +2701,7 @@ public enum LinkerDriverChoice: String, Equatable, Hashable, EnumerationMacroTyp case clang case swiftc + case auto } /// Enumeration macro type for the value of the `INFOPLIST_KEY_LSApplicationCategoryType` build setting. diff --git a/Sources/SWBCore/SpecImplementations/ProductTypes.swift b/Sources/SWBCore/SpecImplementations/ProductTypes.swift index 7f26983b..345c0d85 100644 --- a/Sources/SWBCore/SpecImplementations/ProductTypes.swift +++ b/Sources/SWBCore/SpecImplementations/ProductTypes.swift @@ -260,31 +260,35 @@ public class ProductTypeSpec : Spec, SpecType, @unchecked Sendable { } /// Computes and returns additional arguments to pass to the linker appropriate for the product type. Also returns a list of additional paths to treat as inputs to the link command, if appropriate. - func computeAdditionalLinkerArgs(_ producer: any CommandProducer, scope: MacroEvaluationScope) -> (args: [String], inputs: [Path]) { + func computeAdditionalLinkerArgs(_ producer: any CommandProducer, scope: MacroEvaluationScope, lookup: @escaping ((MacroDeclaration) -> MacroStringExpression?)) -> (args: [String], inputs: [Path]) { return ([], []) } - fileprivate func computeDylibArgs(_ producer: any CommandProducer, _ scope: MacroEvaluationScope) -> [String] { + fileprivate func computeDylibArgs(_ producer: any CommandProducer, _ scope: MacroEvaluationScope, lookup: @escaping ((MacroDeclaration) -> MacroStringExpression?)) -> [String] { var args = [String]() if producer.isApplePlatform { - let compatibilityVersion = scope.evaluate(BuiltinMacros.DYLIB_COMPATIBILITY_VERSION) + let compatibilityVersion = scope.evaluate(BuiltinMacros.DYLIB_COMPATIBILITY_VERSION, lookup: lookup) if !compatibilityVersion.isEmpty { - switch scope.evaluate(BuiltinMacros.LINKER_DRIVER) { + switch scope.evaluate(BuiltinMacros.LINKER_DRIVER, lookup: lookup) { case .clang: args += ["-compatibility_version", compatibilityVersion] case .swiftc: args += ["-Xlinker", "-compatibility_version", "-Xlinker", compatibilityVersion] + case .auto: + preconditionFailure("Expected LINKER_DRIVER to be bound to a concrete value") } } - let currentVersion = scope.evaluate(BuiltinMacros.DYLIB_CURRENT_VERSION) + let currentVersion = scope.evaluate(BuiltinMacros.DYLIB_CURRENT_VERSION, lookup: lookup) if !currentVersion.isEmpty { - switch scope.evaluate(BuiltinMacros.LINKER_DRIVER) { + switch scope.evaluate(BuiltinMacros.LINKER_DRIVER, lookup: lookup) { case .clang: args += ["-current_version", currentVersion] case .swiftc: args += ["-Xlinker", "-current_version", "-Xlinker", currentVersion] + case .auto: + preconditionFailure("Expected LINKER_DRIVER to be bound to a concrete value") } } } @@ -563,9 +567,9 @@ public class FrameworkProductTypeSpec : BundleProductTypeSpec, @unchecked Sendab ]) */ - override func computeAdditionalLinkerArgs(_ producer: any CommandProducer, scope: MacroEvaluationScope) -> (args: [String], inputs: [Path]) { + override func computeAdditionalLinkerArgs(_ producer: any CommandProducer, scope: MacroEvaluationScope, lookup: @escaping ((MacroDeclaration) -> MacroStringExpression?)) -> (args: [String], inputs: [Path]) { if scope.evaluate(BuiltinMacros.MACH_O_TYPE) != "staticlib" { - return (computeDylibArgs(producer, scope), []) + return (computeDylibArgs(producer, scope, lookup: lookup), []) } return ([], []) } @@ -801,9 +805,9 @@ public final class DynamicLibraryProductTypeSpec : LibraryProductTypeSpec, @unch return true } - override func computeAdditionalLinkerArgs(_ producer: any CommandProducer, scope: MacroEvaluationScope) -> (args: [String], inputs: [Path]) { + override func computeAdditionalLinkerArgs(_ producer: any CommandProducer, scope: MacroEvaluationScope, lookup: @escaping ((MacroDeclaration) -> MacroStringExpression?)) -> (args: [String], inputs: [Path]) { if scope.evaluate(BuiltinMacros.MACH_O_TYPE) != "staticlib" { - return (computeDylibArgs(producer, scope), []) + return (computeDylibArgs(producer, scope, lookup: lookup), []) } return ([], []) } diff --git a/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift b/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift index d00cb7da..9c36a2c1 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift @@ -233,16 +233,6 @@ public struct DiscoveredLdLinkerToolSpecInfo: DiscoveredCommandLineToolSpecInfo public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchecked Sendable { public static let identifier = "com.apple.pbx.linkers.ld" - public override func computeExecutablePath(_ cbc: CommandBuildContext) -> String { - // TODO: We should also provide an "auto" option which chooses based on the source files in the target - switch cbc.scope.evaluate(BuiltinMacros.LINKER_DRIVER) { - case .clang: - return cbc.producer.hostOperatingSystem.imageFormat.executableName(basename: "clang") - case .swiftc: - return cbc.producer.hostOperatingSystem.imageFormat.executableName(basename: "swiftc") - } - } - override public var toolBasenameAliases: [String] { // We use clang as our linker, so return ld and libtool in aliases in // order to parse the errors from the actual linker. @@ -281,7 +271,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec } // FIXME: Is there a better way to figure out if we are linking Swift? - private func isUsingSwift(_ usedTools: [CommandLineToolSpec: Set]) -> Bool { + private static func isUsingSwift(_ usedTools: [CommandLineToolSpec: Set]) -> Bool { return usedTools.keys.map({ type(of: $0) }).contains(where: { $0 == SwiftCompilerSpec.self }) } @@ -304,10 +294,35 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec return runpathSearchPaths } + static func resolveLinkerDriver(_ cbc: CommandBuildContext, usedTools: [CommandLineToolSpec: Set]) -> LinkerDriverChoice { + switch cbc.scope.evaluate(BuiltinMacros.LINKER_DRIVER) { + case .clang: + return .clang + case .swiftc: + return.swiftc + case .auto: + if Self.isUsingSwift(usedTools) { + return .swiftc + } else { + return .clang + } + } + } + override public func constructLinkerTasks(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, libraries: [LibrarySpecifier], usedTools: [CommandLineToolSpec: Set]) async { + let resolvedLinkerDriver = Self.resolveLinkerDriver(cbc, usedTools: usedTools) + let linkerDriverLookup: ((MacroDeclaration) -> MacroStringExpression?) = { macro in + switch macro { + case BuiltinMacros.LINKER_DRIVER: + return cbc.scope.namespace.parseString(resolvedLinkerDriver.rawValue) + default: + return nil + } + } + // Validate that OTHER_LDFLAGS doesn't contain flags for constructs which we have dedicated settings for. This should be expanded over time. let dyldEnvDiagnosticBehavior: Diagnostic.Behavior = SWBFeatureFlag.useStrictLdEnvironmentBuildSetting.value ? .error : .warning - let originalLdFlags = cbc.scope.evaluate(BuiltinMacros.OTHER_LDFLAGS) + let originalLdFlags = cbc.scope.evaluate(BuiltinMacros.OTHER_LDFLAGS, lookup: linkerDriverLookup) enumerateLinkerCommandLine(arguments: originalLdFlags) { arg, value in switch arg { case "-dyld_env": @@ -354,7 +369,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec specialArgs.append(contentsOf: sparseSDKSearchPathArguments(cbc)) // Define the linker file list. - let fileListPath = cbc.scope.evaluate(BuiltinMacros.__INPUT_FILE_LIST_PATH__) + let fileListPath = cbc.scope.evaluate(BuiltinMacros.__INPUT_FILE_LIST_PATH__, lookup: linkerDriverLookup) if !fileListPath.isEmpty { let contents = OutputByteStream() for input in cbc.inputs { @@ -385,7 +400,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec } // Add linker flags desired by the product type. - let productTypeArgs = cbc.producer.productType?.computeAdditionalLinkerArgs(cbc.producer, scope: cbc.scope) + let productTypeArgs = cbc.producer.productType?.computeAdditionalLinkerArgs(cbc.producer, scope: cbc.scope, lookup: linkerDriverLookup) specialArgs += productTypeArgs?.args ?? [] inputPaths += productTypeArgs?.inputs ?? [] @@ -425,7 +440,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec inputPaths.append(contentsOf: inputs) } - let isLinkUsingSwift = isUsingSwift(usedTools) + let isLinkUsingSwift = Self.isUsingSwift(usedTools) if !isLinkUsingSwift { // Check if we need to link with Swift's standard library // when linking a pure Objective-C/C++ target. This might be needed @@ -483,6 +498,9 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec let frameworkSearchPathsExpr = cbc.scope.namespace.parseStringList(frameworkSearchPaths) func lookup(_ macro: MacroDeclaration) -> MacroExpression? { + if let result = linkerDriverLookup(macro) { + return result + } switch macro { case BuiltinMacros.LD_RUNPATH_SEARCH_PATHS: return runpathSearchPathsExpr @@ -589,7 +607,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec // Select the driver to use based on the input file types, replacing the value computed by commandLineFromTemplate(). let usedCXX = usedTools.values.contains(where: { $0.contains(where: { $0.languageDialect?.isPlusPlus ?? false }) }) - commandLine[0] = await resolveExecutablePath(cbc, computeLinkerPath(cbc, usedCXX: usedCXX), delegate: delegate).str + commandLine[0] = await resolveExecutablePath(cbc, computeLinkerPath(cbc, usedCXX: usedCXX, lookup: linkerDriverLookup), delegate: delegate).str let entitlementsSection = cbc.scope.evaluate(BuiltinMacros.LD_ENTITLEMENTS_SECTION) if !entitlementsSection.isEmpty { @@ -763,6 +781,15 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec } public func constructPreviewShimLinkerTasks(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, libraries: [LibrarySpecifier], usedTools: [CommandLineToolSpec: Set], rpaths: [String], ldflags: [String]?) async { + let resolvedLinkerDriver = Self.resolveLinkerDriver(cbc, usedTools: usedTools) + let linkerDriverLookup: ((MacroDeclaration) -> MacroStringExpression?) = { macro in + switch macro { + case BuiltinMacros.LINKER_DRIVER: + return cbc.scope.namespace.parseString(resolvedLinkerDriver.rawValue) + default: + return nil + } + } // Construct the "special args". var specialArgs = [String]() var inputPaths = cbc.inputs.map({ $0.absolutePath }) @@ -782,6 +809,9 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec } func lookup(_ macro: MacroDeclaration) -> MacroExpression? { + if let result = linkerDriverLookup(macro) { + return result + } switch macro { case BuiltinMacros.LD_ENTRY_POINT where cbc.scope.previewStyle == .xojit: return cbc.scope.namespace.parseLiteralString("___debug_blank_executor_main") @@ -835,7 +865,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec // Select the driver to use based on the input file types, replacing the value computed by commandLineFromTemplate(). let usedCXX = usedTools.values.contains(where: { $0.contains(where: { $0.languageDialect?.isPlusPlus ?? false }) }) - commandLine[0] = await resolveExecutablePath(cbc, computeLinkerPath(cbc, usedCXX: usedCXX), delegate: delegate).str + commandLine[0] = await resolveExecutablePath(cbc, computeLinkerPath(cbc, usedCXX: usedCXX, lookup: linkerDriverLookup), delegate: delegate).str let entitlementsSection = cbc.scope.evaluate(BuiltinMacros.LD_ENTITLEMENTS_SECTION) if !entitlementsSection.isEmpty { @@ -1105,31 +1135,45 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec ] } - private func computeLinkerPath(_ cbc: CommandBuildContext, usedCXX: Bool) -> Path { + public override func computeExecutablePath(_ cbc: CommandBuildContext) -> String { + // Placeholder fallback + return cbc.producer.hostOperatingSystem.imageFormat.executableName(basename: "clang") + } + + public func computeLinkerPath(_ cbc: CommandBuildContext, usedCXX: Bool, lookup: @escaping ((MacroDeclaration) -> MacroStringExpression?)) -> Path { if usedCXX { - let perArchValue = cbc.scope.evaluate(BuiltinMacros.PER_ARCH_LDPLUSPLUS) + let perArchValue = cbc.scope.evaluate(BuiltinMacros.PER_ARCH_LDPLUSPLUS, lookup: lookup) if !perArchValue.isEmpty { - return Path(perArchValue) + return Path(cbc.producer.hostOperatingSystem.imageFormat.executableName(basename: perArchValue)) } - let value = cbc.scope.evaluate(BuiltinMacros.LDPLUSPLUS) + let value = cbc.scope.evaluate(BuiltinMacros.LDPLUSPLUS, lookup: lookup) if !value.isEmpty { - return Path(value) + return Path(cbc.producer.hostOperatingSystem.imageFormat.executableName(basename: value)) } - - return Path("clang++") } else { - let perArchValue = cbc.scope.evaluate(BuiltinMacros.PER_ARCH_LD) + let perArchValue = cbc.scope.evaluate(BuiltinMacros.PER_ARCH_LD, lookup: lookup) if !perArchValue.isEmpty { - return Path(perArchValue) + return Path(cbc.producer.hostOperatingSystem.imageFormat.executableName(basename: perArchValue)) } - let value = cbc.scope.evaluate(BuiltinMacros.LD) + let value = cbc.scope.evaluate(BuiltinMacros.LD, lookup: lookup) if !value.isEmpty { - return Path(value) + return Path(cbc.producer.hostOperatingSystem.imageFormat.executableName(basename: value)) } + } - return Path(computeExecutablePath(cbc)) + switch cbc.scope.evaluate(BuiltinMacros.LINKER_DRIVER, lookup: lookup) { + case .clang: + if usedCXX { + return Path(cbc.producer.hostOperatingSystem.imageFormat.executableName(basename: "clang++")) + } else { + return Path(cbc.producer.hostOperatingSystem.imageFormat.executableName(basename: "clang")) + } + case .swiftc: + return Path(cbc.producer.hostOperatingSystem.imageFormat.executableName(basename: "swiftc")) + case .auto: + preconditionFailure("LINKER_DRIVER was expected to be bound to a concrete value") } } diff --git a/Tests/SWBCoreTests/CommandLineSpecTests.swift b/Tests/SWBCoreTests/CommandLineSpecTests.swift index 69419765..826e08ab 100644 --- a/Tests/SWBCoreTests/CommandLineSpecTests.swift +++ b/Tests/SWBCoreTests/CommandLineSpecTests.swift @@ -1560,7 +1560,7 @@ import SWBMacro } // Check with just LD. - for (name, expected) in [("file.c", "SomeCLinker"), ("file.cpp", "clang++")] { + for (name, expected) in [("file.c", core.hostOperatingSystem.imageFormat.executableName(basename:"SomeCLinker")), ("file.cpp", core.hostOperatingSystem.imageFormat.executableName(basename:"clang++"))] { try await check(name: name, expectedLinker: expected, macros: [ BuiltinMacros.LD: "SomeCLinker" // NOTE: One wonders whether this shouldn't change the C++ linker. @@ -1568,7 +1568,7 @@ import SWBMacro } // Check with LD & LDPLUSPLUS. - for (name, expected) in [("file.c", "SomeCLinker"), ("file.cpp", "SomeC++Linker")] { + for (name, expected) in [("file.c", core.hostOperatingSystem.imageFormat.executableName(basename:"SomeCLinker")), ("file.cpp", core.hostOperatingSystem.imageFormat.executableName(basename:"SomeC++Linker"))] { try await check(name: name, expectedLinker: expected, macros: [ BuiltinMacros.LD: "SomeCLinker", BuiltinMacros.LDPLUSPLUS: "SomeC++Linker" @@ -1576,7 +1576,7 @@ import SWBMacro } // Check with arch specific LD. - for (name, expected) in [("file.c", "SomeCLinker_x86_64"), ("file.cpp", "SomeC++Linker_x86_64")] { + for (name, expected) in [("file.c", core.hostOperatingSystem.imageFormat.executableName(basename:"SomeCLinker_x86_64")), ("file.cpp", core.hostOperatingSystem.imageFormat.executableName(basename:"SomeC++Linker_x86_64"))] { try await check(name: name, expectedLinker: expected, macros: [ BuiltinMacros.CURRENT_ARCH: "x86_64", try core.specRegistry.internalMacroNamespace.declareStringMacro("LD_x86_64"): "SomeCLinker_x86_64", diff --git a/Tests/SWBTaskConstructionTests/LinkerTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/LinkerTaskConstructionTests.swift new file mode 100644 index 00000000..48b02778 --- /dev/null +++ b/Tests/SWBTaskConstructionTests/LinkerTaskConstructionTests.swift @@ -0,0 +1,97 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Testing + +import SWBCore +import SWBTaskConstruction +import SWBTestSupport +import SWBUtil + +@Suite +fileprivate struct LinkerTaskConstructionTests: CoreBasedTests { + @Test(.requireSDKs(.host)) + func linkerDriverSelection() async throws { + let testProject = TestProject( + "aProject", + groupTree: TestGroup( + "SomeFiles", + children: [ + TestFile("c.c"), + TestFile("cxx.cpp"), + TestFile("s.swift"), + ]), + buildConfigurations: [ + TestBuildConfiguration("Debug", buildSettings: [ + "PRODUCT_NAME": "$(TARGET_NAME)", + "SWIFT_EXEC": try await swiftCompilerPath.str, + "SWIFT_VERSION": try await swiftVersion + ]), + ], + targets: [ + TestStandardTarget( + "Library", + type: .dynamicLibrary, + buildConfigurations: [ + TestBuildConfiguration("Debug", buildSettings: [:]), + ], + buildPhases: [ + TestSourcesBuildPhase(["c.c", "cxx.cpp", "s.swift"]), + ] + ), + ]) + let core = try await getCore() + let tester = try TaskConstructionTester(core, testProject) + + await tester.checkBuild(BuildParameters(configuration: "Debug", overrides: [:]), runDestination: .host) { results in + results.checkNoDiagnostics() + results.checkTask(.matchRuleType("Ld")) { task in + task.checkCommandLineMatches([.contains("clang++"), .anySequence]) + } + } + + await tester.checkBuild(BuildParameters(configuration: "Debug", overrides: ["EXCLUDED_SOURCE_FILE_NAMES": "cxx.cpp"]), runDestination: .host) { results in + results.checkNoDiagnostics() + results.checkTask(.matchRuleType("Ld")) { task in + task.checkCommandLineMatches([.contains("clang"), .anySequence]) + } + } + + await tester.checkBuild(BuildParameters(configuration: "Debug", overrides: ["LINKER_DRIVER": "swiftc"]), runDestination: .host) { results in + results.checkNoDiagnostics() + results.checkTask(.matchRuleType("Ld")) { task in + task.checkCommandLineMatches([.contains("swiftc"), .anySequence]) + } + } + + await tester.checkBuild(BuildParameters(configuration: "Debug", overrides: ["LINKER_DRIVER": "auto"]), runDestination: .host) { results in + results.checkNoDiagnostics() + results.checkTask(.matchRuleType("Ld")) { task in + task.checkCommandLineMatches([.contains("swiftc"), .anySequence]) + } + } + + await tester.checkBuild(BuildParameters(configuration: "Debug", overrides: ["LINKER_DRIVER": "auto", "EXCLUDED_SOURCE_FILE_NAMES": "s.swift"]), runDestination: .host) { results in + results.checkNoDiagnostics() + results.checkTask(.matchRuleType("Ld")) { task in + task.checkCommandLineMatches([.contains("clang++"), .anySequence]) + } + } + + await tester.checkBuild(BuildParameters(configuration: "Debug", overrides: ["LINKER_DRIVER": "auto", "EXCLUDED_SOURCE_FILE_NAMES": "s.swift cxx.cpp"]), runDestination: .host) { results in + results.checkNoDiagnostics() + results.checkTask(.matchRuleType("Ld")) { task in + task.checkCommandLineMatches([.contains("clang"), .anySequence]) + } + } + } +} From 51fc69ec782725ec2f002ce400e5fadcf37d97af Mon Sep 17 00:00:00 2001 From: Stephen Verderame Date: Mon, 30 Jun 2025 18:54:23 -0700 Subject: [PATCH 54/54] Add Metal 4 Deployment Target Adds the Metal4 deployment target for MSL. --- Sources/SWBApplePlatform/Specs/MetalCompiler.xcspec | 3 +++ Sources/SWBApplePlatform/Specs/MetalLinker.xcspec | 1 + .../Specs/en.lproj/com.apple.compilers.metal.strings | 2 ++ .../BuildToolTaskConstructionTests.swift | 4 ++++ 4 files changed, 10 insertions(+) diff --git a/Sources/SWBApplePlatform/Specs/MetalCompiler.xcspec b/Sources/SWBApplePlatform/Specs/MetalCompiler.xcspec index b697a333..67c10a8e 100644 --- a/Sources/SWBApplePlatform/Specs/MetalCompiler.xcspec +++ b/Sources/SWBApplePlatform/Specs/MetalCompiler.xcspec @@ -252,6 +252,7 @@ Metal30, Metal31, Metal32, + Metal40, ); Category = BuildOptions; }, @@ -272,6 +273,7 @@ Metal30, Metal31, Metal32, + Metal40, ); CommandLineArgs = { UseDeploymentTarget = ( ); @@ -286,6 +288,7 @@ Metal30 = ( "-std=metal3.0", ); Metal31 = ( "-std=metal3.1", ); Metal32 = ( "-std=metal3.2", ); + Metal40 = ( "-std=metal4.0", ); }; }, { diff --git a/Sources/SWBApplePlatform/Specs/MetalLinker.xcspec b/Sources/SWBApplePlatform/Specs/MetalLinker.xcspec index cefed1ad..57359ef7 100644 --- a/Sources/SWBApplePlatform/Specs/MetalLinker.xcspec +++ b/Sources/SWBApplePlatform/Specs/MetalLinker.xcspec @@ -83,6 +83,7 @@ Metal30 = ( "-std=metal3.0", ); Metal31 = ( "-std=metal3.1", ); Metal32 = ( "-std=metal3.2", ); + Metal40 = ( "-std=metal4.0", ); }; }, { diff --git a/Sources/SWBApplePlatform/Specs/en.lproj/com.apple.compilers.metal.strings b/Sources/SWBApplePlatform/Specs/en.lproj/com.apple.compilers.metal.strings index 150c27d7..565687a1 100644 --- a/Sources/SWBApplePlatform/Specs/en.lproj/com.apple.compilers.metal.strings +++ b/Sources/SWBApplePlatform/Specs/en.lproj/com.apple.compilers.metal.strings @@ -90,6 +90,8 @@ "[MTL_LANGUAGE_REVISION]-description-[Metal31]" = "Metal 3.1"; "[MTL_LANGUAGE_REVISION]-value-[Metal32]" = "Metal 3.2"; "[MTL_LANGUAGE_REVISION]-description-[Metal32]" = "Metal 3.2"; +"[MTL_LANGUAGE_REVISION]-value-[Metal40]" = "Metal 4.0"; +"[MTL_LANGUAGE_REVISION]-description-[Metal40]" = "Metal 4.0"; "[MTL_ENABLE_DEBUG_INFO]-name" = "Produce Debugging Information"; "[MTL_ENABLE_DEBUG_INFO]-description" = "Debugging information is required for shader debugging and profiling."; diff --git a/Tests/SWBTaskConstructionTests/BuildToolTaskConstructionTests.swift b/Tests/SWBTaskConstructionTests/BuildToolTaskConstructionTests.swift index 5bcdaf39..38f8ed00 100644 --- a/Tests/SWBTaskConstructionTests/BuildToolTaskConstructionTests.swift +++ b/Tests/SWBTaskConstructionTests/BuildToolTaskConstructionTests.swift @@ -1567,6 +1567,7 @@ fileprivate struct BuildToolTaskConstructionTests: CoreBasedTests { ["macosx", "Metal30"] : "-std=metal3.0", ["macosx", "Metal31"] : "-std=metal3.1", ["macosx", "Metal32"] : "-std=metal3.2", + ["macosx", "Metal40"] : "-std=metal4.0", ["iphoneos", "UseDeploymentTarget"] : "", ["iphoneos", "iOSMetal10"] : "-std=ios-metal1.0", @@ -1580,6 +1581,7 @@ fileprivate struct BuildToolTaskConstructionTests: CoreBasedTests { ["iphoneos", "Metal30"] : "-std=metal3.0", ["iphoneos", "Metal31"] : "-std=metal3.1", ["iphoneos", "Metal32"] : "-std=metal3.2", + ["iphoneos", "Metal40"] : "-std=metal4.0", ["appletvos", "UseDeploymentTarget"] : "", ["appletvos", "Metal11"] : "-std=ios-metal1.1", @@ -1592,10 +1594,12 @@ fileprivate struct BuildToolTaskConstructionTests: CoreBasedTests { ["appletvos", "Metal30"] : "-std=metal3.0", ["appletvos", "Metal31"] : "-std=metal3.1", ["appletvos", "Metal32"] : "-std=metal3.2", + ["appletvos", "Metal40"] : "-std=metal4.0", ["xros", "UseDeploymentTarget"] : "", ["xros", "Metal31"] : "-std=metal3.1", ["xros", "Metal32"] : "-std=metal3.2", + ["xros", "Metal40"] : "-std=metal4.0", ] for (language, expectedOption) in optionForLanguage {