Unify stub binary processing for XCTRunner/messages apps/watch apps

Author: owenv

Sources/SWBApplePlatform/StubBinaryTaskProducer.swift

@@ -41,7 +41,8 @@ fileprivate func generateStubBinaryTasks(stubBinarySourcePath: Path, archs: [Str
     func copyBinary(_ cbc: CommandBuildContext, executionDescription: String) async {
         // If we have no architectures, then simply copy the binary as-is
         if thinArchs && !archs.isEmpty {
-            context.lipoSpec.constructCopyAndProcessArchsTasks(cbc, delegate, executionDescription: executionDescription, archsToPreserve: archs)
+            delegate.access(path: stubBinarySourcePath)
+            context.lipoSpec.constructCopyAndProcessArchsTasks(cbc, delegate, executionDescription: executionDescription, archsToPreserve: context.availableMatchingArchitecturesInStubBinary(at: stubBinarySourcePath, requestedArchs: archs))
         } else {
             await context.copySpec.constructCopyTasks(cbc, delegate, executionDescription: executionDescription, stripUnsignedBinaries: false, stripBitcode: false)
         }

Sources/SWBCore/SpecImplementations/Tools/Lipo.swift

@@ -72,11 +72,6 @@ public final class LipoToolSpec: GenericCommandLineToolSpec, SpecIdentifierType,
         delegate.createTask(type: self, ruleInfo: ["CreateUniversalBinary", outputPath.str, variant, archsString], commandLine: commandLine, environment: EnvironmentBindings(), workingDirectory: cbc.producer.defaultWorkingDirectory, inputs: cbc.inputs.map({ delegate.createNode($0.absolutePath) }), outputs: outputs, action: delegate.taskActionCreationDelegate.createDeferredExecutionTaskActionIfRequested(userPreferences: cbc.producer.userPreferences), execDescription: resolveExecutionDescription(cbc, delegate), enableSandboxing: enableSandboxing)
     }
 
-    /// Invoke lipo to copy a fat Mach-O to a destination path with certain architectures removed.
-    public func constructCopyAndProcessArchsTasks(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, executionDescription: String? = nil, archsToRemove: [String]) {
-        return constructCopyAndProcessArchsTasks(cbc, delegate, executionDescription: executionDescription, archsToProcess: archsToRemove, flag: "-remove", ruleName: "CopyAndRemoveArchs")
-    }
-
     /// Invoke lipo to copy a fat Mach-O to a destination path with only certain architectures preserved, and the rest removed.
     public func constructCopyAndProcessArchsTasks(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, executionDescription: String? = nil, archsToPreserve: [String]) {
         return constructCopyAndProcessArchsTasks(cbc, delegate, executionDescription: executionDescription, archsToProcess: archsToPreserve, flag: "-extract", ruleName: "CopyAndPreserveArchs")

Sources/SWBTaskConstruction/TaskProducers/OtherTaskProducers/XCTestProductTypeTaskProducer.swift

@@ -81,27 +81,6 @@ final class XCTestProductPostprocessingTaskProducer: PhasedTaskProducer, TaskPro
         return tasks
     }
 
-    private func archsForXCTRunner(_ scope: MacroEvaluationScope) -> (required: Set<String>, optional: Set<String>) {
-        // The XCTestRunner is required to have at least the same ARCHS as our test bundle, minus the arm64e and/or x86_64h subtypes of their respective architecture family. The removal logic ensures we always get a link-compatible result.
-        var requiredArchs = Set(scope.evaluate(BuiltinMacros.ARCHS))
-        var optionalArchs = Set<String>()
-
-        let linkCompatibleArchPairs = [
-            ("arm64e", "arm64"),
-            ("x86_64h", "x86_64"),
-        ]
-
-        for (arch, baseline) in linkCompatibleArchPairs {
-            if requiredArchs.contains(baseline) {
-                if let foundArch = requiredArchs.remove(arch) {
-                    optionalArchs.insert(foundArch)
-                }
-            }
-        }
-
-        return (requiredArchs, optionalArchs)
-    }
-
     /// A test target which is configured to use an XCTRunner will do the following:
     ///
     /// - Copy `XCTRunner.app` out of the platform being built for into the original target build dir using the name `$(XCTRUNNER_PRODUCT_NAME)` (where `$(PRODUCT_NAME)` is the test target's product name).
@@ -218,19 +197,7 @@ final class XCTestProductPostprocessingTaskProducer: PhasedTaskProducer, TaskPro
                         continue
                     }
 
-                    let (requiredArchs, optionalArchs) = archsForXCTRunner(scope)
-                    archsToPreserve = requiredArchs
-
-                    // If we have any optional architectures, add the subset of those which the XCTRunner binary actually has
-                    if !optionalArchs.isEmpty {
-                        archsToPreserve.formUnion(optionalArchs.intersection(runnerArchs))
-                    }
-
-                    let missingArchs = archsToPreserve.subtracting(runnerArchs)
-                    if !missingArchs.isEmpty {
-                        // This is a warning instead of an error to ease future architecture bringup
-                        delegate.warning("missing architectures (\(missingArchs.sorted().joined(separator: ", "))) in XCTRunner.app: \(inputPath.str)")
-                    }
+                    archsToPreserve = Set(context.availableMatchingArchitecturesInStubBinary(at: inputPath, requestedArchs: scope.evaluate(BuiltinMacros.ARCHS)))
 
                     // If the runner has only one architecture, perform a direct file copy since lipo can't use -extract on thin Mach-Os
                     if runnerArchs.count == 1 {

Sources/SWBTaskConstruction/TaskProducers/TaskProducer.swift

@@ -818,6 +818,31 @@ public class TaskProducerContext: StaleFileRemovalContext, BuildFileResolution
         return false
     }
 
+    public func availableMatchingArchitecturesInStubBinary(at stubBinary: Path, requestedArchs: [String]) -> [String] {
+        let stubArchs: Set<String>
+        do {
+            stubArchs = try globalProductPlan.planRequest.buildRequestContext.getCachedMachOInfo(at: stubBinary).architectures
+        } catch {
+            delegate.error("unable to create tasks to copy stub binary: can't determine architectures of binary: \(stubBinary.str): \(error)")
+            return []
+        }
+        var archsToExtract: Set<String> = []
+        for arch in requestedArchs {
+            if stubArchs.contains(arch) {
+                archsToExtract.insert(arch)
+            } else {
+                let specLookupContext = SpecLookupCtxt(specRegistry: workspaceContext.core.specRegistry, platform: settings.platform)
+                let compatibilityArchs = (specLookupContext.getSpec(arch) as? ArchitectureSpec)?.compatibilityArchs
+                if let compatibleArch = compatibilityArchs?.first(where: { stubArchs.contains($0) }) {
+                    archsToExtract.insert(compatibleArch)
+                } else {
+                    delegate.warning("stub binary at '\(stubBinary.str)' does not contain a slice for '\(arch)' or a compatible architecture")
+                }
+            }
+        }
+        return archsToExtract.sorted()
+    }
+
     /// Returns true if we should emit errors when there are tasks that delay eager compilation.
     func requiresEagerCompilation(_ scope: MacroEvaluationScope) -> Bool {
         return scope.evaluate(BuiltinMacros.EAGER_COMPILATION_REQUIRE)

Tests/SWBBuildSystemTests/WatchBuildOperationTests.swift

@@ -141,9 +141,11 @@ fileprivate struct WatchBuildOperationTests: CoreBasedTests {
             try fs.createDirectory(Path("/Users/whoever/Library/MobileDevice/Provisioning Profiles"), recursive: true)
             try fs.write(Path("/Users/whoever/Library/MobileDevice/Provisioning Profiles/8db0e92c-592c-4f06-bfed-9d945841b78d.mobileprovision"), contents: "profile")
             try fs.createDirectory(core.loadSDK(.watchOS).path.join("Library/Application Support/WatchKit"), recursive: true)
-            try fs.write(core.loadSDK(.watchOS).path.join("Library/Application Support/WatchKit/WK"), contents: "WatchKitStub")
+            let stubPath = core.loadSDK(.watchOS).path.join("Library/Application Support/WatchKit/WK")
+            try fs.write(stubPath, contents: localFS.read(stubPath))
             try fs.createDirectory(core.loadSDK(.watchOSSimulator).path.join("Library/Application Support/WatchKit"), recursive: true)
-            try fs.write(core.loadSDK(.watchOSSimulator).path.join("Library/Application Support/WatchKit/WK"), contents: "WatchKitStub")
+            let simStubPath = core.loadSDK(.watchOSSimulator).path.join("Library/Application Support/WatchKit/WK")
+            try fs.write(simStubPath, contents: localFS.read(simStubPath))
 
             // Create the iOS stub assets in the wrong SDK so the build doesn't fail early due to missing inputs.
             try fs.createDirectory(core.loadSDK(.watchOS).path.join("../../../Library/Application Support/MessagesApplicationStub"), recursive: true)
@@ -157,6 +159,7 @@ fileprivate struct WatchBuildOperationTests: CoreBasedTests {
                 }
                 results.checkWarning(.contains("Could not read serialized diagnostics file"))
                 results.checkError(.equal("[targetIntegrity] Watch-Only Application Stubs are not available when building for watchOS. (in target 'Watchable' from project 'aProject')"))
+                results.checkError(.prefix("unable to create tasks to copy stub binary"))
                 results.checkWarning(.contains("Unable to perform dependency info modifications"))
                 results.checkNoDiagnostics()
             }

Tests/SWBTaskConstructionTests/PlatformTaskConstructionTests.swift

@@ -551,9 +551,11 @@ fileprivate struct PlatformTaskConstructionTests: CoreBasedTests {
         try fs.write(Path("/Users/whoever/Library/MobileDevice/Provisioning Profiles/8db0e92c-592c-4f06-bfed-9d945841b78d.mobileprovision"), contents: "profile")
         try fs.createDirectory(Path("/tmp/Test/aProject/Stickers.xcstickers/Sticker Pack.stickerpack"), recursive: true)
         try fs.createDirectory(platformPath.join("Library/Application Support/MessagesApplicationStub"), recursive: true)
-        try fs.write(platformPath.join("Library/Application Support/MessagesApplicationStub/MessagesApplicationStub"), contents: "MessagesApplicationStub")
+        let stubPath = platformPath.join("Library/Application Support/MessagesApplicationStub/MessagesApplicationStub")
+        try fs.write(stubPath, contents: localFS.read(stubPath))
         try fs.createDirectory(platformPath.join("Library/Application Support/MessagesApplicationExtensionStub"), recursive: true)
-        try fs.write(platformPath.join("Library/Application Support/MessagesApplicationExtensionStub/MessagesApplicationExtensionStub"), contents: "MessagesApplicationExtensionStub")
+        let extensionStubPath = platformPath.join("Library/Application Support/MessagesApplicationExtensionStub/MessagesApplicationExtensionStub")
+        try fs.write(extensionStubPath, contents: localFS.read(extensionStubPath))
         try fs.createDirectory(tester.workspace.projects[0].sourceRoot, recursive: true)
         try fs.write(tester.workspace.projects[0].sourceRoot.join("Info.plist"), contents: "<dict/>")
 

Tests/SWBTaskConstructionTests/UnitTestTaskConstructionTests.swift

@@ -1310,9 +1310,11 @@ fileprivate struct UnitTestTaskConstructionTests: CoreBasedTests {
             try fs.write(watchsimframeworkPath, contents: ByteString(encodingAsUTF8: watchosframeworkPath.basename))
         }
         try fs.createDirectory(core.loadSDK(.watchOS).path.join("Library/Application Support/WatchKit"), recursive: true)
-        try fs.write(core.loadSDK(.watchOS).path.join("Library/Application Support/WatchKit/WK"), contents: "WatchKitStub")
+        let stubPath = core.loadSDK(.watchOS).path.join("Library/Application Support/WatchKit/WK")
+        try fs.write(stubPath, contents: localFS.read(stubPath))
         try fs.createDirectory(core.loadSDK(.watchOSSimulator).path.join("Library/Application Support/WatchKit"), recursive: true)
-        try fs.write(core.loadSDK(.watchOSSimulator).path.join("Library/Application Support/WatchKit/WK"), contents: "WatchKitStub")
+        let simStubPath = core.loadSDK(.watchOSSimulator).path.join("Library/Application Support/WatchKit/WK")
+        try fs.write(simStubPath, contents: localFS.read(simStubPath))
 
         // We build the app target and the test target.
         let topLevelTargets = [tester.workspace.projects[0].targets[0], tester.workspace.projects[0].targets[2]]