Allow logical dependencies to be forward-declared and verified at use-time

This is a skeletal implementation scoped to support a specific functional milestone.

Author: Luke Daley

Sources/SWBCore/Dependencies.swift

@@ -0,0 +1,91 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2025 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+public import SWBUtil
+public import SWBMacro
+
+// Needed to resolve BuiltinMacros … why?
+import SWBCore
+
+import Foundation
+
+// Global/target dependency settings
+public struct DependencySettings : Serializable, Sendable, Encodable {
+    public let dependencies: OrderedSet<String>
+    public let verification: Bool
+
+
+    public init(_ scope: MacroEvaluationScope) {
+        self.init(
+            dependencies: scope.evaluate(BuiltinMacros.DEPENDENCIES),
+            verification: scope.evaluate(BuiltinMacros.DEPENDENCIES_VERIFICATION)
+        )
+    }
+
+    public init(
+        dependencies: [String],
+        verification: Bool
+    ) {
+        self.dependencies = OrderedSet(dependencies)
+        self.verification = verification
+    }
+
+    public func serialize<T: Serializer>(to serializer: T) {
+        serializer.serializeAggregate(2) {
+            serializer.serialize(dependencies)
+            serializer.serialize(verification)
+        }
+    }
+
+    public init(from deserializer: any Deserializer) throws {
+        try deserializer.beginAggregate(2)
+        self.dependencies = try deserializer.deserialize()
+        self.verification = try deserializer.deserialize()
+    }
+
+}
+
+// Task-specific settings
+public struct TaskDependencySettings : Serializable, Sendable, Encodable {
+
+    public let traceFile: Path
+    public let dependencySettings: DependencySettings
+
+    init(traceFile: Path, dependencySettings: DependencySettings) {
+        assert(!traceFile.isEmpty, "traceFile should never be empty")
+        self.traceFile = traceFile
+        self.dependencySettings = dependencySettings
+    }
+
+    public func serialize<T: Serializer>(to serializer: T) {
+        serializer.serializeAggregate(2) {
+            serializer.serialize(traceFile)
+            serializer.serialize(dependencySettings)
+        }
+    }
+
+    public init(from deserializer: any Deserializer) throws {
+        try deserializer.beginAggregate(2)
+        self.traceFile = try deserializer.deserialize()
+        self.dependencySettings = try deserializer.deserialize()
+    }
+
+    func signatureData() -> String {
+        return "verify:\(dependencySettings.verification),deps:\(dependencySettings.dependencies.joined(separator: ":"))"
+    }
+
+}
+
+// Protocol for task payloads
+public protocol TaskDependencySettingsPayload {
+    var taskDependencySettings: TaskDependencySettings? { get }
+}

Sources/SWBCore/PlannedTaskAction.swift

@@ -330,6 +330,7 @@ public protocol TaskActionCreationDelegate
     func createValidateProductTaskAction() -> any PlannedTaskAction
     func createConstructStubExecutorInputFileListTaskAction() -> any PlannedTaskAction
     func createClangCompileTaskAction() -> any PlannedTaskAction
+    func createClangNonModularCompileTaskAction() -> any PlannedTaskAction
     func createClangScanTaskAction() -> any PlannedTaskAction
     func createSwiftDriverTaskAction() -> any PlannedTaskAction
     func createSwiftCompilationRequirementTaskAction() -> any PlannedTaskAction
@@ -339,6 +340,7 @@ public protocol TaskActionCreationDelegate
     func createSignatureCollectionTaskAction() -> any PlannedTaskAction
     func createClangModuleVerifierInputGeneratorTaskAction() -> any PlannedTaskAction
     func createProcessSDKImportsTaskAction() -> any PlannedTaskAction
+    func createLdTaskAction() -> any PlannedTaskAction
 }
 
 extension TaskActionCreationDelegate {

Sources/SWBCore/Settings/BuiltinMacros.swift

@@ -584,6 +584,8 @@ public final class BuiltinMacros {
     public static let DEFAULT_COMPILER = BuiltinMacros.declareStringMacro("DEFAULT_COMPILER")
     public static let DEFAULT_KEXT_INSTALL_PATH = BuiltinMacros.declareStringMacro("DEFAULT_KEXT_INSTALL_PATH")
     public static let DEFINES_MODULE = BuiltinMacros.declareBooleanMacro("DEFINES_MODULE")
+    public static let DEPENDENCIES = BuiltinMacros.declareStringListMacro("DEPENDENCIES")
+    public static let DEPENDENCIES_VERIFICATION = BuiltinMacros.declareBooleanMacro("DEPENDENCIES_VERIFICATION")
     public static let DEPENDENCY_SCOPE_INCLUDES_DIRECT_DEPENDENCIES = BuiltinMacros.declareBooleanMacro("DEPENDENCY_SCOPE_INCLUDES_DIRECT_DEPENDENCIES")
     public static let DERIVE_MACCATALYST_PRODUCT_BUNDLE_IDENTIFIER = BuiltinMacros.declareBooleanMacro("DERIVE_MACCATALYST_PRODUCT_BUNDLE_IDENTIFIER")
     public static let __DIAGNOSE_DERIVE_MACCATALYST_PRODUCT_BUNDLE_IDENTIFIER_ERROR = BuiltinMacros.declareBooleanMacro("__DIAGNOSE_DERIVE_MACCATALYST_PRODUCT_BUNDLE_IDENTIFIER_ERROR")
@@ -810,6 +812,7 @@ public final class BuiltinMacros {
     public static let LD_RUNPATH_SEARCH_PATHS = BuiltinMacros.declareStringListMacro("LD_RUNPATH_SEARCH_PATHS")
     public static let LD_SDK_IMPORTS_FILE = BuiltinMacros.declarePathMacro("LD_SDK_IMPORTS_FILE")
     public static let LD_WARN_UNUSED_DYLIBS = BuiltinMacros.declareBooleanMacro("LD_WARN_UNUSED_DYLIBS")
+    public static let LD_TRACE_FILE = BuiltinMacros.declarePathMacro("LD_TRACE_FILE")
     public static let _LD_MULTIARCH = BuiltinMacros.declareBooleanMacro("_LD_MULTIARCH")
     public static let _LD_MULTIARCH_PREFIX_MAP = BuiltinMacros.declareStringListMacro("_LD_MULTIARCH_PREFIX_MAP")
     public static let LEX = BuiltinMacros.declarePathMacro("LEX")
@@ -1568,6 +1571,8 @@ public final class BuiltinMacros {
         DEFAULT_COMPILER,
         DEFAULT_KEXT_INSTALL_PATH,
         DEFINES_MODULE,
+        DEPENDENCIES,
+        DEPENDENCIES_VERIFICATION,
         DEPENDENCY_SCOPE_INCLUDES_DIRECT_DEPENDENCIES,
         DERIVE_MACCATALYST_PRODUCT_BUNDLE_IDENTIFIER,
         __DIAGNOSE_DERIVE_MACCATALYST_PRODUCT_BUNDLE_IDENTIFIER_ERROR,
@@ -1872,6 +1877,7 @@ public final class BuiltinMacros {
         LD_RUNPATH_SEARCH_PATHS,
         LD_SDK_IMPORTS_FILE,
         LD_WARN_UNUSED_DYLIBS,
+        LD_TRACE_FILE,
         _LD_MULTIARCH,
         _LD_MULTIARCH_PREFIX_MAP,
         LEGACY_DEVELOPER_DIR,

Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift

@@ -415,7 +415,7 @@ struct ClangModuleVerifierPayload: ClangModuleVerifierPayloadType {
     }
 }
 
-public struct ClangTaskPayload: ClangModuleVerifierPayloadType, DependencyInfoEditableTaskPayload, Encodable {
+public struct ClangTaskPayload: ClangModuleVerifierPayloadType, DependencyInfoEditableTaskPayload, TaskDependencySettingsPayload, Encodable {
     let dependencyInfoEditPayload: DependencyInfoEditPayload?
 
     /// The path to the serialized diagnostic output.  Every clang task must provide this path.
@@ -432,7 +432,9 @@ public struct ClangTaskPayload: ClangModuleVerifierPayloadType, DependencyInfoEd
 
     public let fileNameMapPath: Path?
 
-    fileprivate init(serializedDiagnosticsPath: Path?, indexingPayload: ClangIndexingPayload?, explicitModulesPayload: ClangExplicitModulesPayload? = nil, outputObjectFilePath: Path? = nil, fileNameMapPath: Path? = nil, developerPathString: String? = nil) {
+    public let taskDependencySettings: TaskDependencySettings?
+
+    fileprivate init(serializedDiagnosticsPath: Path?, indexingPayload: ClangIndexingPayload?, explicitModulesPayload: ClangExplicitModulesPayload? = nil, outputObjectFilePath: Path? = nil, fileNameMapPath: Path? = nil, developerPathString: String? = nil, taskDependencySettings: TaskDependencySettings? = nil) {
         if let developerPathString, explicitModulesPayload == nil {
             self.dependencyInfoEditPayload = .init(removablePaths: [], removableBasenames: [], developerPath: Path(developerPathString))
         } else {
@@ -443,16 +445,18 @@ public struct ClangTaskPayload: ClangModuleVerifierPayloadType, DependencyInfoEd
         self.explicitModulesPayload = explicitModulesPayload
         self.outputObjectFilePath = outputObjectFilePath
         self.fileNameMapPath = fileNameMapPath
+        self.taskDependencySettings = taskDependencySettings
     }
 
     public func serialize<T: Serializer>(to serializer: T) {
-        serializer.serializeAggregate(6) {
+        serializer.serializeAggregate(7) {
             serializer.serialize(serializedDiagnosticsPath)
             serializer.serialize(indexingPayload)
             serializer.serialize(explicitModulesPayload)
             serializer.serialize(outputObjectFilePath)
             serializer.serialize(fileNameMapPath)
             serializer.serialize(dependencyInfoEditPayload)
+            serializer.serialize(taskDependencySettings)
         }
     }
 
@@ -464,6 +468,7 @@ public struct ClangTaskPayload: ClangModuleVerifierPayloadType, DependencyInfoEd
         self.outputObjectFilePath = try deserializer.deserialize()
         self.fileNameMapPath = try deserializer.deserialize()
         self.dependencyInfoEditPayload = try deserializer.deserialize()
+        self.taskDependencySettings = try deserializer.deserialize()
     }
 }
 
@@ -1158,6 +1163,24 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
             dependencyData = nil
         }
 
+        let taskDependencySettings = TaskDependencySettings(
+            traceFile: Path(outputNode.path.str + ".trace.json"),
+            dependencySettings: DependencySettings(cbc.scope)
+        )
+
+        if (taskDependencySettings.dependencySettings.verification) {
+            commandLine += [
+                "-Xclang",
+                "-header-include-file",
+                "-Xclang",
+                taskDependencySettings.traceFile.str,
+                "-Xclang",
+                "-header-include-filtering=only-direct-system",
+                "-Xclang",
+                "-header-include-format=json"
+            ]
+        }
+
         // Add the diagnostics serialization flag.  We currently place the diagnostics file right next to the output object file.
         let diagFilePath: Path?
         if let serializedDiagnosticsOptions = self.serializedDiagnosticsOptions(scope: cbc.scope, outputPath: outputNode.path) {
@@ -1268,7 +1291,8 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
             explicitModulesPayload: explicitModulesPayload,
             outputObjectFilePath: shouldGenerateRemarks ? outputNode.path : nil,
             fileNameMapPath: verifierPayload?.fileNameMapPath,
-            developerPathString: recordSystemHeaderDepsOutsideSysroot ? cbc.scope.evaluate(BuiltinMacros.DEVELOPER_DIR).str : nil
+            developerPathString: recordSystemHeaderDepsOutsideSysroot ? cbc.scope.evaluate(BuiltinMacros.DEVELOPER_DIR).str : nil,
+            taskDependencySettings: taskDependencySettings
         )
 
         var inputNodes: [any PlannedNode] = inputDeps.map { delegate.createNode($0) }
@@ -1318,8 +1342,10 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
             extraInputs = []
         }
 
+        additionalSignatureData += taskDependencySettings.signatureData()
+
         // Finally, create the task.
-        delegate.createTask(type: self, dependencyData: dependencyData, payload: payload, ruleInfo: ruleInfo, additionalSignatureData: additionalSignatureData, commandLine: commandLine, additionalOutput: additionalOutput, environment: environmentBindings, workingDirectory: compilerWorkingDirectory(cbc), inputs: inputNodes + extraInputs, outputs: [outputNode], action: action ?? delegate.taskActionCreationDelegate.createDeferredExecutionTaskActionIfRequested(userPreferences: cbc.producer.userPreferences), execDescription: resolveExecutionDescription(cbc, delegate), enableSandboxing: enableSandboxing, additionalTaskOrderingOptions: [.compilationForIndexableSourceFile], usesExecutionInputs: usesExecutionInputs, showEnvironment: true, priority: .preferred)
+        delegate.createTask(type: self, dependencyData: dependencyData, payload: payload, ruleInfo: ruleInfo, additionalSignatureData: additionalSignatureData, commandLine: commandLine, additionalOutput: additionalOutput, environment: environmentBindings, workingDirectory: compilerWorkingDirectory(cbc), inputs: inputNodes + extraInputs, outputs: [outputNode], action: action ?? delegate.taskActionCreationDelegate.createClangNonModularCompileTaskAction(), execDescription: resolveExecutionDescription(cbc, delegate), enableSandboxing: enableSandboxing, additionalTaskOrderingOptions: [.compilationForIndexableSourceFile], usesExecutionInputs: usesExecutionInputs, showEnvironment: true, priority: .preferred)
 
         // If the object file verifier is enabled and we are building with explicit modules, also create a job to produce adjacent objects using implicit modules, then compare the results.
         if cbc.scope.evaluate(BuiltinMacros.CLANG_ENABLE_EXPLICIT_MODULES_OBJECT_FILE_VERIFIER) && action != nil {

Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift

@@ -52,7 +52,7 @@ struct LdLinkerTaskPreviewPayload: Serializable, Encodable {
     }
 }
 
-fileprivate struct LdLinkerTaskPayload: DependencyInfoEditableTaskPayload {
+fileprivate struct LdLinkerTaskPayload: DependencyInfoEditableTaskPayload, TaskDependencySettingsPayload {
     /// Path that points to the output linker file.
     let outputPath: Path
 
@@ -68,17 +68,21 @@ fileprivate struct LdLinkerTaskPayload: DependencyInfoEditableTaskPayload {
     /// Path to the object file emitted during LTO, used for optimization remarks.
     fileprivate let objectPathLTO: Path?
 
+    public let taskDependencySettings: TaskDependencySettings?
+
     init(
         outputPath: Path,
         dependencyInfoEditPayload: DependencyInfoEditPayload? = nil,
         previewPayload: LdLinkerTaskPreviewPayload? = nil,
         previewStyle: PreviewStyle? = nil,
-        objectPathLTO: Path? = nil
+        objectPathLTO: Path? = nil,
+        taskDependencySettings: TaskDependencySettings? = nil,
     ) {
         self.outputPath = outputPath
         self.dependencyInfoEditPayload = dependencyInfoEditPayload
         self.previewPayload = previewPayload
         self.objectPathLTO = objectPathLTO
+        self.taskDependencySettings = taskDependencySettings
         switch previewStyle {
         case .dynamicReplacement:
             self.previewStyle = .dynamicReplacement
@@ -90,22 +94,24 @@ fileprivate struct LdLinkerTaskPayload: DependencyInfoEditableTaskPayload {
     }
 
     public func serialize<T: Serializer>(to serializer: T) {
-        serializer.serializeAggregate(5) {
+        serializer.serializeAggregate(6) {
             serializer.serialize(outputPath)
             serializer.serialize(dependencyInfoEditPayload)
             serializer.serialize(previewPayload)
             serializer.serialize(objectPathLTO)
             serializer.serialize(previewStyle)
+            serializer.serialize(taskDependencySettings)
         }
     }
 
     public init(from deserializer: any Deserializer) throws {
-        try deserializer.beginAggregate(5)
+        try deserializer.beginAggregate(6)
         self.outputPath = try deserializer.deserialize()
         self.dependencyInfoEditPayload = try deserializer.deserialize()
         self.previewPayload = try deserializer.deserialize()
         self.objectPathLTO = try deserializer.deserialize()
         self.previewStyle = try deserializer.deserialize()
+        self.taskDependencySettings = try deserializer.deserialize()
     }
 }
 
@@ -664,20 +670,35 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec
             editPayload = nil
         }
 
+        let taskDependencySettings = TaskDependencySettings(
+            traceFile: cbc.scope.evaluate(BuiltinMacros.LD_TRACE_FILE),
+            dependencySettings: DependencySettings(cbc.scope)
+        )
+
+        if (taskDependencySettings.dependencySettings.verification) {
+            commandLine += [
+                "-Xlinker",
+                "-trace_file",
+                "-Xlinker",
+                taskDependencySettings.traceFile.str,
+            ]
+        }
+
         let payload = LdLinkerTaskPayload(
             outputPath: cbc.output,
             dependencyInfoEditPayload: editPayload,
             previewPayload: previewPayload,
             previewStyle: cbc.scope.previewStyle,
-            objectPathLTO: shouldGenerateRemarks ? objectPathLTO : nil
+            objectPathLTO: shouldGenerateRemarks ? objectPathLTO : nil,
+            taskDependencySettings: taskDependencySettings,
         )
 
         // Add dependencies on any directories in our input search paths for which the build system is creating those directories.
         let ldSearchPaths = Set(cbc.scope.evaluate(BuiltinMacros.FRAMEWORK_SEARCH_PATHS) + cbc.scope.evaluate(BuiltinMacros.LIBRARY_SEARCH_PATHS))
         let otherInputs = delegate.buildDirectories.sorted().compactMap { path in ldSearchPaths.contains(path.str) ? delegate.createBuildDirectoryNode(absolutePath: path) : nil } + cbc.commandOrderingInputs
 
         // Create the task.
-        delegate.createTask(type: self, dependencyData: dependencyInfo, payload: payload, ruleInfo: defaultRuleInfo(cbc, delegate), commandLine: commandLine, environment: environment, workingDirectory: cbc.producer.defaultWorkingDirectory, inputs: inputs + otherInputs, outputs: outputs, action: delegate.taskActionCreationDelegate.createDeferredExecutionTaskActionIfRequested(userPreferences: cbc.producer.userPreferences), execDescription: resolveExecutionDescription(cbc, delegate), enableSandboxing: enableSandboxing)
+        delegate.createTask(type: self, dependencyData: dependencyInfo, payload: payload, ruleInfo: defaultRuleInfo(cbc, delegate), additionalSignatureData: taskDependencySettings.signatureData(), commandLine: commandLine, environment: environment, workingDirectory: cbc.producer.defaultWorkingDirectory, inputs: inputs + otherInputs, outputs: outputs, action: delegate.taskActionCreationDelegate.createLdTaskAction(), execDescription: resolveExecutionDescription(cbc, delegate), enableSandboxing: enableSandboxing)
     }
 
     public static func addAdditionalDependenciesFromCommandLine(_ cbc: CommandBuildContext, _ commandLine: [String], _ environment: EnvironmentBindings, _ inputs: inout [any PlannedNode], _ outputs: inout [any PlannedNode], _ delegate: any TaskGenerationDelegate) {

Sources/SWBCore/Specs/CoreBuildSystem.xcspec

@@ -4532,6 +4532,15 @@ When this setting is enabled:
                 Type = Boolean;
                 DefaultValue = "NO";
             },
+            {
+                Name = "DEPENDENCIES";
+                Type = StringList;
+            },
+            {
+                Name = "DEPENDENCIES_VERIFICATION";
+                Type = Boolean;
+                DefaultValue = YES;
+            },
         );
     },
 )

Sources/SWBTaskExecution/BuildDescriptionManager.swift

@@ -856,6 +856,10 @@ extension BuildSystemTaskPlanningDelegate: TaskActionCreationDelegate {
         return ClangCompileTaskAction()
     }
 
+    func createClangNonModularCompileTaskAction() -> any PlannedTaskAction {
+        return ClangNonModularCompileTaskAction()
+    }
+
     func createClangScanTaskAction() -> any PlannedTaskAction {
         return ClangScanTaskAction()
     }
@@ -891,6 +895,10 @@ extension BuildSystemTaskPlanningDelegate: TaskActionCreationDelegate {
     func createProcessSDKImportsTaskAction() -> any PlannedTaskAction {
         return ProcessSDKImportsTaskAction()
     }
+
+    func createLdTaskAction() -> any PlannedTaskAction {
+        return LdTaskAction()
+    }
 }
 
 fileprivate extension BuildDescription {

Sources/SWBTaskExecution/BuiltinTaskActionsExtension.swift

@@ -51,7 +51,9 @@ public struct BuiltinTaskActionsExtension: TaskActionExtension {
             36: ConstructStubExecutorInputFileListTaskAction.self,
             37: ConcatenateTaskAction.self,
             38: GenericCachingTaskAction.self,
-            39: ProcessSDKImportsTaskAction.self
+            39: ProcessSDKImportsTaskAction.self,
+            40: LdTaskAction.self,
+            42: ClangNonModularCompileTaskAction.self,
         ]
     }
 }