Validate MODULE_DEPENDENCIES using tracing from Clang

This currently only checks Clang compilations with modules because there
were some issues with introducing a new task action for non-modular Clang
compilations.

rdar://150313957

Author: bob-wilson

Sources/SWBCore/Dependencies.swift

@@ -80,6 +80,48 @@ public struct ModuleDependenciesContext: Sendable, SerializableCodable {
         self.init(validate: validate, moduleDependencies: settings.moduleDependencies, fixItContext: fixItContext)
     }
 
+    // For Clang imports, we do not get the import locations and do not know whether it was a public
+    // import (which depends on whether the import is in an installed header file).
+    public func makeDiagnostics(files: [Path]) -> [Diagnostic] {
+        guard validate != .no else { return [] }
+
+        // The following is a provisional/incomplete mechanism for resolving a module dependency from a file path.
+        // For now, just grab the framework name and assume there is a module with the same name.
+        func findFrameworkName(_ file: Path) -> String? {
+            if file.fileExtension == "framework" {
+                return file.basenameWithoutSuffix
+            }
+            return file.dirname.isEmpty || file.dirname.isRoot ? nil : findFrameworkName(file.dirname)
+        }
+
+        let moduleDependencyNames = moduleDependencies.map { $0.name }
+        let fileNames = files.compactMap { findFrameworkName($0) }
+        let missingDeps = fileNames.filter {
+            return !moduleDependencyNames.contains($0)
+        }.map {
+            ModuleDependency(name: $0, accessLevel: .Private)
+        }
+
+        guard !missingDeps.isEmpty else { return [] }
+
+        let behavior: Diagnostic.Behavior = validate == .yesError ? .error : .warning
+
+        let fixIt = fixItContext?.makeFixIt(newModules: missingDeps)
+        let fixIts = fixIt.map { [$0] } ?? []
+
+        let message = "Missing entries in \(BuiltinMacros.MODULE_DEPENDENCIES.name): \(missingDeps.map { $0.asBuildSettingEntryQuotedIfNeeded }.sorted().joined(separator: " "))"
+
+        let location: Diagnostic.Location = fixIt.map {
+            Diagnostic.Location.path($0.sourceRange.path, line: $0.sourceRange.endLine, column: $0.sourceRange.endColumn)
+        } ?? Diagnostic.Location.buildSetting(BuiltinMacros.MODULE_DEPENDENCIES)
+
+        return [Diagnostic(
+            behavior: behavior,
+            location: location,
+            data: DiagnosticData(message),
+            fixIts: fixIts)]
+    }
+
     /// Nil `imports` means the current toolchain doesn't have the features to gather imports. This is temporarily required to support running against older toolchains.
     public func makeDiagnostics(imports: [(ModuleDependency, importLocations: [Diagnostic.Location])]?) -> [Diagnostic] {
         guard validate != .no else { return [] }
@@ -181,4 +223,9 @@ public struct ModuleDependenciesContext: Sendable, SerializableCodable {
             return Diagnostic.FixIt(sourceRange: sourceRange, newText: newText)
         }
     }
+
+    func signatureData() -> String {
+        let moduleNames = moduleDependencies.map { $0.name }
+        return "validate:\(validate),modules:\(moduleNames.joined(separator: ":"))"
+    }
 }

Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift

@@ -432,7 +432,10 @@ public struct ClangTaskPayload: ClangModuleVerifierPayloadType, DependencyInfoEd
 
     public let fileNameMapPath: Path?
 
-    fileprivate init(serializedDiagnosticsPath: Path?, indexingPayload: ClangIndexingPayload?, explicitModulesPayload: ClangExplicitModulesPayload? = nil, outputObjectFilePath: Path? = nil, fileNameMapPath: Path? = nil, developerPathString: String? = nil) {
+    public let moduleDependenciesContext: ModuleDependenciesContext?
+    public let traceFile: Path?
+
+    fileprivate init(serializedDiagnosticsPath: Path?, indexingPayload: ClangIndexingPayload?, explicitModulesPayload: ClangExplicitModulesPayload? = nil, outputObjectFilePath: Path? = nil, fileNameMapPath: Path? = nil, developerPathString: String? = nil, moduleDependenciesContext: ModuleDependenciesContext? = nil, traceFile: Path? = nil) {
         if let developerPathString, explicitModulesPayload == nil {
             self.dependencyInfoEditPayload = .init(removablePaths: [], removableBasenames: [], developerPath: Path(developerPathString))
         } else {
@@ -443,27 +446,33 @@ public struct ClangTaskPayload: ClangModuleVerifierPayloadType, DependencyInfoEd
         self.explicitModulesPayload = explicitModulesPayload
         self.outputObjectFilePath = outputObjectFilePath
         self.fileNameMapPath = fileNameMapPath
+        self.moduleDependenciesContext = moduleDependenciesContext
+        self.traceFile = traceFile
     }
 
     public func serialize<T: Serializer>(to serializer: T) {
-        serializer.serializeAggregate(6) {
+        serializer.serializeAggregate(8) {
             serializer.serialize(serializedDiagnosticsPath)
             serializer.serialize(indexingPayload)
             serializer.serialize(explicitModulesPayload)
             serializer.serialize(outputObjectFilePath)
             serializer.serialize(fileNameMapPath)
             serializer.serialize(dependencyInfoEditPayload)
+            serializer.serialize(moduleDependenciesContext)
+            serializer.serialize(traceFile)
         }
     }
 
     public init(from deserializer: any Deserializer) throws {
-        try deserializer.beginAggregate(6)
+        try deserializer.beginAggregate(8)
         self.serializedDiagnosticsPath = try deserializer.deserialize()
         self.indexingPayload = try deserializer.deserialize()
         self.explicitModulesPayload = try deserializer.deserialize()
         self.outputObjectFilePath = try deserializer.deserialize()
         self.fileNameMapPath = try deserializer.deserialize()
         self.dependencyInfoEditPayload = try deserializer.deserialize()
+        self.moduleDependenciesContext = try deserializer.deserialize()
+        self.traceFile = try deserializer.deserialize()
     }
 }
 
@@ -1156,6 +1165,22 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
             dependencyData = nil
         }
 
+        let moduleDependenciesContext = cbc.producer.moduleDependenciesContext
+        let traceFile: Path?
+        if clangInfo?.hasFeature("print-headers-direct-per-file") ?? false,
+            (moduleDependenciesContext?.validate ?? .defaultValue) != .no {
+            let file = Path(outputNode.path.str + ".trace.json")
+            commandLine += [
+                "-Xclang", "-header-include-file",
+                "-Xclang", file.str,
+                "-Xclang", "-header-include-filtering=direct-per-file",
+                "-Xclang", "-header-include-format=json"
+            ]
+            traceFile = file
+        } else {
+            traceFile = nil
+        }
+
         // Add the diagnostics serialization flag.  We currently place the diagnostics file right next to the output object file.
         let diagFilePath: Path?
         if let serializedDiagnosticsOptions = self.serializedDiagnosticsOptions(scope: cbc.scope, outputPath: outputNode.path) {
@@ -1266,7 +1291,9 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
             explicitModulesPayload: explicitModulesPayload,
             outputObjectFilePath: shouldGenerateRemarks ? outputNode.path : nil,
             fileNameMapPath: verifierPayload?.fileNameMapPath,
-            developerPathString: recordSystemHeaderDepsOutsideSysroot ? cbc.scope.evaluate(BuiltinMacros.DEVELOPER_DIR).str : nil
+            developerPathString: recordSystemHeaderDepsOutsideSysroot ? cbc.scope.evaluate(BuiltinMacros.DEVELOPER_DIR).str : nil,
+            moduleDependenciesContext: moduleDependenciesContext,
+            traceFile: traceFile
         )
 
         var inputNodes: [any PlannedNode] = inputDeps.map { delegate.createNode($0) }
@@ -1316,8 +1343,12 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
             extraInputs = []
         }
 
+        if let moduleDependenciesContext {
+            additionalSignatureData += moduleDependenciesContext.signatureData()
+        }
+
         // Finally, create the task.
-        delegate.createTask(type: self, dependencyData: dependencyData, payload: payload, ruleInfo: ruleInfo, additionalSignatureData: additionalSignatureData, commandLine: commandLine, additionalOutput: additionalOutput, environment: environmentBindings, workingDirectory: compilerWorkingDirectory(cbc), inputs: inputNodes + extraInputs, outputs: [outputNode], action: action ?? delegate.taskActionCreationDelegate.createDeferredExecutionTaskActionIfRequested(userPreferences: cbc.producer.userPreferences), execDescription: resolveExecutionDescription(cbc, delegate), enableSandboxing: enableSandboxing, additionalTaskOrderingOptions: [.compilationForIndexableSourceFile], usesExecutionInputs: usesExecutionInputs, showEnvironment: true, priority: .preferred)
+        delegate.createTask(type: self, dependencyData: dependencyData, payload: payload, ruleInfo: ruleInfo, additionalSignatureData: additionalSignatureData, commandLine: commandLine, additionalOutput: additionalOutput, environment: environmentBindings, workingDirectory: compilerWorkingDirectory(cbc), inputs: inputNodes + extraInputs, outputs: [outputNode], action: action ?? delegate.taskActionCreationDelegate.createClangNonModularCompileTaskAction(), execDescription: resolveExecutionDescription(cbc, delegate), enableSandboxing: enableSandboxing, additionalTaskOrderingOptions: [.compilationForIndexableSourceFile], usesExecutionInputs: usesExecutionInputs, showEnvironment: true, priority: .preferred)
 
         // If the object file verifier is enabled and we are building with explicit modules, also create a job to produce adjacent objects using implicit modules, then compare the results.
         if cbc.scope.evaluate(BuiltinMacros.CLANG_ENABLE_EXPLICIT_MODULES_OBJECT_FILE_VERIFIER) && action != nil {

Sources/SWBCore/ToolInfo/ClangToolInfo.swift

@@ -38,6 +38,7 @@ public struct DiscoveredClangToolSpecInfo: DiscoveredCommandLineToolSpecInfo {
         case wSystemHeadersInModule = "Wsystem-headers-in-module"
         case extractAPISupportsCPlusPlus = "extract-api-supports-cpp"
         case deploymentTargetEnvironmentVariables = "deployment-target-environment-variables"
+        case printHeadersDirectPerFile = "print-headers-direct-per-file"
     }
     public var toolFeatures: ToolFeatures<FeatureFlag>
     public func hasFeature(_ feature: String) -> Bool {
@@ -46,6 +47,10 @@ public struct DiscoveredClangToolSpecInfo: DiscoveredCommandLineToolSpecInfo {
         if feature == FeatureFlag.extractAPISupportsCPlusPlus.rawValue {
             return clangVersion > Version(17)
         }
+        // FIXME: Remove once the feature flag is added to clang.
+        if feature == FeatureFlag.printHeadersDirectPerFile.rawValue, let clangVersion {
+            return clangVersion >= Version(1700, 3, 10, 2)
+        }
         return toolFeatures.has(feature)
     }
 

Sources/SWBTaskExecution/TaskActions/ClangCompileTaskAction.swift

@@ -248,6 +248,20 @@ public final class ClangCompileTaskAction: TaskAction, BuildValueValidatingTaskA
                 casDBs = nil
             }
 
+            // Check if verifying dependencies from trace data is enabled.
+            var traceFile: Path? = nil
+            var moduleDependenciesContext: ModuleDependenciesContext? = nil
+            if let payload = task.payload as? ClangTaskPayload {
+                traceFile = payload.traceFile
+                moduleDependenciesContext = payload.moduleDependenciesContext
+            }
+            if let traceFile {
+                // Remove the trace output file if it already exists.
+                if executionDelegate.fs.exists(traceFile) {
+                    try executionDelegate.fs.remove(traceFile)
+                }
+            }
+
             var lastResult: CommandResult? = nil
             for command in dependencyInfo.commands {
                 if let casDBs {
@@ -304,6 +318,24 @@ public final class ClangCompileTaskAction: TaskAction, BuildValueValidatingTaskA
                     return lastResult ?? .failed
                 }
             }
+
+            if let moduleDependenciesContext, let traceFile, lastResult == .succeeded {
+                // Verify the dependencies from the trace data.
+                let fs = executionDelegate.fs
+                let traceData = try JSONDecoder().decode(Array<TraceData>.self, from: fs.readMemoryMapped(traceFile))
+
+                var allFiles = Set<Path>()
+                traceData.forEach { allFiles.formUnion(Set($0.includes)) }
+                let diagnostics = moduleDependenciesContext.makeDiagnostics(files: Array(allFiles))
+                for diagnostic in diagnostics {
+                    outputDelegate.emit(diagnostic)
+                }
+
+                if diagnostics.contains(where: { $0.behavior == .error }) {
+                    return .failed
+                }
+            }
+
             return lastResult ?? .failed
         } catch {
             outputDelegate.emitError("\(error)")
@@ -431,3 +463,10 @@ public final class ClangCompileTaskAction: TaskAction, BuildValueValidatingTaskA
         )
     }
 }
+
+// Results from tracing header includes with "direct-per-file" filtering.
+// This is used to validate dependencies.
+fileprivate struct TraceData: Decodable {
+    let source: Path
+    let includes: [Path]
+}

Sources/SWBTestSupport/CoreBasedTests.swift

@@ -147,6 +147,10 @@ extension CoreBasedTests {
             if clangInfo.clangVersion > Version(17) {
                 realToolFeatures.insert(.extractAPISupportsCPlusPlus)
             }
+            if let clangVersion = clangInfo.clangVersion, clangVersion >= Version(1700, 3, 10, 2) {
+                realToolFeatures.insert(.printHeadersDirectPerFile)
+            }
+
             return ToolFeatures(realToolFeatures)
         }
     }

Tests/SWBBuildSystemTests/DependencyVerificationBuildOperationTests.swift

@@ -0,0 +1,99 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2025 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Testing
+
+import SWBCore
+import SWBTestSupport
+@_spi(Testing) import SWBUtil
+
+import SWBTaskExecution
+import SWBProtocol
+
+@Suite(.requireSDKs(.macOS))
+fileprivate struct DependencyVerificationBuildOperationTests: CoreBasedTests {
+
+    func canVerifyDeclaredDependencies() async throws {
+        try await withTemporaryDirectory { tmpDirPath async throws -> Void in
+            let testWorkspace = TestWorkspace(
+                "Test",
+                sourceRoot: tmpDirPath.join("Test"),
+                projects: [
+                    TestProject(
+                        "aProject",
+                        groupTree: TestGroup(
+                            "Sources", path: "Sources",
+                            children: [
+                                TestFile("CoreFoo.m")
+                            ]),
+                        buildConfigurations: [
+                            TestBuildConfiguration(
+                                "Debug",
+                                buildSettings: [
+                                    "PRODUCT_NAME": "$(TARGET_NAME)",
+                                    "CLANG_ENABLE_MODULES": "YES",
+                                    "GENERATE_INFOPLIST_FILE": "YES",
+                                    "MODULE_DEPENDENCIES": "Foundation",
+                                    "VALIDATE_MODULE_DEPENDENCIES": "YES_ERROR",
+                                    // Disable the SetOwnerAndGroup action by setting them to empty values.
+                                    "INSTALL_GROUP": "",
+                                    "INSTALL_OWNER": "",
+                                ]
+                            )
+                        ],
+                        targets: [
+                            TestStandardTarget(
+                                "CoreFoo", type: .framework,
+                                buildPhases: [
+                                    TestSourcesBuildPhase(["CoreFoo.m"]),
+                                    TestFrameworksBuildPhase()
+                                ])
+                        ])
+                ]
+            )
+
+            let tester = try await BuildOperationTester(getCore(), testWorkspace, simulated: false)
+            let SRCROOT = testWorkspace.sourceRoot.join("aProject")
+
+            // Write the source files.
+            try await tester.fs.writeFileContents(SRCROOT.join("Sources/CoreFoo.m")) { contents in
+                contents <<< """
+                    #include <Foundation/Foundation.h>
+                    #include <Accelerate/Accelerate.h>
+
+                    void f0(void) { };
+                """
+            }
+
+            func parameters(_ overrides: [String: String] = [:]) -> BuildParameters {
+                return BuildParameters(
+                    action: .install, configuration: "Debug",
+                    overrides: [
+                        "DSTROOT": tmpDirPath.join("dst").str
+                    ].merging(overrides, uniquingKeysWith: { _, new in new })
+                )
+            }
+
+            if try await clangFeatures.has(.printHeadersDirectPerFile) {
+                // Expect complaint about undeclared dependency
+                try await tester.checkBuild(parameters: parameters(), runDestination: .macOS, persistent: true) { results in
+                    results.checkError(.contains("Missing entries in MODULE_DEPENDENCIES: Accelerate"))
+                }
+
+                // Declaring dependencies resolves the problem
+                try await tester.checkBuild(parameters: parameters(["MODULE_DEPENDENCIES": "Foundation Accelerate"]), runDestination: .macOS, persistent: true) { results in
+                    results.checkNoErrors()
+                }
+            }
+        }
+    }
+}