[WIP] Add new clang build setting EMIT_COMPILER_SOURCE_METADATA that

Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift

@@ -591,10 +591,10 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
 
         required init() { }
 
-        func getStandardFlags(_ spec: ClangCompilerSpec, producer: any CommandProducer, scope: MacroEvaluationScope, optionContext: (any BuildOptionGenerationContext)?, delegate: any TaskGenerationDelegate, inputFileType: FileTypeSpec) -> ConstantFlags {
+        func getStandardFlags(_ spec: ClangCompilerSpec, cbc: CommandBuildContext, optionContext: (any BuildOptionGenerationContext)?, delegate: any TaskGenerationDelegate, inputFileType: FileTypeSpec) -> ConstantFlags {
             // This cache is per-producer, so it is guaranteed to be invariant based on that.
-            constantFlagsCache.getOrInsert(ConstantFlagsKey(scope: scope, inputFileType: inputFileType)) {
-                return spec.standardFlags(producer, scope: scope, optionContext: optionContext, delegate: delegate, inputFileType: inputFileType)
+            constantFlagsCache.getOrInsert(ConstantFlagsKey(scope: cbc.scope, inputFileType: inputFileType)) {
+                return spec.standardFlags(cbc, optionContext: optionContext, delegate: delegate, inputFileType: inputFileType)
             }
         }
     }
@@ -646,8 +646,9 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
         super.init(parser, basedOnSpec, isGeneric: false)
     }
 
-    private func standardFlags(_ producer: any CommandProducer, scope: MacroEvaluationScope, optionContext: (any BuildOptionGenerationContext)?, delegate: any TaskGenerationDelegate, inputFileType: FileTypeSpec) -> ConstantFlags {
+    private func standardFlags(_ cbc: CommandBuildContext, optionContext: (any BuildOptionGenerationContext)?, delegate: any TaskGenerationDelegate, inputFileType: FileTypeSpec) -> ConstantFlags {
         var commandLine = Array<String>()
+        let producer = cbc.producer, scope = cbc.scope
 
         // Add the arguments from the specification.
         commandLine += self.commandLineFromOptions(producer, scope: scope, inputFileType: inputFileType, optionContext: optionContext, buildOptionsFilter: .specOnly, lookup: { declaration in
@@ -657,7 +658,7 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
                     return BuiltinMacros.namespace.parseString("NO")
                 }
             }
-            return nil
+            return self.lookup(declaration, cbc, delegate)
         }).map(\.asString)
 
         // Add the common header search paths.
@@ -1136,7 +1137,8 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
         //
         // FIXME: Eventually we should just apply this optimization to all generic specs, and then find a way to piggy back on that.
         let dataCache = cbc.producer.getSpecDataCache(self, cacheType: DataCache.self)
-        let constantFlags = dataCache.getStandardFlags(self, producer: cbc.producer, scope: cbc.scope, optionContext: clangInfo, delegate: delegate, inputFileType: resolvedInputFileType)
+        let cbcWithOutput = cbc.outputs.isEmpty ? cbc.appendingOutputs([outputNode.path]) : cbc
+        let constantFlags = dataCache.getStandardFlags(self, cbc: cbcWithOutput, optionContext: clangInfo, delegate: delegate, inputFileType: resolvedInputFileType)
         commandLine += constantFlags.flags
         let responseFileAdditionalOutput = constantFlags.responseFileMapping.keys.sorted().map({"Using response file: \($0.str)"})
         additionalOutput.append(contentsOf: responseFileAdditionalOutput)
@@ -1157,7 +1159,6 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
         inputDeps.append(contentsOf: constantFlags.headerSearchPaths.inputPaths)
 #endif
 
-        let cbcWithOutput = cbc.outputs.isEmpty ? cbc.appendingOutputs([outputNode.path]) : cbc
         commandLine += self.commandLineFromOptions(cbc.producer, scope: cbc.scope, inputFileType: resolvedInputFileType, optionContext: clangInfo, buildOptionsFilter: .extendedOnly, lookup: {
             self.lookup($0, cbcWithOutput, delegate)
         }).map(\.asString)
@@ -1681,7 +1682,7 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
 
         // Add “standard flags”, which are ones that depend only on the variant, architecture, and language (in addition to the identifier, of course).
         let dataCache = cbc.producer.getSpecDataCache(self, cacheType: DataCache.self)
-        let constantFlags = dataCache.getStandardFlags(self, producer: cbc.producer, scope: cbc.scope, optionContext: clangInfo, delegate: delegate, inputFileType: inputFileType)
+        let constantFlags = dataCache.getStandardFlags(self, cbc: cbc, optionContext: clangInfo, delegate: delegate, inputFileType: inputFileType)
         let responseFileAdditionalOutput = constantFlags.responseFileMapping.keys.sorted().map({"Using response file: \($0.str)"})
         commandLine += constantFlags.flags
 

Sources/SWBUniversalPlatform/Specs/Clang.xcspec

@@ -109,6 +109,24 @@
                     NO = ();
                 };
             },
+            {
+                Name = "EMIT_COMPILER_SOURCE_METADATA";
+                Type = Boolean;
+                DefaultValue = No;
+            },
+            {
+                Name = "__CLANG_SOURCE_METADATA_PATH";
+                Type = Path;
+                Condition = "$(EMIT_COMPILER_SOURCE_METADATA)";
+                DefaultValue = "$(OutputPath).source-metadata.json";
+                CommandLinePrefixFlag = "-fdiagnostics-add-output=sarif:file=";
+                OutputDependencies = (
+                    {
+                        Path = "$(__CLANG_SOURCE_METADATA_PATH)";
+                        FileType = "text.json.compiler-metadata.source";
+                    },
+                );
+            },
             {
                 Name = "CLANG_MACRO_BACKTRACE_LIMIT";
                 Type = String;

Sources/SWBUniversalPlatform/Specs/StandardFileTypes.xcspec

@@ -1720,4 +1720,11 @@
         CanSetIncludeInIndex = YES;
         UTI = "public.protobuf-source";
     },
+    {
+        Identifier = text.json.compiler-metadata.source;
+        Type = FileType;
+        Name = "Source metadata emitted from compiler";
+        UTI = "com.apple.compiler-metadata.source";
+        BasedOn = text.json;
+    },
 )

Tests/SWBTaskConstructionTests/TaskConstructionTests.swift

@@ -9512,4 +9512,81 @@ fileprivate struct TaskConstructionTests: CoreBasedTests {
             }
         }
     }
+
+    @Test(.requireSDKs(.macOS))
+    func testSourceMetadata() async throws {
+        try await withTemporaryDirectory { (tmpDir: Path) async throws -> Void in
+            let sources = [
+                "SourceFile0.c",
+                "SourceFile1.mm",
+                "SourceFile2.m",
+                "SourceFile3.cpp"
+            ]
+
+            let testProject = TestProject(
+                "aProject",
+                sourceRoot: tmpDir,
+                groupTree: TestGroup(
+                    "SomeFiles", path: "Sources",
+                    children: sources.map{TestFile($0)}
+                ),
+                buildConfigurations: [
+                    TestBuildConfiguration("Debug")
+                ],
+                targets: [
+                    TestStandardTarget(
+                        "AppTarget",
+                        type: .application,
+                        buildConfigurations: [
+                            TestBuildConfiguration("Debug", buildSettings: [
+                                "TARGET_BUILD_DIR": "/tmp/SomeFiles.dst",
+                                "GENERATE_INFOPLIST_FILE": "YES",
+                                "PRODUCT_NAME": "$(TARGET_NAME)",
+                                "ARCHS": "x86_64 arm64",
+                                "EMIT_COMPILER_SOURCE_METADATA": "YES",
+                                "COMPILER_SOURCE_METADATA_LIST": "/tmp/test.json"
+                            ])
+                        ],
+                        buildPhases: [
+                            TestSourcesBuildPhase(sources.map{TestBuildFile($0)})
+                        ]
+                    )]
+            )
+
+            let fs = PseudoFS()
+
+            let core = try await getCore()
+            let tester = try TaskConstructionTester(core, testProject)
+
+            await tester.checkBuild(BuildParameters(configuration: "Debug"), runDestination: .anyMac, fs: fs) { results -> Void in
+                results.checkTarget("AppTarget") { target -> Void in
+                    let command = "-fdiagnostics-add-output=sarif:file="
+                    let buildPath = tmpDir.join("build/aProject.build/Debug/AppTarget.build/Objects-normal/")
+                    for arch in ["x86_64", "arm64"] {
+                        let metadataPath = buildPath.join(arch)
+                        let inputs = sources.map{metadataPath.join(Path($0).basenameWithoutSuffix + ".o.source-metadata.json").str}
+
+                        for (source, input) in zip(sources, inputs) {
+                            results.checkTask(.matchTarget(target), .matchRuleType("CompileC"), .matchRuleItemBasename(source), .matchRuleItem(arch), body: { task in
+                                task.checkCommandLineContains([command + input])
+                                task.checkInputs([
+                                    .path(tmpDir.join("Sources").join(source).str),
+                                    .namePattern(.and(.prefix("target-"), .suffix("-generated-headers"))),
+                                    .namePattern(.and(.prefix("target-"), .suffix("-swift-generated-headers"))),
+                                    .namePattern(.and(.prefix("target-"), .suffix("-ModuleVerifierTaskProducer"))),
+                                    .namePattern(.and(.prefix("target-"), .suffix("-begin-compiling"))),
+                                    .name("WorkspaceHeaderMapVFSFilesWritten"),
+                                ])
+                                task.checkOutputs([
+                                    .path(buildPath.join(arch).join(Path(source).basenameWithoutSuffix + ".o").str),
+                                    .path(metadataPath.join(Path(source).basenameWithoutSuffix + ".o.source-metadata.json").str)
+                                ])
+                            })
+
+                        }
+                    }
+                }
+            }
+        }
+    }
 }