diff --git a/Sources/SWBBuildSystem/BuildOperation.swift b/Sources/SWBBuildSystem/BuildOperation.swift index 8bf3c230..f078f5af 100644 --- a/Sources/SWBBuildSystem/BuildOperation.swift +++ b/Sources/SWBBuildSystem/BuildOperation.swift @@ -1563,6 +1563,8 @@ internal final class OperationSystemAdaptor: SWBLLBuild.BuildSystemDelegate, Act // The build should be complete, validate the consistency of the target/task counts. self.validateTargetCompletion(buildSucceeded: buildSucceeded) + self.diagnoseInvalidNegativeStatCacheEntries(buildSucceeded: buildSucceeded) + // If the build failed, make sure we flush any pending incremental build records. // Usually, driver instances are cleaned up and write out their incremental build records when a target finishes building. However, this won't necessarily be the case if the build fails. Ensure we write out any pending records before tearing down the graph so we don't use a stale record on a subsequent build. if !buildSucceeded { @@ -1574,6 +1576,16 @@ internal final class OperationSystemAdaptor: SWBLLBuild.BuildSystemDelegate, Act } } + func diagnoseInvalidNegativeStatCacheEntries(buildSucceeded: Bool) { + let settings = operation.requestContext.getCachedSettings(operation.request.parameters) + guard settings.globalScope.evaluate(BuiltinMacros.VERIFY_CLANG_SCANNER_NEGATIVE_STAT_CACHE) || !buildSucceeded else { + return + } + for entry in dynamicOperationContext.clangModuleDependencyGraph.diagnoseInvalidNegativeStatCacheEntries() { + buildOutputDelegate.warning(Path(entry), "Clang reported an invalid negative stat cache entry for '\(entry)'; this may indicate a missing dependency which caused the file to be modified after being read by a dependent") + } + } + /// Cleanup the compilation cache to reduce resource usage in environments not configured to preserve it. func cleanupCompilationCache() { let settings = operation.requestContext.getCachedSettings(operation.request.parameters) diff --git a/Sources/SWBCSupport/CLibclang.cpp b/Sources/SWBCSupport/CLibclang.cpp index 040e6f51..ab636b32 100644 --- a/Sources/SWBCSupport/CLibclang.cpp +++ b/Sources/SWBCSupport/CLibclang.cpp @@ -1259,6 +1259,11 @@ extern "C" { */ CXDiagnosticSet (*clang_experimental_DepGraph_getDiagnostics)(CXDepGraph); + /** + * Checks negatively cached paths in the stat cache against the current state of the filesystem and returns a list of discrepancies. + */ + CXCStringArray (*clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths)(CXDependencyScannerService); + // MARK: Driver API /** @@ -1337,6 +1342,7 @@ struct LibclangWrapper { bool hasDependencyScanner; bool hasStructuredScanningDiagnostics; bool hasCAS; + bool hasNegativeStatCacheDiagnostics; LibclangWrapper(std::string path) : path(path), @@ -1348,7 +1354,7 @@ struct LibclangWrapper { #else handle(dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL)), #endif - isLeaked(false), hasRequiredAPI(true), hasDependencyScanner(true), hasStructuredScanningDiagnostics(true), hasCAS(true) { + isLeaked(false), hasRequiredAPI(true), hasDependencyScanner(true), hasStructuredScanningDiagnostics(true), hasCAS(true), hasNegativeStatCacheDiagnostics(false) { #if defined(_WIN32) DWORD cchLength = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, path.c_str(), -1, nullptr, 0); std::unique_ptr wszPath(new wchar_t[cchLength]); @@ -1488,6 +1494,11 @@ struct LibclangWrapper { hasCAS = false; } + LOOKUP_OPTIONAL(clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths); + if (fns.clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths) { + hasNegativeStatCacheDiagnostics = true; + } + LOOKUP_OPTIONAL(clang_Driver_getExternalActionsForCommand_v0); LOOKUP_OPTIONAL(clang_Driver_ExternalActionList_dispose); LOOKUP_OPTIONAL(clang_install_aborting_llvm_fatal_error_handler); @@ -1760,6 +1771,10 @@ extern "C" { return lib->wrapper->hasStructuredScanningDiagnostics; } + bool libclang_has_negative_stat_cache_diagnostics(libclang_t lib) { + return lib->wrapper->hasNegativeStatCacheDiagnostics; + } + libclang_scanner_t libclang_scanner_create(libclang_t lib, libclang_casdatabases_t casdbs, libclang_casoptions_t casOpts) { return new libclang_scanner_t_{new LibclangScanner( lib->wrapper, LibclangFunctions::CXDependencyMode_Full, @@ -2043,6 +2058,17 @@ extern "C" { return diagnostic_set; } + void libclang_scanner_diagnose_invalid_negative_stat_cache_entries(libclang_scanner_t scanner, void (^path_callback)(const char *)) { + auto lib = scanner->scanner->lib; + if (!lib->fns.clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths) { + return; + } + LibclangFunctions::CXCStringArray paths = lib->fns.clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths(scanner->scanner->service); + for (size_t i = 0; i < paths.Count; ++i) { + path_callback(paths.Strings[i]); + } + } + bool libclang_scanner_scan_dependencies( libclang_scanner_t scanner, int argc, char *const *argv, const char *workingDirectory, __attribute__((noescape)) module_lookup_output_t module_lookup_output, diff --git a/Sources/SWBCSupport/CLibclang.h b/Sources/SWBCSupport/CLibclang.h index 4d0af80a..d5b5646f 100644 --- a/Sources/SWBCSupport/CLibclang.h +++ b/Sources/SWBCSupport/CLibclang.h @@ -107,6 +107,9 @@ CSUPPORT_EXPORT bool libclang_has_scanner(libclang_t lib); /// Whether libclang supports reporting structured scanning diagnostics. CSUPPORT_EXPORT bool libclang_has_structured_scanner_diagnostics(libclang_t lib); +/// Whether libclang supports reporting negative stat caching diagnostics. +CSUPPORT_EXPORT bool libclang_has_negative_stat_cache_diagnostics(libclang_t lib); + /// Create a new scanner instance with optional CAS databases. CSUPPORT_EXPORT libclang_scanner_t libclang_scanner_create(libclang_t lib, libclang_casdatabases_t, libclang_casoptions_t); @@ -184,6 +187,9 @@ typedef size_t (^module_lookup_output_t)( const char *module_name, const char *context_hash, clang_output_kind_t kind, char *output, size_t max_len); +/// Reports invalid entries in the scanner's negative stat cache. +CSUPPORT_EXPORT void libclang_scanner_diagnose_invalid_negative_stat_cache_entries(libclang_scanner_t scanner, void (^path_callback)(const char *)); + /// Scan the given Clang "cc1" invocation, looking for dependencies. /// /// NOTE: This function is thread-safe. diff --git a/Sources/SWBCore/LibclangVendored/Libclang.swift b/Sources/SWBCore/LibclangVendored/Libclang.swift index c6a1baa2..7d57bd7b 100644 --- a/Sources/SWBCore/LibclangVendored/Libclang.swift +++ b/Sources/SWBCore/LibclangVendored/Libclang.swift @@ -91,6 +91,10 @@ public final class Libclang { libclang_has_structured_scanner_diagnostics(lib) } + public var supportsNegativeStatCacheDiagnostics: Bool { + libclang_has_negative_stat_cache_diagnostics(lib) + } + public var supportsCASPruning: Bool { libclang_has_cas_pruning_feature(lib) } @@ -274,6 +278,17 @@ public final class DependencyScanner { return fileDeps } + public func diagnoseInvalidNegativeStatCacheEntries() -> [String] { + var entries: [String] = [] + libclang_scanner_diagnose_invalid_negative_stat_cache_entries(scanner, { cString in + guard let cString else { + return + } + entries.append(String(cString: cString)) + }) + return entries + } + public func generateReproducer( commandLine: [String], workingDirectory: String diff --git a/Sources/SWBCore/Settings/BuiltinMacros.swift b/Sources/SWBCore/Settings/BuiltinMacros.swift index 156cee04..dd7f2728 100644 --- a/Sources/SWBCore/Settings/BuiltinMacros.swift +++ b/Sources/SWBCore/Settings/BuiltinMacros.swift @@ -1165,6 +1165,7 @@ public final class BuiltinMacros { public static let VERSION_INFO_STRING = BuiltinMacros.declareStringMacro("VERSION_INFO_STRING") public static let VERSION_INFO_SUFFIX = BuiltinMacros.declareStringMacro("VERSION_INFO_SUFFIX") public static let ValidateForStore = BuiltinMacros.declareBooleanMacro("ValidateForStore") + public static let VERIFY_CLANG_SCANNER_NEGATIVE_STAT_CACHE = BuiltinMacros.declareBooleanMacro("VERIFY_CLANG_SCANNER_NEGATIVE_STAT_CACHE") public static let WARNING_CFLAGS = BuiltinMacros.declareStringListMacro("WARNING_CFLAGS") public static let WARNING_LDFLAGS = BuiltinMacros.declareStringListMacro("WARNING_LDFLAGS") public static let WATCHKIT_2_SUPPORT_FOLDER_PATH = BuiltinMacros.declareStringMacro("WATCHKIT_2_SUPPORT_FOLDER_PATH") @@ -2402,6 +2403,7 @@ public final class BuiltinMacros { VERSION_INFO_STRING, VERSION_INFO_SUFFIX, ValidateForStore, + VERIFY_CLANG_SCANNER_NEGATIVE_STAT_CACHE, WARNING_CFLAGS, WARNING_LDFLAGS, WATCHKIT_2_SUPPORT_FOLDER_PATH, diff --git a/Sources/SWBTaskExecution/DynamicTaskSpecs/ClangModuleDependencyGraph.swift b/Sources/SWBTaskExecution/DynamicTaskSpecs/ClangModuleDependencyGraph.swift index a8dcc71d..c4420761 100644 --- a/Sources/SWBTaskExecution/DynamicTaskSpecs/ClangModuleDependencyGraph.swift +++ b/Sources/SWBTaskExecution/DynamicTaskSpecs/ClangModuleDependencyGraph.swift @@ -560,6 +560,17 @@ package final class ClangModuleDependencyGraph { return clangWithScanner.casDBs } + package func diagnoseInvalidNegativeStatCacheEntries() -> [String] { + registryQueue.blocking_sync { + self.scannerRegistry.values.flatMap { libClangWithScanner in + guard libClangWithScanner.scanner.libclang.supportsNegativeStatCacheDiagnostics else { + return Array() + } + return libClangWithScanner.scanner.diagnoseInvalidNegativeStatCacheEntries() + } + } + } + package func generateReproducer(forFailedDependency dependency: DependencyInfo, libclangPath: Path, casOptions: CASOptions?) throws -> String? { let clangWithScanner = try libclangWithScanner(