[clang Dependency Scanning] Out-of-Date Scanning File System Cache Entry Reporting C-APIs (#10927)

This PR implements the C-APIs to report a scanning file system cache's out-of-date entries. The C-APIs contains a function to return a set of file system cache out-of-date entries, functions to facilitate looping through all the entries, and reporting the relevant information from the entries. 

The APIs are based on llvm#144105. 

rdar://152247357

Author: qiongsiwu

clang/include/clang-c/Dependencies.h

@@ -585,10 +585,98 @@ const char *clang_experimental_DepGraph_getTUContextHash(CXDepGraph);
 CINDEX_LINKAGE
 CXDiagnosticSet clang_experimental_DepGraph_getDiagnostics(CXDepGraph);
 
-CINDEX_LINKAGE
-CXCStringArray
-    clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths(
-        CXDependencyScannerService);
+/**
+ * The kind of scanning file system cache out-of-date entries.
+ */
+typedef enum {
+  /**
+   * The entry is negatively stat cached (which indicates the file did not exist
+   * the first time it was looked up during scanning), but the cached file
+   * exists on the underlying file system.
+   */
+  NegativelyCached,
+
+  /**
+   * The entry indicates that for the cached file, its cached size
+   * is different from its size reported by the underlying file system.
+   */
+  SizeChanged
+} CXDepScanFSCacheOutOfDateKind;
+
+/**
+ * The opaque object that contains the scanning file system cache's out-of-date
+ * entires.
+ */
+typedef struct CXOpaqueDepScanFSOutOfDateEntrySet *CXDepScanFSOutOfDateEntrySet;
+
+/**
+ * The opaque object that represents a single scanning file system cache's out-
+ * of-date entry.
+ */
+typedef struct CXOpaqueDepScanFSOutOfDateEntry *CXDepScanFSOutOfDateEntry;
+
+/**
+ * Returns all the file system cache out-of-date entries given a
+ * \c CXDependencyScannerService .
+ *
+ * This function is intended to be called when the build has finished,
+ * and the \c CXDependencyScannerService instance is about to be disposed.
+ *
+ * The \c CXDependencyScannerService instance owns the strings used
+ * by the out-of-date entries and should be disposed after the
+ * out-of-date entries are used and disposed.
+ */
+CXDepScanFSOutOfDateEntrySet
+clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet(
+    CXDependencyScannerService S);
+
+/**
+ * Returns the number of out-of-date entries contained in a
+ * \c CXDepScanFSOutOfDateEntrySet .
+ */
+size_t clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries(
+    CXDepScanFSOutOfDateEntrySet Entries);
+
+/**
+ * Returns the out-of-date entry at offset \p Idx of the \c
+ * CXDepScanFSOutOfDateEntrySet instance.
+ */
+CXDepScanFSOutOfDateEntry
+clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry(
+    CXDepScanFSOutOfDateEntrySet Entries, size_t Idx);
+
+/**
+ * Given an instance of \c CXDepScanFSOutOfDateEntry, returns its Kind.
+ */
+CXDepScanFSCacheOutOfDateKind
+clang_experimental_DepScanFSCacheOutOfDateEntry_getKind(
+    CXDepScanFSOutOfDateEntry Entry);
+
+/**
+ * Given an instance of \c CXDepScanFSOutOfDateEntry, returns the path.
+ */
+CXString clang_experimental_DepScanFSCacheOutOfDateEntry_getPath(
+    CXDepScanFSOutOfDateEntry Entry);
+
+/**
+ * Given an instance of \c CXDepScanFSOutOfDateEntry of kind SizeChanged,
+ * returns the cached size.
+ */
+uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize(
+    CXDepScanFSOutOfDateEntry Entry);
+
+/**
+ * Given an instance of \c CXDepScanFSOutOfDateEntry of kind SizeChanged,
+ * returns the actual size on the underlying file system.
+ */
+uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize(
+    CXDepScanFSOutOfDateEntry Entry);
+
+/**
+ * Dispose the \c CXDepScanFSOutOfDateEntrySet instance.
+ */
+void clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet(
+    CXDepScanFSOutOfDateEntrySet Entries);
 
 /**
  * Options used to generate a reproducer.

clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp

@@ -133,12 +133,22 @@ DependencyScanningFilesystemSharedCache::getOutOfDateEntries(
           InvalidDiagInfo.emplace_back(Path.data());
         } else {
           llvm::vfs::Status CachedStatus = Entry->getStatus();
-          uint64_t CachedSize = CachedStatus.getSize();
-          uint64_t ActualSize = Status->getSize();
-          if (CachedSize != ActualSize) {
-            // This is the case where the cached file has a different size
-            // from the actual file that comes from the underlying FS.
-            InvalidDiagInfo.emplace_back(Path.data(), CachedSize, ActualSize);
+          if (Status->getType() == llvm::sys::fs::file_type::regular_file &&
+              Status->getType() == CachedStatus.getType()) {
+            // We only check regular files. Directory files sizes could change
+            // due to content changes, and reporting directory size changes can
+            // lead to false positives.
+            // TODO: At the moment, we do not detect symlinks to files whose
+            // size may change. We need to decide if we want to detect cached
+            // symlink size changes. We can also expand this to detect file
+            // type changes.
+            uint64_t CachedSize = CachedStatus.getSize();
+            uint64_t ActualSize = Status->getSize();
+            if (CachedSize != ActualSize) {
+              // This is the case where the cached file has a different size
+              // from the actual file that comes from the underlying FS.
+              InvalidDiagInfo.emplace_back(Path.data(), CachedSize, ActualSize);
+            }
           }
         }
       }

clang/test/ClangScanDeps/error-c-api.cpp

@@ -4,4 +4,4 @@
 
 // CHECK: error: failed to get dependencies
 // CHECK-NEXT: 'missing.h' file not found
-// CHECK-NEXT: number of invalid negatively stat cached paths: 0
+// CHECK-NEXT: number of out of date file system cache entries: 0

clang/tools/c-index-test/core_main.cpp

@@ -916,12 +916,18 @@ static int scanDeps(ArrayRef<const char *> Args, std::string WorkingDirectory,
     clang_disposeDiagnostic(Diag);
   }
 
-  CXCStringArray InvalidNegativeStatCachedPaths =
-      clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths(
+  CXDepScanFSOutOfDateEntrySet OutOfDateEntrySet =
+      clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet(
           Service);
 
-  llvm::errs() << "note: number of invalid negatively stat cached paths: "
-               << InvalidNegativeStatCachedPaths.Count << "\n";
+  llvm::errs()
+      << "note: number of out of date file system cache entries: "
+      << clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries(
+             OutOfDateEntrySet)
+      << "\n";
+
+  clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet(
+      OutOfDateEntrySet);
 
   return 1;
 }

clang/tools/libclang/CDependencies.cpp

@@ -81,17 +81,11 @@ struct CStringsManager {
     return createCStringsRef(*OwnedStdStr.back());
   }
 };
-
-struct DependencyScannerService {
-  DependencyScanningService Service;
-  CStringsManager StrMgr{};
-};
 } // end anonymous namespace
 
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerServiceOptions,
                                    CXDependencyScannerServiceOptions)
-
-DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerService,
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningService,
                                    CXDependencyScannerService)
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningWorker,
                                    CXDependencyScannerWorker)
@@ -172,9 +166,9 @@ clang_experimental_DependencyScannerService_create_v0(CXDependencyMode Format) {
   // FIXME: Pass default CASOpts and nullptr as CachingOnDiskFileSystem now.
   CASOptions CASOpts;
   IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> FS;
-  return wrap(new DependencyScannerService{DependencyScanningService(
+  return wrap(new DependencyScanningService(
       ScanningMode::DependencyDirectivesScan, unwrap(Format), CASOpts,
-      /*CAS=*/nullptr, /*ActionCache=*/nullptr, FS)});
+      /*CAS=*/nullptr, /*ActionCache=*/nullptr, FS));
 }
 
 ScanningOutputFormat DependencyScannerServiceOptions::getFormat() const {
@@ -210,10 +204,10 @@ clang_experimental_DependencyScannerService_create_v1(
     FS = llvm::cantFail(
         llvm::cas::createCachingOnDiskFileSystem(CAS));
   }
-  return wrap(new DependencyScannerService{DependencyScanningService(
+  return wrap(new DependencyScanningService(
       ScanningMode::DependencyDirectivesScan, Format, unwrap(Opts)->CASOpts,
       std::move(CAS), std::move(Cache), std::move(FS),
-      unwrap(Opts)->OptimizeArgs)});
+      unwrap(Opts)->OptimizeArgs));
 }
 
 void clang_experimental_DependencyScannerService_dispose_v0(
@@ -223,16 +217,16 @@ void clang_experimental_DependencyScannerService_dispose_v0(
 
 CXDependencyScannerWorker clang_experimental_DependencyScannerWorker_create_v0(
     CXDependencyScannerService S) {
-  ScanningOutputFormat Format = unwrap(S)->Service.getFormat();
+  ScanningOutputFormat Format = unwrap(S)->getFormat();
   bool IsIncludeTreeOutput = Format == ScanningOutputFormat::IncludeTree ||
                              Format == ScanningOutputFormat::FullIncludeTree;
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS =
       llvm::vfs::createPhysicalFileSystem();
   if (IsIncludeTreeOutput)
-    FS = llvm::cas::createCASProvidingFileSystem(unwrap(S)->Service.getCAS(),
+    FS = llvm::cas::createCASProvidingFileSystem(unwrap(S)->getCAS(),
                                                  std::move(FS));
 
-  return wrap(new DependencyScanningWorker(unwrap(S)->Service, FS));
+  return wrap(new DependencyScanningWorker(*unwrap(S), FS));
 }
 
 void clang_experimental_DependencyScannerWorker_dispose_v0(
@@ -566,43 +560,6 @@ CXDiagnosticSet clang_experimental_DepGraph_getDiagnostics(CXDepGraph Graph) {
   return unwrap(Graph)->getDiagnosticSet();
 }
 
-CXCStringArray
-clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths(
-    CXDependencyScannerService S) {
-  DependencyScanningService &Service = unwrap(S)->Service;
-  CStringsManager &StrMgr = unwrap(S)->StrMgr;
-
-  // FIXME: CAS currently does not use the shared cache, and cannot produce
-  // the same diagnostics. We should add such a diagnostics to CAS as well.
-  if (Service.useCASFS())
-    return {nullptr, 0};
-
-  DependencyScanningFilesystemSharedCache &SharedCache =
-      Service.getSharedCache();
-
-  // Note that it is critical that this FS is the same as the default virtual
-  // file system we pass to the DependencyScanningWorkers.
-  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS =
-      llvm::vfs::createPhysicalFileSystem();
-
-  auto OutOfDateEntries = SharedCache.getOutOfDateEntries(*FS);
-
-  // FIXME: replace this code with proper APIs that handles the
-  // OutOfDateEntries.
-  std::vector<const char *> OutOfDatePaths;
-  for (const auto &E : OutOfDateEntries)
-    OutOfDatePaths.emplace_back(E.Path);
-
-  // FIXME: This code here creates copies of strings from
-  // InvaidNegStatCachedPaths. It is acceptable because this C-API is expected
-  // to be called only at the end of a CXDependencyScannerService's lifetime.
-  // In other words, it is called very infrequently. We can change
-  // CStringsManager's interface to accommodate handling arbitrary StringRefs
-  // (which may not be null terminated) if we want to avoid copying.
-  return StrMgr.createCStringsOwned(
-      {OutOfDatePaths.begin(), OutOfDatePaths.end()});
-}
-
 static std::string
 lookupModuleOutput(const ModuleDeps &MD, ModuleOutputKind MOK, void *MLOContext,
                    std::variant<CXModuleLookupOutputCallback *,
@@ -646,6 +603,98 @@ std::string OutputLookup::lookupModuleOutput(const ModuleDeps &MD,
   return PCMPath.first->second;
 }
 
+namespace {
+typedef std::vector<DependencyScanningFilesystemSharedCache::OutOfDateEntry>
+    DependencyScannerFSOutOfDateEntrySet;
+
+typedef DependencyScanningFilesystemSharedCache::OutOfDateEntry
+    DependencyScannerFSOutOfDateEntry;
+} // namespace
+
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerFSOutOfDateEntrySet,
+                                   CXDepScanFSOutOfDateEntrySet)
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerFSOutOfDateEntry,
+                                   CXDepScanFSOutOfDateEntry)
+
+CXDepScanFSOutOfDateEntrySet
+clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet(
+    CXDependencyScannerService S) {
+  DependencyScanningService &Service = *unwrap(S);
+
+  // FIXME: CAS FS currently does not use the shared cache, and cannot produce
+  // the same diagnostics. We should add such a diagnostics to CAS as well.
+  if (Service.useCASFS())
+    return nullptr;
+
+  // Note that it is critical that this FS is the same as the default virtual
+  // file system we pass to the DependencyScanningWorkers.
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS =
+      llvm::vfs::createPhysicalFileSystem();
+
+  DependencyScannerFSOutOfDateEntrySet *OODEntrySet =
+      new DependencyScannerFSOutOfDateEntrySet();
+  *OODEntrySet = Service.getSharedCache().getOutOfDateEntries(*FS);
+
+  return wrap(OODEntrySet);
+}
+
+size_t clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries(
+    CXDepScanFSOutOfDateEntrySet Entries) {
+  return unwrap(Entries)->size();
+}
+
+CXDepScanFSOutOfDateEntry
+clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry(
+    CXDepScanFSOutOfDateEntrySet Entries, size_t Idx) {
+  DependencyScannerFSOutOfDateEntrySet *EntSet = unwrap(Entries);
+  return wrap(&(*EntSet)[Idx]);
+}
+
+CXDepScanFSCacheOutOfDateKind
+clang_experimental_DepScanFSCacheOutOfDateEntry_getKind(
+    CXDepScanFSOutOfDateEntry Entry) {
+  DependencyScannerFSOutOfDateEntry *E = unwrap(Entry);
+  auto &Info = E->Info;
+  return std::visit(
+      llvm::makeVisitor(
+          [](const DependencyScannerFSOutOfDateEntry::NegativelyCachedInfo
+                 &Info) { return NegativelyCached; },
+          [](const DependencyScannerFSOutOfDateEntry::SizeChangedInfo &Info) {
+            return SizeChanged;
+          }),
+      Info);
+}
+
+CXString clang_experimental_DepScanFSCacheOutOfDateEntry_getPath(
+    CXDepScanFSOutOfDateEntry Entry) {
+  return cxstring::createRef(unwrap(Entry)->Path);
+}
+
+static DependencyScannerFSOutOfDateEntry::SizeChangedInfo *
+getOutOfDateEntrySizeChangedInfo(DependencyScannerFSOutOfDateEntry *E) {
+  auto *SizeInfo =
+      std::get_if<DependencyScannerFSOutOfDateEntry::SizeChangedInfo>(&E->Info);
+  assert(SizeInfo && "Wrong entry kind to get size changed info!");
+  return SizeInfo;
+}
+
+uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize(
+    CXDepScanFSOutOfDateEntry Entry) {
+  DependencyScannerFSOutOfDateEntry *E = unwrap(Entry);
+  return getOutOfDateEntrySizeChangedInfo(E)->CachedSize;
+}
+
+uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize(
+    CXDepScanFSOutOfDateEntry Entry) {
+  DependencyScannerFSOutOfDateEntry *E = unwrap(Entry);
+  return getOutOfDateEntrySizeChangedInfo(E)->ActualSize;
+}
+
+void clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet(
+    CXDepScanFSOutOfDateEntrySet Entries) {
+  delete unwrap(Entries);
+}
+
 namespace {
 struct DependencyScannerReproducerOptions {
   std::vector<std::string> BuildArgs;

clang/tools/libclang/libclang.map

@@ -577,7 +577,6 @@ LLVM_21 {
     clang_experimental_DepGraphModule_isCWDIgnored;
     clang_experimental_DepGraphModule_isInStableDirs; 
     clang_getFullyQualifiedName;
-    clang_experimental_DependencyScannerService_getInvalidNegStatCachedPaths;
     clang_experimental_DependencyScannerReproducerOptions_create;
     clang_experimental_DependencyScannerReproducerOptions_dispose;
     clang_experimental_DependencyScanner_generateReproducer;
@@ -590,6 +589,14 @@ LLVM_21 {
     clang_Cursor_getGCCAssemblyNumClobbers;
     clang_Cursor_getGCCAssemblyClobber;
     clang_Cursor_isGCCAssemblyVolatile;
+    clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet;
+    clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries;
+    clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry;
+    clang_experimental_DepScanFSCacheOutOfDateEntry_getKind;
+    clang_experimental_DepScanFSCacheOutOfDateEntry_getPath;
+    clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize;
+    clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize;
+    clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet;
 };
 
 # Example of how to add a new symbol version entry.  If you do add a new symbol

clang/unittests/libclang/CMakeLists.txt

@@ -1,6 +1,7 @@
 add_clang_unittest(libclangTests
   LibclangTest.cpp
   DriverTest.cpp
+  DependencyScanningCAPITests.cpp
   LINK_LIBS
   libclang
   )