[clang Dependency Scanning] Out-of-Date Scanning File System Cache Entry Reporting C-APIs (swiftlang#10927)

This PR implements the C-APIs to report a scanning file system cache's out-of-date entries. The C-APIs contains a function to return a set of file system cache out-of-date entries, functions to facilitate looping through all the entries, and reporting the relevant information from the entries.

The APIs are based on llvm#144105.

rdar://152247357
(cherry picked from commit 6fbcea6)

Author: qiongsiwu

clang/include/clang-c/Dependencies.h

@@ -785,6 +785,99 @@ CINDEX_LINKAGE CXString clang_experimental_DepGraphModuleLinkLibrary_getLibrary(
 CINDEX_LINKAGE bool clang_experimental_DepGraphModuleLinkLibrary_isFramework(
     CXDepGraphModuleLinkLibrary);
 
+/**
+ * The kind of scanning file system cache out-of-date entries.
+ */
+typedef enum {
+  /**
+   * The entry is negatively stat cached (which indicates the file did not exist
+   * the first time it was looked up during scanning), but the cached file
+   * exists on the underlying file system.
+   */
+  NegativelyCached,
+
+  /**
+   * The entry indicates that for the cached file, its cached size
+   * is different from its size reported by the underlying file system.
+   */
+  SizeChanged
+} CXDepScanFSCacheOutOfDateKind;
+
+/**
+ * The opaque object that contains the scanning file system cache's out-of-date
+ * entires.
+ */
+typedef struct CXOpaqueDepScanFSOutOfDateEntrySet *CXDepScanFSOutOfDateEntrySet;
+
+/**
+ * The opaque object that represents a single scanning file system cache's out-
+ * of-date entry.
+ */
+typedef struct CXOpaqueDepScanFSOutOfDateEntry *CXDepScanFSOutOfDateEntry;
+
+/**
+ * Returns all the file system cache out-of-date entries given a
+ * \c CXDependencyScannerService .
+ *
+ * This function is intended to be called when the build has finished,
+ * and the \c CXDependencyScannerService instance is about to be disposed.
+ *
+ * The \c CXDependencyScannerService instance owns the strings used
+ * by the out-of-date entries and should be disposed after the
+ * out-of-date entries are used and disposed.
+ */
+CXDepScanFSOutOfDateEntrySet
+clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet(
+    CXDependencyScannerService S);
+
+/**
+ * Returns the number of out-of-date entries contained in a
+ * \c CXDepScanFSOutOfDateEntrySet .
+ */
+size_t clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries(
+    CXDepScanFSOutOfDateEntrySet Entries);
+
+/**
+ * Returns the out-of-date entry at offset \p Idx of the \c
+ * CXDepScanFSOutOfDateEntrySet instance.
+ */
+CXDepScanFSOutOfDateEntry
+clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry(
+    CXDepScanFSOutOfDateEntrySet Entries, size_t Idx);
+
+/**
+ * Given an instance of \c CXDepScanFSOutOfDateEntry, returns its Kind.
+ */
+CXDepScanFSCacheOutOfDateKind
+clang_experimental_DepScanFSCacheOutOfDateEntry_getKind(
+    CXDepScanFSOutOfDateEntry Entry);
+
+/**
+ * Given an instance of \c CXDepScanFSOutOfDateEntry, returns the path.
+ */
+CXString clang_experimental_DepScanFSCacheOutOfDateEntry_getPath(
+    CXDepScanFSOutOfDateEntry Entry);
+
+/**
+ * Given an instance of \c CXDepScanFSOutOfDateEntry of kind SizeChanged,
+ * returns the cached size.
+ */
+uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize(
+    CXDepScanFSOutOfDateEntry Entry);
+
+/**
+ * Given an instance of \c CXDepScanFSOutOfDateEntry of kind SizeChanged,
+ * returns the actual size on the underlying file system.
+ */
+uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize(
+    CXDepScanFSOutOfDateEntry Entry);
+
+/**
+ * Dispose the \c CXDepScanFSOutOfDateEntrySet instance.
+ */
+void clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet(
+    CXDepScanFSOutOfDateEntrySet Entries);
+
 /**
  * @}
  */

clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp

@@ -134,12 +134,22 @@ DependencyScanningFilesystemSharedCache::getOutOfDateEntries(
           InvalidDiagInfo.emplace_back(Path.data());
         } else {
           llvm::vfs::Status CachedStatus = Entry->getStatus();
-          uint64_t CachedSize = CachedStatus.getSize();
-          uint64_t ActualSize = Status->getSize();
-          if (CachedSize != ActualSize) {
-            // This is the case where the cached file has a different size
-            // from the actual file that comes from the underlying FS.
-            InvalidDiagInfo.emplace_back(Path.data(), CachedSize, ActualSize);
+          if (Status->getType() == llvm::sys::fs::file_type::regular_file &&
+              Status->getType() == CachedStatus.getType()) {
+            // We only check regular files. Directory files sizes could change
+            // due to content changes, and reporting directory size changes can
+            // lead to false positives.
+            // TODO: At the moment, we do not detect symlinks to files whose
+            // size may change. We need to decide if we want to detect cached
+            // symlink size changes. We can also expand this to detect file
+            // type changes.
+            uint64_t CachedSize = CachedStatus.getSize();
+            uint64_t ActualSize = Status->getSize();
+            if (CachedSize != ActualSize) {
+              // This is the case where the cached file has a different size
+              // from the actual file that comes from the underlying FS.
+              InvalidDiagInfo.emplace_back(Path.data(), CachedSize, ActualSize);
+            }
           }
         }
       }

clang/test/ClangScanDeps/error-c-api.cpp

@@ -4,3 +4,4 @@
 
 // CHECK: error: failed to get dependencies
 // CHECK-NEXT: 'missing.h' file not found
+// CHECK-NEXT: number of out of date file system cache entries: 0

clang/tools/c-index-test/core_main.cpp

@@ -916,6 +916,20 @@ static int scanDeps(ArrayRef<const char *> Args, std::string WorkingDirectory,
     clang_disposeString(Spelling);
     clang_disposeDiagnostic(Diag);
   }
+
+  CXDepScanFSOutOfDateEntrySet OutOfDateEntrySet =
+      clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet(
+          Service);
+
+  llvm::errs()
+      << "note: number of out of date file system cache entries: "
+      << clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries(
+             OutOfDateEntrySet)
+      << "\n";
+
+  clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet(
+      OutOfDateEntrySet);
+
   return 1;
 }
 

clang/tools/libclang/CDependencies.cpp

@@ -45,7 +45,6 @@ struct DependencyScannerServiceOptions {
 
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerServiceOptions,
                                    CXDependencyScannerServiceOptions)
-
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningService,
                                    CXDependencyScannerService)
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningWorker,
@@ -816,3 +815,96 @@ bool clang_experimental_DepGraphModuleLinkLibrary_isFramework(
   return Lib->IsFramework;
 }
 
+
+namespace {
+typedef std::vector<DependencyScanningFilesystemSharedCache::OutOfDateEntry>
+    DependencyScannerFSOutOfDateEntrySet;
+
+typedef DependencyScanningFilesystemSharedCache::OutOfDateEntry
+    DependencyScannerFSOutOfDateEntry;
+} // namespace
+
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerFSOutOfDateEntrySet,
+                                   CXDepScanFSOutOfDateEntrySet)
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerFSOutOfDateEntry,
+                                   CXDepScanFSOutOfDateEntry)
+
+CXDepScanFSOutOfDateEntrySet
+clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet(
+    CXDependencyScannerService S) {
+  DependencyScanningService &Service = *unwrap(S);
+
+  // FIXME: CAS FS currently does not use the shared cache, and cannot produce
+  // the same diagnostics. We should add such a diagnostics to CAS as well.
+  if (Service.useCASFS())
+    return nullptr;
+
+  // Note that it is critical that this FS is the same as the default virtual
+  // file system we pass to the DependencyScanningWorkers.
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS =
+      llvm::vfs::createPhysicalFileSystem();
+
+  DependencyScannerFSOutOfDateEntrySet *OODEntrySet =
+      new DependencyScannerFSOutOfDateEntrySet();
+  *OODEntrySet = Service.getSharedCache().getOutOfDateEntries(*FS);
+
+  return wrap(OODEntrySet);
+}
+
+size_t clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries(
+    CXDepScanFSOutOfDateEntrySet Entries) {
+  return unwrap(Entries)->size();
+}
+
+CXDepScanFSOutOfDateEntry
+clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry(
+    CXDepScanFSOutOfDateEntrySet Entries, size_t Idx) {
+  DependencyScannerFSOutOfDateEntrySet *EntSet = unwrap(Entries);
+  return wrap(&(*EntSet)[Idx]);
+}
+
+CXDepScanFSCacheOutOfDateKind
+clang_experimental_DepScanFSCacheOutOfDateEntry_getKind(
+    CXDepScanFSOutOfDateEntry Entry) {
+  DependencyScannerFSOutOfDateEntry *E = unwrap(Entry);
+  auto &Info = E->Info;
+  return std::visit(
+      llvm::makeVisitor(
+          [](const DependencyScannerFSOutOfDateEntry::NegativelyCachedInfo
+                 &Info) { return NegativelyCached; },
+          [](const DependencyScannerFSOutOfDateEntry::SizeChangedInfo &Info) {
+            return SizeChanged;
+          }),
+      Info);
+}
+
+CXString clang_experimental_DepScanFSCacheOutOfDateEntry_getPath(
+    CXDepScanFSOutOfDateEntry Entry) {
+  return cxstring::createRef(unwrap(Entry)->Path);
+}
+
+static DependencyScannerFSOutOfDateEntry::SizeChangedInfo *
+getOutOfDateEntrySizeChangedInfo(DependencyScannerFSOutOfDateEntry *E) {
+  auto *SizeInfo =
+      std::get_if<DependencyScannerFSOutOfDateEntry::SizeChangedInfo>(&E->Info);
+  assert(SizeInfo && "Wrong entry kind to get size changed info!");
+  return SizeInfo;
+}
+
+uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize(
+    CXDepScanFSOutOfDateEntry Entry) {
+  DependencyScannerFSOutOfDateEntry *E = unwrap(Entry);
+  return getOutOfDateEntrySizeChangedInfo(E)->CachedSize;
+}
+
+uint64_t clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize(
+    CXDepScanFSOutOfDateEntry Entry) {
+  DependencyScannerFSOutOfDateEntry *E = unwrap(Entry);
+  return getOutOfDateEntrySizeChangedInfo(E)->ActualSize;
+}
+
+void clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet(
+    CXDepScanFSOutOfDateEntrySet Entries) {
+  delete unwrap(Entries);
+}
+

clang/tools/libclang/libclang.map

@@ -580,6 +580,14 @@ LLVM_21 {
     clang_experimental_DepGraphModuleLinkLibrarySet_getLinkLibrary;
     clang_experimental_DepGraphModuleLinkLibrary_getLibrary;
     clang_experimental_DepGraphModuleLinkLibrary_isFramework;
+    clang_experimental_DependencyScannerService_getFSCacheOutOfDateEntrySet;
+    clang_experimental_DepScanFSCacheOutOfDateEntrySet_getNumOfEntries;
+    clang_experimental_DepScanFSCacheOutOfDateEntrySet_getEntry;
+    clang_experimental_DepScanFSCacheOutOfDateEntry_getKind;
+    clang_experimental_DepScanFSCacheOutOfDateEntry_getPath;
+    clang_experimental_DepScanFSCacheOutOfDateEntry_getCachedSize;
+    clang_experimental_DepScanFSCacheOutOfDateEntry_getActualSize;
+    clang_experimental_DepScanFSCacheOutOfDateEntrySet_disposeSet;
 };
 
 # Example of how to add a new symbol version entry.  If you do add a new symbol

clang/unittests/libclang/CMakeLists.txt

@@ -1,6 +1,7 @@
 add_clang_unittest(libclangTests
   LibclangTest.cpp
   DriverTest.cpp
+  DependencyScanningCAPITests.cpp
   )
 
 target_link_libraries(libclangTests