[clang][DependencyScanning] Track modules that resolve from "stable" locations (#130634)

That patch tracks whether all the file & module dependencies of a module
resolve to a stable location. This information will later be queried by
build systems for determining where to store the accompanying pcms.

Author: cyndyishida

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

@@ -114,6 +114,14 @@ struct ModuleDeps {
   /// Whether this is a "system" module.
   bool IsSystem;
 
+  /// Whether this module is fully composed of file & module inputs from
+  /// locations likely to stay the same across the active development and build
+  /// cycle. For example, when all those input paths only resolve in Sysroot.
+  ///
+  /// External paths, as opposed to virtual file paths, are always used
+  /// for computing this value.
+  bool IsInStableDirectories;
+
   /// The path to the modulemap file which defines this module.
   ///
   /// This can be used to explicitly build this module. This file will
@@ -219,6 +227,9 @@ class ModuleDepCollectorPP final : public PPCallbacks {
                               llvm::DenseSet<const Module *> &AddedModules);
   void addAffectingClangModule(const Module *M, ModuleDeps &MD,
                           llvm::DenseSet<const Module *> &AddedModules);
+
+  /// Add discovered module dependency for the given module.
+  void addOneModuleDep(const Module *M, const ModuleID ID, ModuleDeps &MD);
 };
 
 /// Collects modular and non-modular dependencies of the main file by attaching
@@ -320,6 +331,13 @@ void resetBenignCodeGenOptions(frontend::ActionKind ProgramAction,
                                const LangOptions &LangOpts,
                                CodeGenOptions &CGOpts);
 
+/// Determine if \c Input can be resolved within a stable directory.
+///
+/// \param Directories Paths known to be in a stable location. e.g. Sysroot.
+/// \param Input Path to evaluate.
+bool isPathInStableDir(const ArrayRef<StringRef> Directories,
+                       const StringRef Input);
+
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang

clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

@@ -157,6 +157,32 @@ static void optimizeCWD(CowCompilerInvocation &BuildInvocation, StringRef CWD) {
   }
 }
 
+/// Check a subset of invocation options to determine whether the current
+/// context can safely be considered as stable.
+static bool areOptionsInStableDir(CowCompilerInvocation &BuildInvocation,
+                                  const ArrayRef<StringRef> StableDirs) {
+  const auto &HSOpts = BuildInvocation.getHeaderSearchOpts();
+  assert(isPathInStableDir(StableDirs, HSOpts.Sysroot) &&
+         "Sysroots differ between module dependencies and current TU");
+
+  assert(isPathInStableDir(StableDirs, HSOpts.ResourceDir) &&
+         "ResourceDirs differ between module dependencies and current TU");
+
+  for (const auto &Entry : HSOpts.UserEntries) {
+    if (!Entry.IgnoreSysRoot)
+      continue;
+    if (!isPathInStableDir(StableDirs, Entry.Path))
+      return false;
+  }
+
+  for (const auto &SysPrefix : HSOpts.SystemHeaderPrefixes) {
+    if (!isPathInStableDir(StableDirs, SysPrefix.Prefix))
+      return false;
+  }
+
+  return true;
+}
+
 static std::vector<std::string> splitString(std::string S, char Separator) {
   SmallVector<StringRef> Segments;
   StringRef(S).split(Segments, Separator, /*MaxSplit=*/-1, /*KeepEmpty=*/false);
@@ -212,6 +238,25 @@ void dependencies::resetBenignCodeGenOptions(frontend::ActionKind ProgramAction,
   }
 }
 
+bool dependencies::isPathInStableDir(const ArrayRef<StringRef> Directories,
+                                     const StringRef Input) {
+  auto PathStartsWith = [](StringRef Prefix, StringRef Path) {
+    auto PrefixIt = llvm::sys::path::begin(Prefix),
+         PrefixEnd = llvm::sys::path::end(Prefix);
+    for (auto PathIt = llvm::sys::path::begin(Path),
+              PathEnd = llvm::sys::path::end(Path);
+         PrefixIt != PrefixEnd && PathIt != PathEnd; ++PrefixIt, ++PathIt) {
+      if (*PrefixIt != *PathIt)
+        return false;
+    }
+    return PrefixIt == PrefixEnd;
+  };
+
+  return any_of(Directories, [&](StringRef Dir) {
+    return !Dir.empty() && PathStartsWith(Dir, Input);
+  });
+}
+
 static CowCompilerInvocation
 makeCommonInvocationForModuleBuild(CompilerInvocation CI) {
   CI.resetNonModularOptions();
@@ -698,6 +743,17 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
 
   MD.ID.ModuleName = M->getFullModuleName();
   MD.IsSystem = M->IsSystem;
+
+  // Start off with the assumption that this module is shareable when there
+  // is a sysroot provided. As more dependencies are discovered, check if those
+  // come from the provided shared directories.
+  const llvm::SmallVector<StringRef> StableDirs = {
+      MDC.ScanInstance.getHeaderSearchOpts().Sysroot,
+      MDC.ScanInstance.getHeaderSearchOpts().ResourceDir};
+  MD.IsInStableDirectories =
+      !StableDirs[0].empty() &&
+      (llvm::sys::path::root_directory(StableDirs[0]) != StableDirs[0]);
+
   // For modules which use export_as link name, the linked product that of the
   // corresponding export_as-named module.
   if (!M->UseExportAsModuleLinkName)
@@ -739,6 +795,12 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
   MDC.ScanInstance.getASTReader()->visitInputFileInfos(
       *MF, /*IncludeSystem=*/true,
       [&](const serialization::InputFileInfo &IFI, bool IsSystem) {
+        if (MD.IsInStableDirectories) {
+          auto FullFilePath = ASTReader::ResolveImportedPath(
+              PathBuf, IFI.UnresolvedImportedFilename, MF->BaseDirectory);
+          MD.IsInStableDirectories =
+              isPathInStableDir(StableDirs, *FullFilePath);
+        }
         if (!(IFI.TopLevel && IFI.ModuleMap))
           return;
         if (IFI.UnresolvedImportedFilenameAsRequested.ends_with(
@@ -780,6 +842,11 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
             }
           });
 
+  // Check provided input paths from the invocation for determining
+  // IsInStableDirectories.
+  if (MD.IsInStableDirectories)
+    MD.IsInStableDirectories = areOptionsInStableDir(CI, StableDirs);
+
   MDC.associateWithContextHash(CI, IgnoreCWD, MD);
 
   // Finish the compiler invocation. Requires dependencies and the context hash.
@@ -821,8 +888,13 @@ void ModuleDepCollectorPP::addModulePrebuiltDeps(
   for (const Module *Import : M->Imports)
     if (Import->getTopLevelModule() != M->getTopLevelModule())
       if (MDC.isPrebuiltModule(Import->getTopLevelModule()))
-        if (SeenSubmodules.insert(Import->getTopLevelModule()).second)
+        if (SeenSubmodules.insert(Import->getTopLevelModule()).second) {
           MD.PrebuiltModuleDeps.emplace_back(Import->getTopLevelModule());
+          // Conservatively consider the module as not coming from stable
+          // directories, as transitive dependencies from the prebuilt module
+          // have not been determined.
+          MD.IsInStableDirectories = false;
+        }
 }
 
 void ModuleDepCollectorPP::addAllSubmoduleDeps(
@@ -835,6 +907,13 @@ void ModuleDepCollectorPP::addAllSubmoduleDeps(
   });
 }
 
+void ModuleDepCollectorPP::addOneModuleDep(const Module *M, const ModuleID ID,
+                                           ModuleDeps &MD) {
+  MD.ClangModuleDeps.push_back(ID);
+  if (MD.IsInStableDirectories)
+    MD.IsInStableDirectories = MDC.ModularDeps[M]->IsInStableDirectories;
+}
+
 void ModuleDepCollectorPP::addModuleDep(
     const Module *M, ModuleDeps &MD,
     llvm::DenseSet<const Module *> &AddedModules) {
@@ -843,7 +922,7 @@ void ModuleDepCollectorPP::addModuleDep(
         !MDC.isPrebuiltModule(Import)) {
       if (auto ImportID = handleTopLevelModule(Import->getTopLevelModule()))
         if (AddedModules.insert(Import->getTopLevelModule()).second)
-          MD.ClangModuleDeps.push_back(*ImportID);
+          addOneModuleDep(Import->getTopLevelModule(), *ImportID, MD);
     }
   }
 }
@@ -867,7 +946,7 @@ void ModuleDepCollectorPP::addAffectingClangModule(
         !MDC.isPrebuiltModule(Affecting)) {
       if (auto ImportID = handleTopLevelModule(Affecting))
         if (AddedModules.insert(Affecting).second)
-          MD.ClangModuleDeps.push_back(*ImportID);
+          addOneModuleDep(Affecting, *ImportID, MD);
     }
   }
 }

clang/test/ClangScanDeps/modules-in-stable-dirs.c

@@ -0,0 +1,109 @@
+// This test verifies modules that are entirely comprised from stable directory inputs are captured in
+// dependency information.
+
+// The first compilation verifies that transitive dependencies on local input are captured.
+// The second compilation verifies that external paths are resolved when a 
+// vfsoverlay for determining is-in-stable-directories.
+
+// REQUIRES: shell
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s|DIR|%/t|g" %t/compile-commands.json.in > %t/compile-commands.json
+// RUN: sed -e "s|DIR|%/t|g" %t/overlay.json.template > %t/overlay.json
+// RUN: clang-scan-deps -compilation-database %t/compile-commands.json \
+// RUN:   -j 1 -format experimental-full > %t/deps.db
+// RUN: cat %t/deps.db | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
+
+// CHECK:   "modules": [
+// CHECK-NEXT:     {
+// CHECK:            "is-in-stable-directories": true,
+// CHECK:            "name": "A"
+
+// Verify that there are no more occurances of sysroot.
+// CHECK-NOT:            "is-in-stable-directories"
+
+// CHECK:            "name": "A"
+// CHECK:            "USE_VFS"
+// CHECK:            "name": "B"
+// CHECK:            "name": "C"
+// CHECK:            "name": "D"
+// CHECK:            "name": "NotInSDK"
+
+//--- compile-commands.json.in
+[
+{
+  "directory": "DIR",
+  "command": "clang -c DIR/client.c -isysroot DIR/Sysroot -IDIR/Sysroot/usr/include -IDIR/BuildDir -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps",
+  "file": "DIR/client.c"
+},
+{
+  "directory": "DIR",
+  "command": "clang -c DIR/client.c -isysroot DIR/Sysroot -IDIR/Sysroot/usr/include -ivfsoverlay DIR/overlay.json -DUSE_VFS -IDIR/BuildDir -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps",
+  "file": "DIR/client.c"
+}
+]
+
+//--- overlay.json.template
+{
+  "version": 0,
+  "case-sensitive": "false",
+  "roots": [
+    {
+          "external-contents": "DIR/SysrootButNotReally/A/A_vfs.h",
+          "name": "DIR/Sysroot/usr/include/A/A_vfs.h",
+          "type": "file"
+    }
+  ]
+}
+
+//--- Sysroot/usr/include/A/module.modulemap
+module A {
+  umbrella "."
+}
+
+//--- Sysroot/usr/include/A/A.h
+#ifdef USE_VFS
+#include <A/A_vfs.h>
+#endif 
+typedef int A_t;
+
+//--- SysrootButNotReally/A/A_vfs.h
+typedef int typeFromVFS;
+
+//--- Sysroot/usr/include/B/module.modulemap
+module B [system] {
+  umbrella "."
+}
+
+//--- Sysroot/usr/include/B/B.h
+#include <C/C.h>
+typedef int B_t;
+
+//--- Sysroot/usr/include/C/module.modulemap
+module C [system] {
+  umbrella "."
+}
+
+//--- Sysroot/usr/include/C/C.h
+#include <D/D.h>
+
+//--- Sysroot/usr/include/D/module.modulemap
+module D [system] {
+  umbrella "."
+}
+
+// Simulate a header that will be resolved in a local directory, from a sysroot header.
+//--- Sysroot/usr/include/D/D.h
+#include <HeaderNotFoundInSDK.h>
+
+//--- BuildDir/module.modulemap
+module NotInSDK [system] {
+  umbrella "."
+}
+
+//--- BuildDir/HeaderNotFoundInSDK.h
+typedef int local_t;
+
+//--- client.c
+#include <A/A.h>
+#include <B/B.h>