Update IsInSysroot -> IsShareable

* We need to track more than just sysroot, so start with sysroot and
  resource directory as a starting point, but allow the ability to
capture more as they are discovered in the future.

* Check a subset of the compiler invocation of module compilations for
  determining if a module can be shared.
* When a module has a prebuilt module dependency, consider is not
  shareable for now.

Author: cyndyishida

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

@@ -114,13 +114,13 @@ struct ModuleDeps {
   /// Whether this is a "system" module.
   bool IsSystem;
 
-  /// Whether this module is fully composed of file & module inputs from the
-  /// sysroot. External paths, as opposed to virtual file paths, are always used
-  /// for computing this value.
+  /// Whether this module is fully composed of file & module inputs from
+  /// locations likely to stay the same across the active development and build
+  /// cycle. For example, when all those input paths only resolve in Sysroot.
   ///
-  /// This attribute is useful for identifying modules that are unlikely to
-  /// change under an active development and build cycle.
-  bool IsInSysroot;
+  /// External paths, as opposed to virtual file paths, are always used
+  /// for computing this value.
+  bool IsShareable;
 
   /// The path to the modulemap file which defines this module.
   ///
@@ -229,7 +229,7 @@ class ModuleDepCollectorPP final : public PPCallbacks {
                           llvm::DenseSet<const Module *> &AddedModules);
 
   /// Add discovered module dependency for the given module.
-  void addClangModule(const Module *M, const ModuleID ID, ModuleDeps &MD);
+  void addOneModuleDep(const Module *M, const ModuleID ID, ModuleDeps &MD);
 };
 
 /// Collects modular and non-modular dependencies of the main file by attaching
@@ -331,6 +331,12 @@ void resetBenignCodeGenOptions(frontend::ActionKind ProgramAction,
                                const LangOptions &LangOpts,
                                CodeGenOptions &CGOpts);
 
+/// Determine if \c Input can be resolved within a shared location.
+///
+/// \param Directories Paths known to be in a shared location. e.g. Sysroot.
+/// \param Input Path to evaluate.
+bool isPathInSharedDir(ArrayRef<StringRef> Directories, const StringRef Input);
+
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang

clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

@@ -157,6 +157,32 @@ static void optimizeCWD(CowCompilerInvocation &BuildInvocation, StringRef CWD) {
   }
 }
 
+/// Check a subset of invocation options to determine whether the current
+/// context can safely be considered as shareable.
+static bool areOptionsInSharedDir(CowCompilerInvocation &BuildInvocation,
+                                  const ArrayRef<StringRef> SharedDirs) {
+  const auto &HSOpts = BuildInvocation.getHeaderSearchOpts();
+  if (!isPathInSharedDir(SharedDirs, HSOpts.Sysroot))
+    return false;
+
+  if (!isPathInSharedDir(SharedDirs, HSOpts.ResourceDir))
+    return false;
+
+  for (const auto &Entry : HSOpts.UserEntries) {
+    if (!Entry.IgnoreSysRoot)
+      continue;
+    if (!isPathInSharedDir(SharedDirs, Entry.Path))
+      return false;
+  }
+
+  for (const auto &SysPrefix : HSOpts.SystemHeaderPrefixes) {
+    if (!isPathInSharedDir(SharedDirs, SysPrefix.Prefix))
+      return false;
+  }
+
+  return true;
+}
+
 static std::vector<std::string> splitString(std::string S, char Separator) {
   SmallVector<StringRef> Segments;
   StringRef(S).split(Segments, Separator, /*MaxSplit=*/-1, /*KeepEmpty=*/false);
@@ -212,6 +238,25 @@ void dependencies::resetBenignCodeGenOptions(frontend::ActionKind ProgramAction,
   }
 }
 
+bool dependencies::isPathInSharedDir(ArrayRef<StringRef> Directories,
+                                     const StringRef Input) {
+  auto PathStartsWith = [](StringRef Prefix, StringRef Path) {
+    auto PrefixIt = llvm::sys::path::begin(Prefix),
+         PrefixEnd = llvm::sys::path::end(Prefix);
+    for (auto PathIt = llvm::sys::path::begin(Path),
+              PathEnd = llvm::sys::path::end(Path);
+         PrefixIt != PrefixEnd && PathIt != PathEnd; ++PrefixIt, ++PathIt) {
+      if (*PrefixIt != *PathIt)
+        return false;
+    }
+    return PrefixIt == PrefixEnd;
+  };
+
+  return any_of(Directories, [&](StringRef Dir) {
+    return !Dir.empty() && PathStartsWith(Dir, Input);
+  });
+}
+
 static CowCompilerInvocation
 makeCommonInvocationForModuleBuild(CompilerInvocation CI) {
   CI.resetNonModularOptions();
@@ -699,13 +744,15 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
   MD.ID.ModuleName = M->getFullModuleName();
   MD.IsSystem = M->IsSystem;
 
-  // Start off with the assumption that this module is in the sysroot when there
+  // Start off with the assumption that this module is shareable when there
   // is a sysroot provided. As more dependencies are discovered, check if those
-  // come from the provided sysroot.
-  const StringRef CurrSysroot = MDC.ScanInstance.getHeaderSearchOpts().Sysroot;
-  MD.IsInSysroot =
-      !CurrSysroot.empty() &&
-      (llvm::sys::path::root_directory(CurrSysroot) != CurrSysroot);
+  // come from the provided shared directories.
+  const llvm::SmallVector<StringRef> SharedDirs = {
+      MDC.ScanInstance.getHeaderSearchOpts().Sysroot,
+      MDC.ScanInstance.getHeaderSearchOpts().ResourceDir};
+  MD.IsShareable =
+      !SharedDirs[0].empty() &&
+      (llvm::sys::path::root_directory(SharedDirs[0]) != SharedDirs[0]);
 
   // For modules which use export_as link name, the linked product that of the
   // corresponding export_as-named module.
@@ -748,10 +795,10 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
   MDC.ScanInstance.getASTReader()->visitInputFileInfos(
       *MF, /*IncludeSystem=*/true,
       [&](const serialization::InputFileInfo &IFI, bool IsSystem) {
-        if (MD.IsInSysroot) {
+        if (MD.IsShareable) {
           auto FullFilePath = ASTReader::ResolveImportedPath(
               PathBuf, IFI.UnresolvedImportedFilename, MF->BaseDirectory);
-          MD.IsInSysroot = FullFilePath->starts_with(CurrSysroot);
+          MD.IsShareable = isPathInSharedDir(SharedDirs, *FullFilePath);
           PathBuf.resize_for_overwrite(256);
         }
         if (!(IFI.TopLevel && IFI.ModuleMap))
@@ -795,6 +842,10 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
             }
           });
 
+  // Check provided input paths from the invocation for determining IsShareable.
+  if (MD.IsShareable)
+    MD.IsShareable = areOptionsInSharedDir(CI, SharedDirs);
+
   MDC.associateWithContextHash(CI, IgnoreCWD, MD);
 
   // Finish the compiler invocation. Requires dependencies and the context hash.
@@ -836,8 +887,13 @@ void ModuleDepCollectorPP::addModulePrebuiltDeps(
   for (const Module *Import : M->Imports)
     if (Import->getTopLevelModule() != M->getTopLevelModule())
       if (MDC.isPrebuiltModule(Import->getTopLevelModule()))
-        if (SeenSubmodules.insert(Import->getTopLevelModule()).second)
+        if (SeenSubmodules.insert(Import->getTopLevelModule()).second) {
           MD.PrebuiltModuleDeps.emplace_back(Import->getTopLevelModule());
+          // Conservatively consider the module as not shareable,
+          // as transitive dependencies from the prebuilt module have not been
+          // determined.
+          MD.IsShareable = false;
+        }
 }
 
 void ModuleDepCollectorPP::addAllSubmoduleDeps(
@@ -850,11 +906,11 @@ void ModuleDepCollectorPP::addAllSubmoduleDeps(
   });
 }
 
-void ModuleDepCollectorPP::addClangModule(const Module *M, const ModuleID ID,
-                                          ModuleDeps &MD) {
+void ModuleDepCollectorPP::addOneModuleDep(const Module *M, const ModuleID ID,
+                                           ModuleDeps &MD) {
   MD.ClangModuleDeps.push_back(ID);
-  if (MD.IsInSysroot)
-    MD.IsInSysroot = MDC.ModularDeps[M]->IsInSysroot;
+  if (MD.IsShareable)
+    MD.IsShareable = MDC.ModularDeps[M]->IsShareable;
 }
 
 void ModuleDepCollectorPP::addModuleDep(
@@ -865,7 +921,7 @@ void ModuleDepCollectorPP::addModuleDep(
         !MDC.isPrebuiltModule(Import)) {
       if (auto ImportID = handleTopLevelModule(Import->getTopLevelModule()))
         if (AddedModules.insert(Import->getTopLevelModule()).second)
-          addClangModule(Import->getTopLevelModule(), *ImportID, MD);
+          addOneModuleDep(Import->getTopLevelModule(), *ImportID, MD);
     }
   }
 }
@@ -889,7 +945,7 @@ void ModuleDepCollectorPP::addAffectingClangModule(
         !MDC.isPrebuiltModule(Affecting)) {
       if (auto ImportID = handleTopLevelModule(Affecting))
         if (AddedModules.insert(Affecting).second)
-          addClangModule(Affecting, *ImportID, MD);
+          addOneModuleDep(Affecting, *ImportID, MD);
     }
   }
 }

clang/test/ClangScanDeps/modules-in-sysroot.c renamed to clang/test/ClangScanDeps/modules-in-shared-dirs.c

@@ -1,8 +1,9 @@
-// This test verifies modules that are entirely comprised from sysroot inputs are captured in
+// This test verifies modules that are entirely comprised from shared directory inputs are captured in
 // dependency information.
 
-// The first compilation verifies that transitive dependencies on non-sysroot input are captured.
-// The second compilation verifies that external paths are resolved when a vfsoverlay is applied when considering sysroot-ness.
+// The first compilation verifies that transitive dependencies on local input are captured.
+// The second compilation verifies that external paths are resolved when a 
+// vfsoverlay for determining is-shareable.
 
 // REQUIRES: shell
 // RUN: rm -rf %t
@@ -15,11 +16,11 @@
 
 // CHECK:   "modules": [
 // CHECK-NEXT:     {
-// CHECK:            "is-in-sysroot": true,
+// CHECK:            "is-shareable": true,
 // CHECK:            "name": "A"
 
 // Verify that there are no more occurances of sysroot.
-// CHECK-NOT:            "is-in-sysroot"
+// CHECK-NOT:            "is-shareable"
 
 // CHECK:            "name": "A"
 // CHECK:            "USE_VFS"
@@ -32,12 +33,12 @@
 [
 {
   "directory": "DIR",
-  "command": "clang -c DIR/client.c -isysroot DIR/MacOSX.sdk -IDIR/MacOSX.sdk/usr/include -IDIR/BuildDir -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps",
+  "command": "clang -c DIR/client.c -isysroot DIR/Sysroot -IDIR/Sysroot/usr/include -IDIR/BuildDir -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps",
   "file": "DIR/client.c"
 },
 {
   "directory": "DIR",
-  "command": "clang -c DIR/client.c -isysroot DIR/MacOSX.sdk -IDIR/MacOSX.sdk/usr/include -ivfsoverlay DIR/overlay.json -DUSE_VFS -IDIR/BuildDir -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps",
+  "command": "clang -c DIR/client.c -isysroot DIR/Sysroot -IDIR/Sysroot/usr/include -ivfsoverlay DIR/overlay.json -DUSE_VFS -IDIR/BuildDir -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps",
   "file": "DIR/client.c"
 }
 ]
@@ -48,51 +49,51 @@
   "case-sensitive": "false",
   "roots": [
     {
-          "external-contents": "DIR/local/A/A_vfs.h",
-          "name": "DIR/MacOSX.sdk/usr/include/A/A_vfs.h",
+          "external-contents": "DIR/SysrootButNotReally/A/A_vfs.h",
+          "name": "DIR/Sysroot/usr/include/A/A_vfs.h",
           "type": "file"
     }
   ]
 }
 
-//--- MacOSX.sdk/usr/include/A/module.modulemap
+//--- Sysroot/usr/include/A/module.modulemap
 module A {
   umbrella "."
 }
 
-//--- MacOSX.sdk/usr/include/A/A.h
+//--- Sysroot/usr/include/A/A.h
 #ifdef USE_VFS
 #include <A/A_vfs.h>
 #endif 
 typedef int A_t;
 
-//--- local/A/A_vfs.h
+//--- SysrootButNotReally/A/A_vfs.h
 typedef int typeFromVFS;
 
-//--- MacOSX.sdk/usr/include/B/module.modulemap
+//--- Sysroot/usr/include/B/module.modulemap
 module B [system] {
   umbrella "."
 }
 
-//--- MacOSX.sdk/usr/include/B/B.h
+//--- Sysroot/usr/include/B/B.h
 #include <C/C.h>
 typedef int B_t;
 
-//--- MacOSX.sdk/usr/include/C/module.modulemap
+//--- Sysroot/usr/include/C/module.modulemap
 module C [system] {
   umbrella "."
 }
 
-//--- MacOSX.sdk/usr/include/C/C.h
+//--- Sysroot/usr/include/C/C.h
 #include <D/D.h>
 
-//--- MacOSX.sdk/usr/include/D/module.modulemap
+//--- Sysroot/usr/include/D/module.modulemap
 module D [system] {
   umbrella "."
 }
 
 // Simulate a header that will be resolved in a local directory, from a sysroot header.
-//--- MacOSX.sdk/usr/include/D/D.h
+//--- Sysroot/usr/include/D/D.h
 #include <HeaderNotFoundInSDK.h>
 
 //--- BuildDir/module.modulemap