[Dependency Scanning] Factor ModuleDependenciesCache out of a CompilerInstance into callers of scanDependencies

It is now the responsibility of the scanDependencies code to instantiate (and share) the cache.
e.g. FrontendTool instantiates a new cache per `-scan-dependencies` invocation, and the DependencyScanningTool keeps one shared cache across its lifetime.

Author: artemcm

include/swift/Frontend/Frontend.h

@@ -491,8 +491,6 @@ class CompilerInstance {
   DependencyTracker *getDependencyTracker() { return DepTracker.get(); }
   const DependencyTracker *getDependencyTracker() const { return DepTracker.get(); }
 
-  ModuleDependenciesCache *getModuleDependencyCache() { return ModDepCache.get(); }
-
   UnifiedStatsReporter *getStatsReporter() const { return Stats.get(); }
 
   /// Retrieve the main module containing the files being compiled.
@@ -558,7 +556,6 @@ class CompilerInstance {
   bool setUpVirtualFileSystemOverlays();
   void setUpLLVMArguments();
   void setUpDiagnosticOptions();
-  void setUpModuleDependencyCacheIfNeeded();
   bool setUpModuleLoaders();
   bool setUpInputs();
   bool setUpASTContextIfNeeded();

include/swift/SwiftScan/DependencyScanningTool.h

@@ -35,15 +35,15 @@ class DependencyScanningTool {
   ///
   /// \returns a \c StringError with the diagnostic output if clang errors
   /// occurred, \c FullDependencies otherwise.
-  std::string
+  llvm::ErrorOr<std::string>
   getFullDependencies(ArrayRef<const char *> Command,
                       const llvm::StringSet<> &InputFiles,
                       const llvm::StringSet<> &PlaceholderModules);
 
 private:
   /// Shared cache of module dependencies, re-used by individual queries
   /// during the lifetime of this Tool
-  ModuleDependenciesCache GlobalCache;
+  std::unique_ptr<ModuleDependenciesCache> SharedCache;
 
   llvm::BumpPtrAllocator Alloc;
   llvm::StringSaver Saver;

include/swift/SwiftScan/ScanDependencies.h

@@ -19,23 +19,27 @@ namespace swift {
 
 class CompilerInvocation;
 class CompilerInstance;
+class ModuleDependenciesCache;
 
 /// Batch scan the dependencies for modules specified in \c batchInputFile.
 bool batchScanModuleDependencies(CompilerInstance &instance,
                                  llvm::StringRef batchInputFile);
 
 /// Scans the dependencies of the main module of \c instance and writes out
-/// the result in JSON
+/// the resulting JSON according to the instance's output parameters.
+/// This method is used for swift-frontend invocations in dependency scanning mode
+/// (-scan-dependencies), where the module dependency cache is not shared.
 bool scanAndOutputDependencies(CompilerInstance &instance);
 
-/// Scans the dependencies of the main module of \c instance.
-bool scanDependencies(CompilerInstance &instance,
-                      llvm::raw_ostream &out);
-
 /// Scans the dependencies of the underlying clang module of the main module
 /// of \c instance.
 bool scanClangDependencies(CompilerInstance &instance);
 
+/// Scans the dependencies of the main module of \c instance.
+bool scanDependencies(CompilerInstance &instance,
+                      ModuleDependenciesCache &cache,
+                      llvm::raw_ostream &out);
+
 } // end namespace swift
 
 #endif

lib/Frontend/Frontend.cpp

@@ -321,22 +321,11 @@ void CompilerInstance::setupDependencyTrackerIfNeeded() {
   DepTracker = std::make_unique<DependencyTracker>(*collectionMode);
 }
 
-void CompilerInstance::setUpModuleDependencyCacheIfNeeded() {
-  const auto &Invocation = getInvocation();
-  const auto &opts = Invocation.getFrontendOptions();
-  if (opts.RequestedAction == FrontendOptions::ActionType::ScanDependencies ||
-      opts.RequestedAction == FrontendOptions::ActionType::ScanClangDependencies) {
-    ModDepCache = std::make_unique<ModuleDependenciesCache>();
-  }
-}
-
 bool CompilerInstance::setup(const CompilerInvocation &Invok) {
   Invocation = Invok;
 
   setupDependencyTrackerIfNeeded();
 
-  setUpModuleDependencyCacheIfNeeded();
-
   // If initializing the overlay file system fails there's no sense in
   // continuing because the compiler will read the wrong files.
   if (setUpVirtualFileSystemOverlays())

lib/FrontendTool/CMakeLists.txt

@@ -4,7 +4,6 @@ add_swift_host_library(swiftFrontendTool STATIC
   ImportedModules.cpp
   LoadedModuleTrace.cpp
   MakeStyleDependencies.cpp
-  ScanDependencies.cpp
   TBD.cpp)
 add_dependencies(swiftFrontendTool
   swift-syntax-generated-headers

lib/FrontendTool/FrontendTool.cpp

@@ -22,7 +22,7 @@
 
 #include "swift/FrontendTool/FrontendTool.h"
 #include "swift/SwiftScan/ScanDependencies.h"
-#include "ImportedModules.h"
+#include "Dependencies.h"
 #include "TBD.h"
 #include "swift/Subsystems.h"
 #include "swift/AST/DiagnosticsFrontend.h"
@@ -1134,6 +1134,7 @@ withSemanticAnalysis(CompilerInstance &Instance, FrontendObserver *observer,
 static bool performScanDependencies(CompilerInstance &Instance) {
   auto batchScanInput =
       Instance.getASTContext().SearchPathOpts.BatchScanInputFilePath;
+  ModuleDependenciesCache SingleUseCache;
   if (batchScanInput.empty()) {
     return scanAndOutputDependencies(Instance);
   } else {

lib/SwiftScan/DependencyScanningTool.cpp

@@ -26,12 +26,10 @@ namespace swift {
 namespace dependencies {
 
 DependencyScanningTool::DependencyScanningTool()
-: GlobalCache(), Alloc(), Saver(Alloc) {
-  
-
-}
+: SharedCache(std::make_unique<ModuleDependenciesCache>()),
+  Alloc(), Saver(Alloc) {}
 
-std::string
+llvm::ErrorOr<std::string>
 DependencyScanningTool::getFullDependencies(ArrayRef<const char *> Command,
                                             const llvm::StringSet<> &InputFiles,
                                             const llvm::StringSet<> &PlaceholderModules) {
@@ -43,7 +41,7 @@ DependencyScanningTool::getFullDependencies(ArrayRef<const char *> Command,
   // Basic error checking on the arguments
   if (Command.empty()) {
     Instance->getDiags().diagnose(SourceLoc(), diag::error_no_frontend_args);
-    return "Error";
+    return std::make_error_code(std::errc::not_supported);
   }
 
   CompilerInvocation Invocation;
@@ -59,15 +57,15 @@ DependencyScanningTool::getFullDependencies(ArrayRef<const char *> Command,
   SmallVector<const char*, 4> Args;
   llvm::cl::TokenizeGNUCommandLine(CommandString, Saver, Args);
   if (Invocation.parseArgs(Args, Instance->getDiags())) {
-    return "Error";
+    return std::make_error_code(std::errc::not_supported);
   }
 
   // Setup the instance
   Instance->setup(Invocation);
 
   std::string JSONOutput;
   llvm::raw_string_ostream Oss(JSONOutput);
-  scanDependencies(*Instance, Oss);
+  scanDependencies(*Instance, *SharedCache, Oss);
   Oss.flush();
 
   return JSONOutput;

lib/SwiftScan/ScanDependencies.cpp

@@ -670,6 +670,7 @@ static bool diagnoseCycle(CompilerInstance &instance,
 }
 
 static bool scanModuleDependencies(CompilerInstance &instance,
+                                   ModuleDependenciesCache &cache,
                                    StringRef moduleName,
                                    bool isClang,
                                    StringRef outputPath) {
@@ -681,10 +682,6 @@ static bool scanModuleDependencies(CompilerInstance &instance,
 
   llvm::SetVector<ModuleDependencyID, std::vector<ModuleDependencyID>,
                   std::set<ModuleDependencyID>> allModules;
-  // Retrieve the instance's module dependency cache.
-  ModuleDependenciesCache *cache = instance.getModuleDependencyCache();
-  assert(cache &&
-         "Dependency Scanner expected a ModuleDependenciesCache on a compiler instance.");
   InterfaceSubContextDelegateImpl ASTDelegate(ctx.SourceMgr, ctx.Diags,
                                               ctx.SearchPathOpts, ctx.LangOpts,
                                               ctx.ClangImporterOpts,
@@ -700,11 +697,11 @@ static bool scanModuleDependencies(CompilerInstance &instance,
   if (isClang) {
     // Loading the clang module using Clang importer.
     // This action will populate the cache with the main module's dependencies.
-    rootDeps = ctx.getModuleDependencies(moduleName, /*IsClang*/true, *cache,
+    rootDeps = ctx.getModuleDependencies(moduleName, /*IsClang*/true, cache,
                                          ASTDelegate);
   } else {
     // Loading the swift module's dependencies.
-    rootDeps = ctx.getSwiftModuleDependencies(moduleName, *cache, ASTDelegate);
+    rootDeps = ctx.getSwiftModuleDependencies(moduleName, cache, ASTDelegate);
   }
   if (!rootDeps.hasValue()) {
     // We cannot find the clang module, abort.
@@ -726,16 +723,17 @@ static bool scanModuleDependencies(CompilerInstance &instance,
        ++currentModuleIdx) {
     auto module = allModules[currentModuleIdx];
     auto discoveredModules =
-        resolveDirectDependencies(instance, module, *cache, ASTDelegate);
+        resolveDirectDependencies(instance, module, cache, ASTDelegate);
     allModules.insert(discoveredModules.begin(), discoveredModules.end());
   }
   // Write out the JSON description.
-  writeJSON(out, instance, *cache, ASTDelegate, allModules.getArrayRef());
+  writeJSON(out, instance, cache, ASTDelegate, allModules.getArrayRef());
   return false;
 }
 
 bool swift::scanClangDependencies(CompilerInstance &instance) {
-  return scanModuleDependencies(instance,
+  ModuleDependenciesCache cache;
+  return scanModuleDependencies(instance, cache,
                                 instance.getMainModule()->getNameStr(),
                                 /*isClang*/true,
                                 instance.getInvocation().getFrontendOptions()
@@ -745,6 +743,9 @@ bool swift::scanClangDependencies(CompilerInstance &instance) {
 bool swift::batchScanModuleDependencies(CompilerInstance &instance,
                                         llvm::StringRef batchInputFile) {
   const CompilerInvocation &invok = instance.getInvocation();
+  // The primary cache used for scans carried out with the compiler instance
+  // we have created
+  ModuleDependenciesCache cache;
 
   (void)instance.getMainModule();
   llvm::BumpPtrAllocator alloc;
@@ -755,20 +756,31 @@ bool swift::batchScanModuleDependencies(CompilerInstance &instance,
     return true;
   auto &diags = instance.getDiags();
   ForwardingDiagnosticConsumer FDC(diags);
-  // Keep track of all compiler instances we have created.
-  llvm::StringMap<std::unique_ptr<CompilerInstance>> subInstanceMap;
+
+  // Keep track of all compiler instances and dependency caches we have created.
+  // TODO: Re-use a single cache across all invocations, once `alreadySeen`
+  // state is no longer shared.
+  llvm::StringMap<std::pair<std::unique_ptr<CompilerInstance>,
+                            std::unique_ptr<ModuleDependenciesCache>>> subInstanceMap;
   for (auto &entry: *results) {
     CompilerInstance *pInstance = nullptr;
+    ModuleDependenciesCache *pCache = nullptr;
     if (entry.arguments.empty()) {
       // Use the compiler's instance if no arguments are specified.
       pInstance = &instance;
+      pCache = &cache;
     } else if (subInstanceMap.count(entry.arguments)) {
       // Use the previously created instance if we've seen the arguments before.
-      pInstance = subInstanceMap[entry.arguments].get();
+      pInstance = subInstanceMap[entry.arguments].first.get();
+      pCache = subInstanceMap[entry.arguments].second.get();
     } else {
       // Create a new instance by the arguments and save it in the map.
-      pInstance = subInstanceMap.insert({entry.arguments,
-        std::make_unique<CompilerInstance>()}).first->getValue().get();
+      subInstanceMap.insert({entry.arguments,
+                  std::make_pair(std::make_unique<CompilerInstance>(),
+                                 std::make_unique<ModuleDependenciesCache>())});
+
+      pInstance = subInstanceMap[entry.arguments].first.get();
+      pCache = subInstanceMap[entry.arguments].second.get();
       SmallVector<const char*, 4> args;
       llvm::cl::TokenizeGNUCommandLine(entry.arguments, saver, args);
       CompilerInvocation subInvok = invok;
@@ -786,8 +798,8 @@ bool swift::batchScanModuleDependencies(CompilerInstance &instance,
     }
     assert(pInstance);
     // Scan using the chosen compiler instance.
-    if (scanModuleDependencies(*pInstance, entry.moduleName, !entry.isSwift,
-                               entry.outputPath)) {
+    if (scanModuleDependencies(*pInstance, *pCache, entry.moduleName,
+                               !entry.isSwift, entry.outputPath)) {
       return true;
     }
   }
@@ -801,6 +813,8 @@ bool swift::scanAndOutputDependencies(CompilerInstance &instance) {
   std::error_code EC;
   llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::F_None);
 
+  ModuleDependenciesCache SingleUseCache;
+
   if (out.has_error() || EC) {
     Context.Diags.diagnose(SourceLoc(), diag::error_opening_output, path,
                            EC.message());
@@ -809,10 +823,11 @@ bool swift::scanAndOutputDependencies(CompilerInstance &instance) {
   }
 
   // Execute scan, writing JSON output to the output stream
-  return scanDependencies(instance, out);
+  return scanDependencies(instance, SingleUseCache, out);
 }
 
 bool swift::scanDependencies(CompilerInstance &instance,
+                             ModuleDependenciesCache &cache,
                              llvm::raw_ostream &out) {
   ASTContext &Context = instance.getASTContext();
   ModuleDecl *mainModule = instance.getMainModule();
@@ -900,11 +915,7 @@ bool swift::scanDependencies(CompilerInstance &instance,
 
   allModules.insert({mainModuleName.str(), mainDependencies.getKind()});
 
-  // Retrieve the instance's module dependency cache.
-  ModuleDependenciesCache *cache = instance.getModuleDependencyCache();
-  assert(cache &&
-         "Dependency Scanner expected a ModuleDependenciesCache on a compiler instance.");
-  cache->recordDependencies(mainModuleName, std::move(mainDependencies));
+  cache.recordDependencies(mainModuleName, std::move(mainDependencies));
 
   auto &ctx = instance.getASTContext();
   auto ModuleCachePath = getModuleCachePathFromClang(ctx
@@ -927,28 +938,28 @@ bool swift::scanDependencies(CompilerInstance &instance,
        ++currentModuleIdx) {
     auto module = allModules[currentModuleIdx];
     auto discoveredModules =
-        resolveDirectDependencies(instance, module, *cache, ASTDelegate);
+        resolveDirectDependencies(instance, module, cache, ASTDelegate);
     allModules.insert(discoveredModules.begin(), discoveredModules.end());
   }
 
   // We have all explicit imports now, resolve cross import overlays.
   discoverCrosssImportOverlayDependencies(instance, mainModuleName,
-      /*All transitive dependencies*/allModules.getArrayRef().slice(1), *cache,
+      /*All transitive dependencies*/allModules.getArrayRef().slice(1), cache,
       ASTDelegate, [&](ModuleDependencyID id) {
     allModules.insert(id);
   });
 
   // Dignose cycle in dependency graph.
-  if (diagnoseCycle(instance, *cache, /*MainModule*/allModules.front(), ASTDelegate))
+  if (diagnoseCycle(instance, cache, /*MainModule*/allModules.front(), ASTDelegate))
     return true;
 
   // Write out the JSON description.
-  writeJSON(out, instance, *cache, ASTDelegate, allModules.getArrayRef());
+  writeJSON(out, instance, cache, ASTDelegate, allModules.getArrayRef());
 
   // Update the dependency tracker.
   if (auto depTracker = instance.getDependencyTracker()) {
     for (auto module : allModules) {
-      auto deps = cache->findDependencies(module.first, module.second);
+      auto deps = cache.findDependencies(module.first, module.second);
       if (!deps)
         continue;
 

unittests/SwiftScan/ModuleDeps.cpp

@@ -160,7 +160,7 @@ export *\n\
                         " -I " + CHeadersDirPath +
                         " -I " + StdLibDir.str().str() +
                         " -I " + ShimsLibDir.str().str();
-  std::string deps = ScannerTool.getFullDependencies(Command.c_str(),
+  llvm::ErrorOr<std::string> deps = ScannerTool.getFullDependencies(Command.c_str(),
                                                      {TestPathStr.c_str()}, {});
 
   // TODO: Output/verify dependency graph correctness