DependenciesScanner: implement protocol for batch module dependencies scan

This scanning mode allows swift-driver to query module dependencies in a batch
and in a more granular way. In short term, it could help solve a problem that
clang module dependencies may vary if target triple changes. In a longer term,
we could break a holistic dependencies graph into smaller pieces for better caching
and reusing.

This change doesn't include the implementation of using the specified scanner
arguments to set up Clang dependencies scanner. It will come in later commits.

Author: nkcsgexi

include/swift/AST/ASTContext.h

@@ -739,6 +739,12 @@ class ASTContext final {
       ModuleDependenciesCache &cache,
       InterfaceSubContextDelegate &delegate);
 
+  /// Retrieve the module dependencies for the Swift module with the given name.
+  Optional<ModuleDependencies> getSwiftModuleDependencies(
+      StringRef moduleName,
+      ModuleDependenciesCache &cache,
+      InterfaceSubContextDelegate &delegate);
+
   /// Load extensions to the given nominal type from the external
   /// module loaders.
   ///

include/swift/AST/DiagnosticsFrontend.def

@@ -274,6 +274,14 @@ ERROR(placeholder_dependency_module_map_corrupted,none,
       "Swift placeholder dependency module map from %0 is malformed",
       (StringRef))
 
+ERROR(batch_scan_input_file_missing,none,
+      "cannot open batch dependencies scan input file from %0",
+      (StringRef))
+
+ERROR(batch_scan_input_file_corrupted,none,
+      "batch dependencies scan input file from %0 is malformed",
+      (StringRef))
+
 REMARK(default_previous_install_name, none,
       "default previous install name for %0 is %1", (StringRef, StringRef))
 

include/swift/AST/SearchPathOptions.h

@@ -93,6 +93,9 @@ class SearchPathOptions {
 
   /// A map of placeholder Swift module dependency information.
   std::string PlaceholderDependencyModuleMap;
+
+  /// A file containing modules we should perform batch scanning.
+  std::string BatchScanInputFilePath;
 private:
   static StringRef
   pathStringFromFrameworkSearchPath(const FrameworkSearchPath &next) {

include/swift/Option/FrontendOptions.td

@@ -228,6 +228,10 @@ def explict_swift_module_map
 def placeholder_dependency_module_map
   : Separate<["-"], "placeholder-dependency-module-map-file">, MetaVarName<"<path>">,
     HelpText<"Specify a JSON file containing information of external Swift module dependencies">;
+
+def batch_scan_input_file
+  : Separate<["-"], "batch-scan-input-file">, MetaVarName<"<path>">,
+    HelpText<"Specify a JSON file containing modules to perform batch dependencies scanning">;
 }
 
 

lib/AST/ASTContext.cpp

@@ -1516,6 +1516,21 @@ Optional<ModuleDependencies> ASTContext::getModuleDependencies(
   return None;
 }
 
+Optional<ModuleDependencies>
+ASTContext::getSwiftModuleDependencies(StringRef moduleName,
+                                       ModuleDependenciesCache &cache,
+                                       InterfaceSubContextDelegate &delegate) {
+  for (auto &loader : getImpl().ModuleLoaders) {
+    if (loader.get() == getImpl().TheClangModuleLoader)
+      continue;
+
+    if (auto dependencies = loader->getModuleDependencies(moduleName, cache,
+                                                          delegate))
+      return dependencies;
+  }
+  return None;
+}
+
 void ASTContext::loadExtensions(NominalTypeDecl *nominal,
                                 unsigned previousGeneration) {
   PrettyStackTraceDecl stackTrace("loading extensions for", nominal);

lib/Frontend/CompilerInvocation.cpp

@@ -926,7 +926,8 @@ static bool ParseSearchPathArgs(SearchPathOptions &Opts,
   }
   if (const Arg *A = Args.getLastArg(OPT_placeholder_dependency_module_map))
     Opts.PlaceholderDependencyModuleMap = A->getValue();
-
+  if (const Arg *A = Args.getLastArg(OPT_batch_scan_input_file))
+    Opts.BatchScanInputFilePath = A->getValue();
   // Opts.RuntimeIncludePath is set by calls to
   // setRuntimeIncludePath() or setMainExecutablePath().
   // Opts.RuntimeImportPath is set by calls to

lib/FrontendTool/FrontendTool.cpp

@@ -1833,8 +1833,14 @@ static bool performCompile(CompilerInstance &Instance,
     return finishPipeline(Context.hadError());
   }
 
-  if (Action == FrontendOptions::ActionType::ScanDependencies)
-    return finishPipeline(scanDependencies(Instance));
+  if (Action == FrontendOptions::ActionType::ScanDependencies) {
+    auto batchScanInput = Instance.getASTContext().SearchPathOpts.BatchScanInputFilePath;
+    if (batchScanInput.empty())
+      return finishPipeline(scanDependencies(Instance));
+    else
+      return finishPipeline(batchScanModuleDependencies(Instance,
+                                                        batchScanInput));
+  }
 
   if (Action == FrontendOptions::ActionType::ScanClangDependencies)
     return finishPipeline(scanClangDependencies(Instance));

lib/FrontendTool/ScanDependencies.cpp

@@ -32,12 +32,101 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/StringSaver.h"
+#include "llvm/Support/YAMLTraits.h"
+#include "llvm/Support/YAMLParser.h"
 #include <set>
 
 using namespace swift;
+using namespace llvm::yaml;
 
 namespace {
+struct BatchScanInput {
+  StringRef moduleName;
+  StringRef arguments;
+  StringRef outputPath;
+};
+
+static std::string getScalaNodeText(Node *N) {
+  SmallString<32> Buffer;
+  return cast<ScalarNode>(N)->getValue(Buffer).str();
+}
+
+/// Parse an entry like this, where the "platforms" key-value pair is optional:
+///  {
+///     "module": "Foo.pcm",
+///     "arguments": "-target 10.15",
+///     "output": "../Foo.json"
+///  },
+static bool parseBatchInputEntries(ASTContext &Ctx, llvm::StringSaver &saver,
+                                   Node *Node, std::vector<BatchScanInput> &result) {
+  auto *SN = cast<SequenceNode>(Node);
+  if (!SN)
+    return true;
+  for (auto It = SN->begin(); It != SN->end(); ++It) {
+    auto *MN = cast<MappingNode>(&*It);
+    BatchScanInput entry;
+    Optional<std::set<int8_t>> Platforms;
+    for (auto &Pair: *MN) {
+      auto Key = getScalaNodeText(Pair.getKey());
+      auto* Value = Pair.getValue();
+      if (Key == "module") {
+        entry.moduleName = saver.save(getScalaNodeText(Value));
+      } else if (Key == "arguments") {
+        entry.arguments = saver.save(getScalaNodeText(Value));
+      } else if (Key == "output") {
+        entry.outputPath = saver.save(getScalaNodeText(Value));
+      } else {
+        // Future proof.
+        continue;
+      }
+    }
+    if (entry.moduleName.empty())
+      return true;
+    if (entry.outputPath.empty())
+      return true;
+    auto ext = llvm::sys::path::extension(entry.moduleName);
+    if (ext != ".swiftmodule" && ext != ".pcm")
+      return true;
+    result.emplace_back(std::move(entry));
+  }
+  return false;
+}
 
+static Optional<std::vector<BatchScanInput>>
+parseBatchScanInputFile(ASTContext &ctx, StringRef batchInputPath,
+                        llvm::StringSaver &saver) {
+  assert(!batchInputPath.empty());
+  namespace yaml = llvm::yaml;
+  std::vector<BatchScanInput> result;
+
+  // Load the input file.
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileBufOrErr =
+    llvm::MemoryBuffer::getFile(batchInputPath);
+  if (!FileBufOrErr) {
+    ctx.Diags.diagnose(SourceLoc(), diag::batch_scan_input_file_missing,
+                       batchInputPath);
+    return None;
+  }
+  StringRef Buffer = FileBufOrErr->get()->getBuffer();
+
+  // Use a new source manager instead of the one from ASTContext because we
+  // don't want the Json file to be persistent.
+  SourceManager SM;
+  yaml::Stream Stream(llvm::MemoryBufferRef(Buffer, batchInputPath),
+                      SM.getLLVMSourceMgr());
+  for (auto DI = Stream.begin(); DI != Stream.end(); ++ DI) {
+    assert(DI != Stream.end() && "Failed to read a document");
+    yaml::Node *N = DI->getRoot();
+    assert(N && "Failed to find a root");
+    if (parseBatchInputEntries(ctx, saver, N, result)) {
+      ctx.Diags.diagnose(SourceLoc(), diag::batch_scan_input_file_corrupted,
+                         batchInputPath);
+      return None;
+    }
+  }
+  return result;
+}
 }
 
 /// Find all of the imported Clang modules starting with the given module name.
@@ -533,15 +622,17 @@ static bool diagnoseCycle(CompilerInstance &instance,
   return false;
 }
 
-bool swift::scanClangDependencies(CompilerInstance &instance) {
+static bool scanModuleDependencies(CompilerInstance &instance,
+                                   StringRef moduleName,
+                                   StringRef arguments,
+                                   bool isClang,
+                                   StringRef outputPath) {
   ASTContext &ctx = instance.getASTContext();
-  ModuleDecl *mainModule = instance.getMainModule();
   auto &FEOpts = instance.getInvocation().getFrontendOptions();
   ModuleInterfaceLoaderOptions LoaderOpts(FEOpts);
   auto ModuleCachePath = getModuleCachePathFromClang(ctx
     .getClangModuleLoader()->getClangInstance());
 
-  StringRef mainModuleName = mainModule->getNameStr();
   llvm::SetVector<ModuleDependencyID, std::vector<ModuleDependencyID>,
                   std::set<ModuleDependencyID>> allModules;
   // Create the module dependency cache.
@@ -555,16 +646,23 @@ bool swift::scanClangDependencies(CompilerInstance &instance) {
                                               FEOpts.PrebuiltModuleCachePath,
                                               FEOpts.SerializeModuleInterfaceDependencyHashes,
                                               FEOpts.shouldTrackSystemDependencies());
-  // Loading the clang module using Clang importer.
-  // This action will populate the cache with the main module's dependencies.
-  auto rootDeps = static_cast<ClangImporter*>(ctx.getClangModuleLoader())
-    ->getModuleDependencies(mainModuleName, cache, ASTDelegate);
+  Optional<ModuleDependencies> rootDeps;
+  if (isClang) {
+    // Loading the clang module using Clang importer.
+    // This action will populate the cache with the main module's dependencies.
+    rootDeps = ctx.getModuleDependencies(moduleName, /*IsClang*/true, cache,
+                                         ASTDelegate);
+  } else {
+    // Loading the swift module's dependencies.
+    rootDeps = ctx.getSwiftModuleDependencies(moduleName, cache, ASTDelegate);
+  }
   if (!rootDeps.hasValue()) {
     // We cannot find the clang module, abort.
     return true;
   }
   // Add the main module.
-  allModules.insert({mainModuleName.str(), ModuleDependenciesKind::Clang});
+  allModules.insert({moduleName.str(), isClang ? ModuleDependenciesKind::Clang:
+    ModuleDependenciesKind::Swift});
 
   // Explore the dependencies of every module.
   for (unsigned currentModuleIdx = 0;
@@ -576,13 +674,40 @@ bool swift::scanClangDependencies(CompilerInstance &instance) {
     allModules.insert(discoveredModules.begin(), discoveredModules.end());
   }
   // Write out the JSON description.
-  std::string path = FEOpts.InputsAndOutputs.getSingleOutputFilename();
   std::error_code EC;
-  llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::F_None);
+  llvm::raw_fd_ostream out(outputPath, EC, llvm::sys::fs::F_None);
   writeJSON(out, instance, cache, ASTDelegate, allModules.getArrayRef());
   return false;
 }
 
+bool swift::scanClangDependencies(CompilerInstance &instance) {
+  return scanModuleDependencies(instance,
+                                instance.getMainModule()->getNameStr(),
+                                StringRef(),
+                                /*isClang*/true,
+                                instance.getInvocation().getFrontendOptions()
+                                  .InputsAndOutputs.getSingleOutputFilename());
+}
+
+bool swift::batchScanModuleDependencies(CompilerInstance &instance,
+                                      llvm::StringRef batchInputFile) {
+  (void)instance.getMainModule();
+  llvm::BumpPtrAllocator alloc;
+  llvm::StringSaver saver(alloc);
+  auto results = parseBatchScanInputFile(instance.getASTContext(),
+                                         batchInputFile, saver);
+  if (!results.hasValue())
+    return true;
+  for (auto &entry: *results) {
+    auto moduleName = llvm::sys::path::stem(entry.moduleName);
+    auto isClang = llvm::sys::path::extension(entry.moduleName) == ".pcm";
+    if (scanModuleDependencies(instance, moduleName, entry.arguments, isClang,
+                               entry.outputPath))
+      return true;
+  }
+  return false;
+}
+
 bool swift::scanDependencies(CompilerInstance &instance) {
   ASTContext &Context = instance.getASTContext();
   ModuleDecl *mainModule = instance.getMainModule();

lib/FrontendTool/ScanDependencies.h

@@ -13,10 +13,15 @@
 #ifndef SWIFT_FRONTENDTOOL_SCANDEPENDENCIES_H
 #define SWIFT_FRONTENDTOOL_SCANDEPENDENCIES_H
 
+#include "llvm/ADT/StringRef.h"
+
 namespace swift {
 
 class CompilerInstance;
 
+/// Batch scan the dependencies for modules specified in \c batchInputFile.
+bool batchScanModuleDependencies(CompilerInstance &instance, llvm::StringRef batchInputFile);
+
 /// Scans the dependencies of the main module of \c instance.
 bool scanDependencies(CompilerInstance &instance);
 

test/ScanDependencies/batch_module_scan.swift

@@ -0,0 +1,39 @@
+// RUN: %empty-directory(%t)
+// RUN: %empty-directory(%t/inputs)
+// RUN: %empty-directory(%t/outputs)
+// RUN: mkdir -p %t/clang-module-cache
+
+// RUN: echo "[{" > %/t/inputs/input.json
+// RUN: echo "\"module\": \"F.swiftmodule\"," >> %/t/inputs/input.json
+// RUN: echo "\"arguments\": \"-target x86_64-apple-macosx10.9\"," >> %/t/inputs/input.json
+// RUN: echo "\"output\": \"%/t/outputs/F.swiftmodule.json\"" >> %/t/inputs/input.json
+// RUN: echo "}," >> %/t/inputs/input.json
+// RUN: echo "{" >> %/t/inputs/input.json
+// RUN: echo "\"module\": \"F.pcm\"," >> %/t/inputs/input.json
+// RUN: echo "\"arguments\": \"-target x86_64-apple-macosx10.9\"," >> %/t/inputs/input.json
+// RUN: echo "\"output\": \"%/t/outputs/F.pcm.json\"" >> %/t/inputs/input.json
+// RUN: echo "}]" >> %/t/inputs/input.json
+
+// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/Inputs/CHeaders/Bridging.h -swift-version 4 -batch-scan-input-file %/t/inputs/input.json
+
+// Check the contents of the JSON output
+// RUN: %FileCheck %s -check-prefix=CHECK-PCM < %t/outputs/F.pcm.json
+// RUN: %FileCheck %s -check-prefix=CHECK-SWIFT < %t/outputs/F.swiftmodule.json
+
+// CHECK-PCM: 		{
+// CHECK-PCM-NEXT:  "mainModuleName": "F",
+// CHECK-PCM-NEXT:  "modules": [
+// CHECK-PCM-NEXT:    {
+// CHECK-PCM-NEXT:      "clang": "F"
+// CHECK-PCM-NEXT:    },
+// CHECK-PCM-NEXT:    {
+// CHECK-PCM-NEXT:      "modulePath": "F.pcm",
+
+// CHECK-SWIFT: {
+// CHECK-SWIFT-NEXT:  "mainModuleName": "F",
+// CHECK-SWIFT-NEXT:  "modules": [
+// CHECK-SWIFT-NEXT:    {
+// CHECK-SWIFT-NEXT:      "swift": "F"
+// CHECK-SWIFT-NEXT:    },
+// CHECK-SWIFT-NEXT:    {
+// CHECK-SWIFT-NEXT:      "modulePath": "F.swiftmodule",