[Dependency Scanning] Refactor Swift Scanner loader to be standalone

- 'SwiftModuleScanner' will now be owned directly by the 'ModuleDependencyScanningWorker' and will contain all the necessary custom logic, instead of being instantiated by the module interface loader for each query
- Moves ownership over module output path and sdk module output path directly into the scanning worker, instead of the cache

Author: artemcm

include/swift/AST/ModuleDependencies.h

@@ -1069,10 +1069,8 @@ class SwiftDependencyScanningService {
 
 // MARK: ModuleDependenciesCache
 /// This "local" dependencies cache persists only for the duration of a given
-/// scanning action, and wraps an instance of a `SwiftDependencyScanningService`
-/// which may carry cached scanning information from prior scanning actions.
-/// This cache maintains a store of references to all dependencies found within
-/// the current scanning action (with their values stored in the global Cache).
+/// scanning action, and maintains a store of references to all dependencies
+/// found within the current scanning action.
 class ModuleDependenciesCache {
 private:
   SwiftDependencyScanningService &globalScanningService;
@@ -1084,10 +1082,6 @@ class ModuleDependenciesCache {
   std::string mainScanModuleName;
   /// The context hash of the current scanning invocation
   std::string scannerContextHash;
-  /// The location of where the explicitly-built modules will be output to
-  std::string moduleOutputPath;
-  /// The location of where the explicitly-built SDK modules will be output to
-  std::string sdkModuleOutputPath;
   /// The timestamp of the beginning of the scanning query action
   /// using this cache
   const llvm::sys::TimePoint<> scanInitializationTime;
@@ -1102,8 +1096,6 @@ class ModuleDependenciesCache {
 public:
   ModuleDependenciesCache(SwiftDependencyScanningService &globalScanningService,
                           const std::string &mainScanModuleName,
-                          const std::string &moduleOutputPath,
-                          const std::string &sdkModuleOutputPath,
                           const std::string &scanningContextHash);
   ModuleDependenciesCache(const ModuleDependenciesCache &) = delete;
   ModuleDependenciesCache &operator=(const ModuleDependenciesCache &) = delete;
@@ -1137,8 +1129,6 @@ class ModuleDependenciesCache {
   void addSeenClangModule(clang::tooling::dependencies::ModuleID newModule) {
     alreadySeenClangModules.insert(newModule);
   }
-  StringRef getModuleOutputPath() const { return moduleOutputPath; }
-  StringRef getSDKModuleOutputPath() const { return sdkModuleOutputPath; }
 
   /// Query all dependencies
   ModuleDependencyIDSetVector

include/swift/AST/ModuleLoader.h

@@ -367,17 +367,6 @@ class ModuleLoader {
   /// Discover overlays declared alongside this file and add information about
   /// them to it.
   void findOverlayFiles(SourceLoc diagLoc, ModuleDecl *module, FileUnit *file);
-
-  /// Retrieve the dependencies for the given, named module, or \c None
-  /// if no such module exists.
-  virtual llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
-  getModuleDependencies(Identifier moduleName,
-                        StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
-                        const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
-                        const std::vector<std::string> &swiftModuleClangCC1CommandLineArgs,
-                        InterfaceSubContextDelegate &delegate,
-                        llvm::PrefixMapper *mapper = nullptr,
-                        bool isTestableImport = false) = 0;
 };
 
 } // namespace swift

include/swift/ClangImporter/ClangImporter.h

@@ -497,7 +497,7 @@ class ClangImporter final : public ClangModuleLoader {
   using LookupModuleOutputCallback =
       llvm::function_ref<std::string(const clang::tooling::dependencies::ModuleDeps &,
                                      clang::tooling::dependencies::ModuleOutputKind)>;
-  
+
   static llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
   bridgeClangModuleDependencies(
       const ASTContext &ctx,
@@ -507,14 +507,6 @@ class ClangImporter final : public ClangModuleLoader {
       LookupModuleOutputCallback LookupModuleOutput,
       RemapPathCallback remapPath = nullptr);
 
-  llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
-  getModuleDependencies(Identifier moduleName, StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
-                        const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
-                        const std::vector<std::string> &swiftModuleClangCC1CommandLineArgs,
-                        InterfaceSubContextDelegate &delegate,
-                        llvm::PrefixMapper *mapper,
-                        bool isTestableImport = false) override;
-
   static void getBridgingHeaderOptions(
       const ASTContext &ctx,
       const clang::tooling::dependencies::TranslationUnitDeps &deps,

include/swift/DependencyScan/ModuleDependencyScanner.h

@@ -15,7 +15,7 @@
 #include "swift/AST/Identifier.h"
 #include "swift/AST/ModuleDependencies.h"
 #include "swift/Frontend/ModuleInterfaceLoader.h"
-#include "swift/Serialization/SerializedModuleLoader.h"
+#include "swift/Serialization/ScanningLoaders.h"
 #include "llvm/CAS/CASReference.h"
 #include "llvm/Support/ThreadPool.h"
 
@@ -38,17 +38,14 @@ class ModuleDependencyScanningWorker {
 private:
   /// Retrieve the module dependencies for the Clang module with the given name.
   ModuleDependencyVector scanFilesystemForClangModuleDependency(
-      Identifier moduleName, StringRef moduleOutputPath,
-      StringRef sdkModuleOutputPath,
+      Identifier moduleName,
       const llvm::DenseSet<clang::tooling::dependencies::ModuleID>
           &alreadySeenModules,
       llvm::PrefixMapper *prefixMapper);
 
   /// Retrieve the module dependencies for the Swift module with the given name.
   ModuleDependencyVector scanFilesystemForSwiftModuleDependency(
-      Identifier moduleName, StringRef moduleOutputPath,
-      StringRef sdkModuleOutputPath, llvm::PrefixMapper *prefixMapper,
-      bool isTestableImport = false);
+      Identifier moduleName, bool isTestableImport = false);
 
   /// Query dependency information for header dependencies
   /// of a binary Swift module.
@@ -90,7 +87,7 @@ class ModuleDependencyScanningWorker {
   // The Clang scanner tool used by this worker.
   clang::tooling::dependencies::DependencyScanningTool clangScanningTool;
   // Swift and Clang module loaders acting as scanners.
-  std::unique_ptr<ModuleInterfaceLoader> swiftScannerModuleLoader;
+  std::unique_ptr<SwiftModuleScanner> swiftModuleScannerLoader;
 
   // Base command line invocation for clang scanner queries (both module and header)
   std::vector<std::string> clangScanningBaseCommandLineArgs;
@@ -102,6 +99,11 @@ class ModuleDependencyScanningWorker {
   // Working directory for clang module lookup queries
   std::string clangScanningWorkingDirectoryPath;
 
+  /// The location of where the explicitly-built modules will be output to
+  std::string moduleOutputPath;
+  /// The location of where the explicitly-built SDK modules will be output to
+  std::string sdkModuleOutputPath;
+
   // CAS instance.
   std::shared_ptr<llvm::cas::ObjectStore> CAS;
   std::shared_ptr<llvm::cas::ActionCache> ActionCache;

include/swift/DependencyScan/ScanDependencies.h

@@ -59,14 +59,14 @@ bool prescanDependencies(CompilerInstance &instance);
 /// Scans the dependencies of the main module of \c instance.
 llvm::ErrorOr<swiftscan_dependency_graph_t>
 performModuleScan(CompilerInstance &instance,
-                  DependencyScanDiagnosticCollector *diagnostics,
-                  ModuleDependenciesCache &cache);
+                  ModuleDependenciesCache &cache,
+                  DependencyScanDiagnosticCollector *diagnostics = nullptr);
 
 /// Scans the main module of \c instance for all direct module imports
 llvm::ErrorOr<swiftscan_import_set_t>
 performModulePrescan(CompilerInstance &instance,
-                     DependencyScanDiagnosticCollector *diagnostics,
-                     ModuleDependenciesCache &cache);
+                     ModuleDependenciesCache &cache,
+                     DependencyScanDiagnosticCollector *diagnostics = nullptr);
 
 namespace incremental {
 /// For the given module dependency graph captured in the 'cache',

include/swift/Sema/SourceLoader.h

@@ -97,14 +97,6 @@ class SourceLoader : public ModuleLoader {
   {
     // Parsing populates the Objective-C method tables.
   }
-
-  llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
-  getModuleDependencies(Identifier moduleName, StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
-                        const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
-                        const std::vector<std::string> &swiftModuleClangCC1CommandLineArgs,
-                        InterfaceSubContextDelegate &delegate,
-                        llvm::PrefixMapper *mapper,
-                        bool isTestableImport) override;
 };
 }
 

include/swift/Serialization/ScanningLoaders.h

@@ -22,23 +22,35 @@ namespace swift {
 /// for the purpose of determining dependencies, but does not attempt to
 /// load the module files.
 class SwiftModuleScanner : public SerializedModuleLoaderBase {
-public:
-  enum ScannerKind { MDS_plain, MDS_placeholder };
-
 private:
-  /// The kind of scanner this is (LLVM-style RTTI)
-  const ScannerKind kind;
-
-  /// The module we're scanning dependencies of.
-  Identifier moduleName;
-
   /// Scan the given interface file to determine dependencies.
   llvm::ErrorOr<ModuleDependencyInfo>
-  scanInterfaceFile(Twine moduleInterfacePath, bool isFramework,
-                    bool isTestableImport);
+  scanInterfaceFile(Identifier moduleID, Twine moduleInterfacePath,
+                    bool isFramework, bool isTestableImport);
 
-  InterfaceSubContextDelegate &astDelegate;
+  /// Scan the given serialized module file to determine dependencies.
+  llvm::ErrorOr<ModuleDependencyInfo>
+  scanBinaryModuleFile(Identifier moduleID, Twine binaryModulePath,
+                       bool isFramework, bool isTestableImport,
+                       bool isCandidateForTextualModule);
+
+  std::error_code findModuleFilesInDirectory(
+      ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
+      SmallVectorImpl<char> *ModuleInterfacePath,
+      SmallVectorImpl<char> *ModuleInterfaceSourcePath,
+      std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
+      std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
+      std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
+      bool SkipBuildingInterface, bool IsFramework,
+      bool IsTestableDependencyLookup) override;
+
+  virtual void collectVisibleTopLevelModuleNames(
+      SmallVectorImpl<Identifier> &names) const override {
+    llvm_unreachable("Not used");
+  }
 
+  /// AST delegate to be used for textual interface scanning
+  InterfaceSubContextDelegate &astDelegate;
   /// Location where pre-built modules are to be built into.
   std::string moduleOutputPath;
   /// Location where pre-built SDK modules are to be built into.
@@ -51,37 +63,19 @@ class SwiftModuleScanner : public SerializedModuleLoaderBase {
   std::optional<ModuleDependencyInfo> dependencies;
 
   SwiftModuleScanner(ASTContext &ctx, ModuleLoadingMode LoadMode,
-                     Identifier moduleName,
                      InterfaceSubContextDelegate &astDelegate,
                      StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
-                     std::vector<std::string> swiftModuleClangCC1CommandLineArgs,
-                     ScannerKind kind = MDS_plain)
+                     std::vector<std::string> swiftModuleClangCC1CommandLineArgs)
       : SerializedModuleLoaderBase(ctx, nullptr, LoadMode,
                                    /*IgnoreSwiftSourceInfoFile=*/true),
-        kind(kind), moduleName(moduleName), astDelegate(astDelegate),
+        astDelegate(astDelegate),
         moduleOutputPath(moduleOutputPath),
         sdkModuleOutputPath(sdkModuleOutputPath),
         swiftModuleClangCC1CommandLineArgs(swiftModuleClangCC1CommandLineArgs)  {}
 
-  std::error_code findModuleFilesInDirectory(
-      ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
-      SmallVectorImpl<char> *ModuleInterfacePath,
-      SmallVectorImpl<char> *ModuleInterfaceSourcePath,
-      std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
-      std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
-      std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
-      bool SkipBuildingInterface, bool IsFramework,
-      bool IsTestableDependencyLookup) override;
-
-  virtual void collectVisibleTopLevelModuleNames(
-      SmallVectorImpl<Identifier> &names) const override {
-    llvm_unreachable("Not used");
-  }
-
-  ScannerKind getKind() const { return kind; }
-  static bool classof(const SwiftModuleScanner *MDS) {
-    return MDS->getKind() == MDS_plain;
-  }
+  /// Perform a filesystem search for a Swift module with a given name
+  llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
+  lookupSwiftModule(Identifier moduleName, bool isTestableImport);
 };
 } // namespace swift
 

include/swift/Serialization/SerializedModuleLoader.h

@@ -164,11 +164,6 @@ class SerializedModuleLoaderBase : public ModuleLoader {
     return false;
   }
 
-  /// Scan the given serialized module file to determine dependencies.
-  llvm::ErrorOr<ModuleDependencyInfo>
-  scanModuleFile(Twine modulePath, bool isFramework,
-                 bool isTestableImport, bool isCandidateForTextualModule);
-
   struct BinaryModuleImports {
     std::vector<ScannerImportStatementInfo> moduleImports;
     std::string headerImport;
@@ -262,14 +257,6 @@ class SerializedModuleLoaderBase : public ModuleLoader {
 
   virtual void verifyAllModules() override;
 
-  virtual llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
-  getModuleDependencies(Identifier moduleName, StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
-                        const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
-                        const std::vector<std::string> &swiftModuleClangCC1CommandLineArgs,
-                        InterfaceSubContextDelegate &delegate,
-                        llvm::PrefixMapper *mapper,
-                        bool isTestableImport) override;
-
   /// A textual reason why the compiler rejected a binary module load
   /// attempt with a given status, to be used for diagnostic output.
   static std::optional<std::string> invalidModuleReason(serialization::Status status);

lib/AST/ModuleDependencies.cpp

@@ -782,13 +782,10 @@ bool SwiftDependencyScanningService::setupCachingDependencyScanningService(
 
 ModuleDependenciesCache::ModuleDependenciesCache(
     SwiftDependencyScanningService &globalScanningService,
-    const std::string &mainScanModuleName, const std::string &moduleOutputPath,
-    const std::string &sdkModuleOutputPath, const std::string &scannerContextHash)
+    const std::string &mainScanModuleName, const std::string &scannerContextHash)
     : globalScanningService(globalScanningService),
       mainScanModuleName(mainScanModuleName),
       scannerContextHash(scannerContextHash),
-      moduleOutputPath(moduleOutputPath),
-      sdkModuleOutputPath(sdkModuleOutputPath),
       scanInitializationTime(std::chrono::system_clock::now()) {
   for (auto kind = ModuleDependencyKind::FirstKind;
        kind != ModuleDependencyKind::LastKind; ++kind)

lib/ClangImporter/ClangModuleDependencyScanner.cpp

@@ -267,15 +267,3 @@ void ClangImporter::getBridgingHeaderOptions(
     swiftArgs.push_back(*Tree);
   }
 }
-
-ModuleDependencyVector
-ClangImporter::getModuleDependencies(Identifier moduleName,
-                                     StringRef moduleOutputPath,
-                                     StringRef sdkModuleOutputPath,
-                                     const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
-                                     const std::vector<std::string> &swiftModuleClangCC1CommandLineArgs,
-                                     InterfaceSubContextDelegate &delegate,
-                                     llvm::PrefixMapper *mapper,
-                                     bool isTestableImport) {
-  return {};
-}