Merge pull request #82429 from artemcm/ScannerInvalidBinaryModuleRefactor

[Dependency Scanning] Refactor the scanner to simplify layering

Author: artemcm

include/swift/AST/ModuleDependencies.h

@@ -992,7 +992,7 @@ class SwiftDependencyScanningService {
   std::shared_ptr<llvm::cas::ActionCache> ActionCache;
 
   /// File prefix mapper.
-  std::unique_ptr<llvm::PrefixMapper> Mapper;
+  std::shared_ptr<llvm::PrefixMapper> Mapper;
 
   /// The global file system cache.
   std::optional<
@@ -1069,13 +1069,10 @@ class SwiftDependencyScanningService {
 
 // MARK: ModuleDependenciesCache
 /// This "local" dependencies cache persists only for the duration of a given
-/// scanning action, and wraps an instance of a `SwiftDependencyScanningService`
-/// which may carry cached scanning information from prior scanning actions.
-/// This cache maintains a store of references to all dependencies found within
-/// the current scanning action (with their values stored in the global Cache).
+/// scanning action, and maintains a store of references to all dependencies
+/// found within the current scanning action.
 class ModuleDependenciesCache {
 private:
-  SwiftDependencyScanningService &globalScanningService;
   /// Discovered dependencies
   ModuleDependenciesKindMap ModuleDependenciesMap;
   /// Set containing all of the Clang modules that have already been seen.
@@ -1084,10 +1081,6 @@ class ModuleDependenciesCache {
   std::string mainScanModuleName;
   /// The context hash of the current scanning invocation
   std::string scannerContextHash;
-  /// The location of where the explicitly-built modules will be output to
-  std::string moduleOutputPath;
-  /// The location of where the explicitly-built SDK modules will be output to
-  std::string sdkModuleOutputPath;
   /// The timestamp of the beginning of the scanning query action
   /// using this cache
   const llvm::sys::TimePoint<> scanInitializationTime;
@@ -1100,10 +1093,7 @@ class ModuleDependenciesCache {
   getDependencyReferencesMap(ModuleDependencyKind kind) const;
 
 public:
-  ModuleDependenciesCache(SwiftDependencyScanningService &globalScanningService,
-                          const std::string &mainScanModuleName,
-                          const std::string &moduleOutputPath,
-                          const std::string &sdkModuleOutputPath,
+  ModuleDependenciesCache(const std::string &mainScanModuleName,
                           const std::string &scanningContextHash);
   ModuleDependenciesCache(const ModuleDependenciesCache &) = delete;
   ModuleDependenciesCache &operator=(const ModuleDependenciesCache &) = delete;
@@ -1124,21 +1114,13 @@ class ModuleDependenciesCache {
   /// Whether we have cached dependency information for the given Swift module.
   bool hasSwiftDependency(StringRef moduleName) const;
 
-  SwiftDependencyScanningService &getScanService() {
-    return globalScanningService;
-  }
-  const SwiftDependencyScanningService &getScanService() const {
-    return globalScanningService;
-  }
   const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &
   getAlreadySeenClangModules() const {
     return alreadySeenClangModules;
   }
   void addSeenClangModule(clang::tooling::dependencies::ModuleID newModule) {
     alreadySeenClangModules.insert(newModule);
   }
-  StringRef getModuleOutputPath() const { return moduleOutputPath; }
-  StringRef getSDKModuleOutputPath() const { return sdkModuleOutputPath; }
 
   /// Query all dependencies
   ModuleDependencyIDSetVector

include/swift/AST/ModuleLoader.h

@@ -367,17 +367,6 @@ class ModuleLoader {
   /// Discover overlays declared alongside this file and add information about
   /// them to it.
   void findOverlayFiles(SourceLoc diagLoc, ModuleDecl *module, FileUnit *file);
-
-  /// Retrieve the dependencies for the given, named module, or \c None
-  /// if no such module exists.
-  virtual llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
-  getModuleDependencies(Identifier moduleName,
-                        StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
-                        const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
-                        const std::vector<std::string> &swiftModuleClangCC1CommandLineArgs,
-                        InterfaceSubContextDelegate &delegate,
-                        llvm::PrefixMapper *mapper = nullptr,
-                        bool isTestableImport = false) = 0;
 };
 
 } // namespace swift

include/swift/ClangImporter/ClangImporter.h

@@ -497,24 +497,15 @@ class ClangImporter final : public ClangModuleLoader {
   using LookupModuleOutputCallback =
       llvm::function_ref<std::string(const clang::tooling::dependencies::ModuleDeps &,
                                      clang::tooling::dependencies::ModuleOutputKind)>;
-  
+
   static llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
   bridgeClangModuleDependencies(
       const ASTContext &ctx,
       clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
       clang::tooling::dependencies::ModuleDepsGraph &clangDependencies,
-      StringRef moduleOutputPath, StringRef stableModuleOutputPath,
       LookupModuleOutputCallback LookupModuleOutput,
       RemapPathCallback remapPath = nullptr);
 
-  llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
-  getModuleDependencies(Identifier moduleName, StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
-                        const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
-                        const std::vector<std::string> &swiftModuleClangCC1CommandLineArgs,
-                        InterfaceSubContextDelegate &delegate,
-                        llvm::PrefixMapper *mapper,
-                        bool isTestableImport = false) override;
-
   static void getBridgingHeaderOptions(
       const ASTContext &ctx,
       const clang::tooling::dependencies::TranslationUnitDeps &deps,

include/swift/DependencyScan/ModuleDependencyScanner.h

@@ -15,7 +15,7 @@
 #include "swift/AST/Identifier.h"
 #include "swift/AST/ModuleDependencies.h"
 #include "swift/Frontend/ModuleInterfaceLoader.h"
-#include "swift/Serialization/SerializedModuleLoader.h"
+#include "swift/Serialization/ScanningLoaders.h"
 #include "llvm/CAS/CASReference.h"
 #include "llvm/Support/ThreadPool.h"
 
@@ -38,17 +38,13 @@ class ModuleDependencyScanningWorker {
 private:
   /// Retrieve the module dependencies for the Clang module with the given name.
   ModuleDependencyVector scanFilesystemForClangModuleDependency(
-      Identifier moduleName, StringRef moduleOutputPath,
-      StringRef sdkModuleOutputPath,
+      Identifier moduleName,
       const llvm::DenseSet<clang::tooling::dependencies::ModuleID>
-          &alreadySeenModules,
-      llvm::PrefixMapper *prefixMapper);
+          &alreadySeenModules);
 
   /// Retrieve the module dependencies for the Swift module with the given name.
   ModuleDependencyVector scanFilesystemForSwiftModuleDependency(
-      Identifier moduleName, StringRef moduleOutputPath,
-      StringRef sdkModuleOutputPath, llvm::PrefixMapper *prefixMapper,
-      bool isTestableImport = false);
+      Identifier moduleName, bool isTestableImport = false);
 
   /// Query dependency information for header dependencies
   /// of a binary Swift module.
@@ -90,7 +86,18 @@ class ModuleDependencyScanningWorker {
   // The Clang scanner tool used by this worker.
   clang::tooling::dependencies::DependencyScanningTool clangScanningTool;
   // Swift and Clang module loaders acting as scanners.
-  std::unique_ptr<ModuleInterfaceLoader> swiftScannerModuleLoader;
+  std::unique_ptr<SwiftModuleScanner> swiftModuleScannerLoader;
+
+  /// The location of where the explicitly-built modules will be output to
+  std::string moduleOutputPath;
+  /// The location of where the explicitly-built SDK modules will be output to
+  std::string sdkModuleOutputPath;
+
+  // CAS instance.
+  std::shared_ptr<llvm::cas::ObjectStore> CAS;
+  std::shared_ptr<llvm::cas::ActionCache> ActionCache;
+  /// File prefix mapper.
+  std::shared_ptr<llvm::PrefixMapper> PrefixMapper;
 
   // Base command line invocation for clang scanner queries (both module and header)
   std::vector<std::string> clangScanningBaseCommandLineArgs;
@@ -101,10 +108,6 @@ class ModuleDependencyScanningWorker {
   std::vector<std::string> swiftModuleClangCC1CommandLineArgs;
   // Working directory for clang module lookup queries
   std::string clangScanningWorkingDirectoryPath;
-
-  // CAS instance.
-  std::shared_ptr<llvm::cas::ObjectStore> CAS;
-  std::shared_ptr<llvm::cas::ActionCache> ActionCache;
   // Restrict access to the parent scanner class.
   friend class ModuleDependencyScanner;
 };
@@ -219,6 +222,8 @@ class ModuleDependencyScanner {
   unsigned NumThreads;
   std::list<std::unique_ptr<ModuleDependencyScanningWorker>> Workers;
   llvm::DefaultThreadPool ScanningThreadPool;
+  /// File prefix mapper.
+  std::shared_ptr<llvm::PrefixMapper> PrefixMapper;
   /// Protect worker access.
   std::mutex WorkersLock;
   /// Count of filesystem queries performed

include/swift/DependencyScan/ScanDependencies.h

@@ -58,15 +58,17 @@ bool prescanDependencies(CompilerInstance &instance);
 // MARK: Dependency scanning execution
 /// Scans the dependencies of the main module of \c instance.
 llvm::ErrorOr<swiftscan_dependency_graph_t>
-performModuleScan(CompilerInstance &instance,
-                  DependencyScanDiagnosticCollector *diagnostics,
-                  ModuleDependenciesCache &cache);
+performModuleScan(SwiftDependencyScanningService &service,
+                  CompilerInstance &instance,
+                  ModuleDependenciesCache &cache,
+                  DependencyScanDiagnosticCollector *diagnostics = nullptr);
 
 /// Scans the main module of \c instance for all direct module imports
 llvm::ErrorOr<swiftscan_import_set_t>
-performModulePrescan(CompilerInstance &instance,
-                     DependencyScanDiagnosticCollector *diagnostics,
-                     ModuleDependenciesCache &cache);
+performModulePrescan(SwiftDependencyScanningService &service,
+                     CompilerInstance &instance,
+                     ModuleDependenciesCache &cache,
+                     DependencyScanDiagnosticCollector *diagnostics = nullptr);
 
 namespace incremental {
 /// For the given module dependency graph captured in the 'cache',
@@ -75,6 +77,7 @@ namespace incremental {
 /// than the serialized dependency graph, it is considered invalidated and must
 /// be re-scanned.
 void validateInterModuleDependenciesCache(
+    const SwiftDependencyScanningService &service,
     const ModuleDependencyID &rootModuleID, ModuleDependenciesCache &cache,
     const llvm::sys::TimePoint<> &cacheTimeStamp, llvm::vfs::FileSystem &fs,
     DiagnosticEngine &diags, bool emitRemarks = false);
@@ -83,7 +86,8 @@ void validateInterModuleDependenciesCache(
 /// with respect to their inputs. Upon encountering such a module, add it to the
 /// set of invalidated modules, along with the path from the root to this
 /// module.
-void outOfDateModuleScan(const ModuleDependencyID &sourceModuleID,
+void outOfDateModuleScan(const SwiftDependencyScanningService &service,
+                         const ModuleDependencyID &sourceModuleID,
                          const ModuleDependenciesCache &cache,
                          const llvm::sys::TimePoint<> &cacheTimeStamp,
                          llvm::vfs::FileSystem &fs, DiagnosticEngine &diags,
@@ -93,6 +97,7 @@ void outOfDateModuleScan(const ModuleDependencyID &sourceModuleID,
 /// Validate whether all inputs of a given module dependency
 /// are older than the cache serialization time.
 bool verifyModuleDependencyUpToDate(
+    const SwiftDependencyScanningService &service,
     const ModuleDependencyID &moduleID, const ModuleDependenciesCache &cache,
     const llvm::sys::TimePoint<> &cacheTimeStamp, llvm::vfs::FileSystem &fs,
     DiagnosticEngine &diags, bool emitRemarks);

include/swift/Sema/SourceLoader.h

@@ -97,14 +97,6 @@ class SourceLoader : public ModuleLoader {
   {
     // Parsing populates the Objective-C method tables.
   }
-
-  llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
-  getModuleDependencies(Identifier moduleName, StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
-                        const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
-                        const std::vector<std::string> &swiftModuleClangCC1CommandLineArgs,
-                        InterfaceSubContextDelegate &delegate,
-                        llvm::PrefixMapper *mapper,
-                        bool isTestableImport) override;
 };
 }
 

include/swift/Serialization/ScanningLoaders.h

@@ -22,23 +22,35 @@ namespace swift {
 /// for the purpose of determining dependencies, but does not attempt to
 /// load the module files.
 class SwiftModuleScanner : public SerializedModuleLoaderBase {
-public:
-  enum ScannerKind { MDS_plain, MDS_placeholder };
-
 private:
-  /// The kind of scanner this is (LLVM-style RTTI)
-  const ScannerKind kind;
-
-  /// The module we're scanning dependencies of.
-  Identifier moduleName;
-
   /// Scan the given interface file to determine dependencies.
   llvm::ErrorOr<ModuleDependencyInfo>
-  scanInterfaceFile(Twine moduleInterfacePath, bool isFramework,
-                    bool isTestableImport);
+  scanInterfaceFile(Identifier moduleID, Twine moduleInterfacePath,
+                    bool isFramework, bool isTestableImport);
 
-  InterfaceSubContextDelegate &astDelegate;
+  /// Scan the given serialized module file to determine dependencies.
+  llvm::ErrorOr<ModuleDependencyInfo>
+  scanBinaryModuleFile(Identifier moduleID, Twine binaryModulePath,
+                       bool isFramework, bool isTestableImport,
+                       bool isCandidateForTextualModule);
+
+  std::error_code findModuleFilesInDirectory(
+      ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
+      SmallVectorImpl<char> *ModuleInterfacePath,
+      SmallVectorImpl<char> *ModuleInterfaceSourcePath,
+      std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
+      std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
+      std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
+      bool SkipBuildingInterface, bool IsFramework,
+      bool IsTestableDependencyLookup) override;
+
+  virtual void collectVisibleTopLevelModuleNames(
+      SmallVectorImpl<Identifier> &names) const override {
+    llvm_unreachable("Not used");
+  }
 
+  /// AST delegate to be used for textual interface scanning
+  InterfaceSubContextDelegate &astDelegate;
   /// Location where pre-built modules are to be built into.
   std::string moduleOutputPath;
   /// Location where pre-built SDK modules are to be built into.
@@ -51,37 +63,19 @@ class SwiftModuleScanner : public SerializedModuleLoaderBase {
   std::optional<ModuleDependencyInfo> dependencies;
 
   SwiftModuleScanner(ASTContext &ctx, ModuleLoadingMode LoadMode,
-                     Identifier moduleName,
                      InterfaceSubContextDelegate &astDelegate,
                      StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
-                     std::vector<std::string> swiftModuleClangCC1CommandLineArgs,
-                     ScannerKind kind = MDS_plain)
+                     std::vector<std::string> swiftModuleClangCC1CommandLineArgs)
       : SerializedModuleLoaderBase(ctx, nullptr, LoadMode,
                                    /*IgnoreSwiftSourceInfoFile=*/true),
-        kind(kind), moduleName(moduleName), astDelegate(astDelegate),
+        astDelegate(astDelegate),
         moduleOutputPath(moduleOutputPath),
         sdkModuleOutputPath(sdkModuleOutputPath),
         swiftModuleClangCC1CommandLineArgs(swiftModuleClangCC1CommandLineArgs)  {}
 
-  std::error_code findModuleFilesInDirectory(
-      ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
-      SmallVectorImpl<char> *ModuleInterfacePath,
-      SmallVectorImpl<char> *ModuleInterfaceSourcePath,
-      std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
-      std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
-      std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
-      bool SkipBuildingInterface, bool IsFramework,
-      bool IsTestableDependencyLookup) override;
-
-  virtual void collectVisibleTopLevelModuleNames(
-      SmallVectorImpl<Identifier> &names) const override {
-    llvm_unreachable("Not used");
-  }
-
-  ScannerKind getKind() const { return kind; }
-  static bool classof(const SwiftModuleScanner *MDS) {
-    return MDS->getKind() == MDS_plain;
-  }
+  /// Perform a filesystem search for a Swift module with a given name
+  llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
+  lookupSwiftModule(Identifier moduleName, bool isTestableImport);
 };
 } // namespace swift
 

include/swift/Serialization/SerializedModuleLoader.h

@@ -164,11 +164,6 @@ class SerializedModuleLoaderBase : public ModuleLoader {
     return false;
   }
 
-  /// Scan the given serialized module file to determine dependencies.
-  llvm::ErrorOr<ModuleDependencyInfo>
-  scanModuleFile(Twine modulePath, bool isFramework,
-                 bool isTestableImport, bool isCandidateForTextualModule);
-
   struct BinaryModuleImports {
     std::vector<ScannerImportStatementInfo> moduleImports;
     std::string headerImport;
@@ -262,14 +257,6 @@ class SerializedModuleLoaderBase : public ModuleLoader {
 
   virtual void verifyAllModules() override;
 
-  virtual llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
-  getModuleDependencies(Identifier moduleName, StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
-                        const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
-                        const std::vector<std::string> &swiftModuleClangCC1CommandLineArgs,
-                        InterfaceSubContextDelegate &delegate,
-                        llvm::PrefixMapper *mapper,
-                        bool isTestableImport) override;
-
   /// A textual reason why the compiler rejected a binary module load
   /// attempt with a given status, to be used for diagnostic output.
   static std::optional<std::string> invalidModuleReason(serialization::Status status);

lib/AST/ModuleDependencies.cpp

@@ -782,14 +782,9 @@ bool SwiftDependencyScanningService::setupCachingDependencyScanningService(
 }
 
 ModuleDependenciesCache::ModuleDependenciesCache(
-    SwiftDependencyScanningService &globalScanningService,
-    const std::string &mainScanModuleName, const std::string &moduleOutputPath,
-    const std::string &sdkModuleOutputPath, const std::string &scannerContextHash)
-    : globalScanningService(globalScanningService),
-      mainScanModuleName(mainScanModuleName),
+    const std::string &mainScanModuleName, const std::string &scannerContextHash)
+    : mainScanModuleName(mainScanModuleName),
       scannerContextHash(scannerContextHash),
-      moduleOutputPath(moduleOutputPath),
-      sdkModuleOutputPath(sdkModuleOutputPath),
       scanInitializationTime(std::chrono::system_clock::now()) {
   for (auto kind = ModuleDependencyKind::FirstKind;
        kind != ModuleDependencyKind::LastKind; ++kind)