Merge pull request swiftlang#81361 from artemcm/DepScanFewerClangs

[Dependency Scanning][Refactor] Reduce the number of `ClangImporter` instances used by the scanner

Author: artemcm

include/swift/AST/ModuleLoader.h

@@ -374,7 +374,7 @@ class ModuleLoader {
   getModuleDependencies(Identifier moduleName,
                         StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
                         const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
-                        clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
+                        const std::vector<std::string> &swiftModuleClangCC1CommandLineArgs,
                         InterfaceSubContextDelegate &delegate,
                         llvm::PrefixMapper *mapper = nullptr,
                         bool isTestableImport = false) = 0;

include/swift/ClangImporter/ClangImporter.h

@@ -65,6 +65,7 @@ namespace tooling {
 namespace dependencies {
   struct ModuleDeps;
   struct TranslationUnitDeps;
+  enum class ModuleOutputKind;
   using ModuleDepsGraph = std::vector<ModuleDeps>;
 }
 }
@@ -211,8 +212,7 @@ class ClangImporter final : public ClangModuleLoader {
                        bool ignoreClangTarget = false);
 
   std::vector<std::string>
-  getClangDepScanningInvocationArguments(ASTContext &ctx,
-      std::optional<StringRef> sourceFileName = std::nullopt);
+  getClangDepScanningInvocationArguments(ASTContext &ctx);
 
   static std::unique_ptr<clang::CompilerInvocation>
   createClangInvocation(ClangImporter *importer,
@@ -499,54 +499,32 @@ class ClangImporter final : public ClangModuleLoader {
   void verifyAllModules() override;
 
   using RemapPathCallback = llvm::function_ref<std::string(StringRef)>;
-  llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
+  using LookupModuleOutputCallback =
+      llvm::function_ref<std::string(const clang::tooling::dependencies::ModuleDeps &,
+                                     clang::tooling::dependencies::ModuleOutputKind)>;
+  
+  static llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
   bridgeClangModuleDependencies(
+      const ASTContext &ctx,
       clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
       clang::tooling::dependencies::ModuleDepsGraph &clangDependencies,
       StringRef moduleOutputPath, StringRef stableModuleOutputPath,
+      LookupModuleOutputCallback LookupModuleOutput,
       RemapPathCallback remapPath = nullptr);
 
   llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
   getModuleDependencies(Identifier moduleName, StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
                         const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
-                        clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
+                        const std::vector<std::string> &swiftModuleClangCC1CommandLineArgs,
                         InterfaceSubContextDelegate &delegate,
                         llvm::PrefixMapper *mapper,
                         bool isTestableImport = false) override;
 
-  void recordBridgingHeaderOptions(
-      ModuleDependencyInfo &MDI,
-      const clang::tooling::dependencies::TranslationUnitDeps &deps);
-
-  void getBridgingHeaderOptions(
+  static void getBridgingHeaderOptions(
+      const ASTContext &ctx,
       const clang::tooling::dependencies::TranslationUnitDeps &deps,
       std::vector<std::string> &swiftArgs);
 
-  /// Query dependency information for header dependencies
-  /// of a binary Swift module.
-  ///
-  /// \param moduleID the name of the Swift module whose dependency
-  /// information will be augmented with information about the given
-  /// textual header inputs.
-  ///
-  /// \param headerPath the path to the header to be scanned.
-  ///
-  /// \param clangScanningTool The clang dependency scanner.
-  ///
-  /// \param cache The module dependencies cache to update, with information
-  /// about new Clang modules discovered along the way.
-  ///
-  /// \returns \c true if an error occurred, \c false otherwise
-  bool getHeaderDependencies(
-      ModuleDependencyID moduleID, std::optional<StringRef> headerPath,
-      std::optional<llvm::MemoryBufferRef> sourceBuffer,
-      clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
-      ModuleDependenciesCache &cache,
-      ModuleDependencyIDSetVector &headerClangModuleDependencies,
-      std::vector<std::string> &headerFileInputs,
-      std::vector<std::string> &bridgingHeaderCommandLine,
-      std::optional<std::string> &includeTreeID);
-
   clang::TargetInfo &getModuleAvailabilityTarget() const override;
   clang::ASTContext &getClangASTContext() const override;
   clang::Preprocessor &getClangPreprocessor() const override;

include/swift/DependencyScan/ModuleDependencyScanner.h

@@ -50,6 +50,32 @@ class ModuleDependencyScanningWorker {
       StringRef sdkModuleOutputPath, llvm::PrefixMapper *prefixMapper,
       bool isTestableImport = false);
 
+  /// Query dependency information for header dependencies
+  /// of a binary Swift module.
+  ///
+  /// \param moduleID the name of the Swift module whose dependency
+  /// information will be augmented with information about the given
+  /// textual header inputs.
+  ///
+  /// \param headerPath the path to the header to be scanned.
+  ///
+  /// \param clangScanningTool The clang dependency scanner.
+  ///
+  /// \param cache The module dependencies cache to update, with information
+  /// about new Clang modules discovered along the way.
+  ///
+  /// \returns \c true if an error occurred, \c false otherwise
+  bool scanHeaderDependenciesOfSwiftModule(
+      const ASTContext &ctx,
+      ModuleDependencyID moduleID, std::optional<StringRef> headerPath,
+      std::optional<llvm::MemoryBufferRef> sourceBuffer,
+      ModuleDependenciesCache &cache,
+      ModuleDependencyIDSetVector &headerClangModuleDependencies,
+      std::vector<std::string> &headerFileInputs,
+      std::vector<std::string> &bridgingHeaderCommandLine,
+      std::optional<std::string> &includeTreeID);
+
+
   /// Store cache entry for include tree.
   llvm::Error
   createCacheKeyForEmbeddedHeader(std::string embeddedHeaderIncludeTree,
@@ -65,7 +91,17 @@ class ModuleDependencyScanningWorker {
   clang::tooling::dependencies::DependencyScanningTool clangScanningTool;
   // Swift and Clang module loaders acting as scanners.
   std::unique_ptr<ModuleInterfaceLoader> swiftScannerModuleLoader;
-  std::unique_ptr<ClangImporter> clangScannerModuleLoader;
+
+  // Base command line invocation for clang scanner queries (both module and header)
+  std::vector<std::string> clangScanningBaseCommandLineArgs;
+  // Command line invocation for clang by-name module lookups
+  std::vector<std::string> clangScanningModuleCommandLineArgs;
+  // Clang-specific (-Xcc) command-line flags to include on
+  // Swift module compilation commands
+  std::vector<std::string> swiftModuleClangCC1CommandLineArgs;
+  // Working directory for clang module lookup queries
+  std::string clangScanningWorkingDirectoryPath;
+
   // CAS instance.
   std::shared_ptr<llvm::cas::ObjectStore> CAS;
   std::shared_ptr<llvm::cas::ActionCache> ActionCache;

include/swift/Sema/SourceLoader.h

@@ -101,7 +101,7 @@ class SourceLoader : public ModuleLoader {
   llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
   getModuleDependencies(Identifier moduleName, StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
                         const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
-                        clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
+                        const std::vector<std::string> &swiftModuleClangCC1CommandLineArgs,
                         InterfaceSubContextDelegate &delegate,
                         llvm::PrefixMapper *mapper,
                         bool isTestableImport) override;

include/swift/Serialization/ScanningLoaders.h

@@ -43,6 +43,9 @@ class SwiftModuleScanner : public SerializedModuleLoaderBase {
   std::string moduleOutputPath;
   /// Location where pre-built SDK modules are to be built into.
   std::string sdkModuleOutputPath;
+  /// Clang-specific (-Xcc) command-line flags to include on
+  /// Swift module compilation commands
+  std::vector<std::string> swiftModuleClangCC1CommandLineArgs;
 
 public:
   std::optional<ModuleDependencyInfo> dependencies;
@@ -51,12 +54,14 @@ class SwiftModuleScanner : public SerializedModuleLoaderBase {
                      Identifier moduleName,
                      InterfaceSubContextDelegate &astDelegate,
                      StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
+                     std::vector<std::string> swiftModuleClangCC1CommandLineArgs,
                      ScannerKind kind = MDS_plain)
       : SerializedModuleLoaderBase(ctx, nullptr, LoadMode,
                                    /*IgnoreSwiftSourceInfoFile=*/true),
         kind(kind), moduleName(moduleName), astDelegate(astDelegate),
         moduleOutputPath(moduleOutputPath),
-        sdkModuleOutputPath(sdkModuleOutputPath) {}
+        sdkModuleOutputPath(sdkModuleOutputPath),
+        swiftModuleClangCC1CommandLineArgs(swiftModuleClangCC1CommandLineArgs)  {}
 
   std::error_code findModuleFilesInDirectory(
       ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
@@ -100,9 +105,8 @@ class PlaceholderSwiftModuleScanner : public SwiftModuleScanner {
                                 StringRef moduleOutputPath,
                                 StringRef sdkModuleOutputPath)
       : SwiftModuleScanner(ctx, LoadMode, moduleName, astDelegate,
-                           moduleOutputPath, sdkModuleOutputPath,
+                           moduleOutputPath, sdkModuleOutputPath, {},
                            MDS_placeholder) {
-
     // FIXME: Find a better place for this map to live, to avoid
     // doing the parsing on every module.
     if (!PlaceholderDependencyModuleMap.empty()) {

include/swift/Serialization/SerializedModuleLoader.h

@@ -266,7 +266,7 @@ class SerializedModuleLoaderBase : public ModuleLoader {
   virtual llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
   getModuleDependencies(Identifier moduleName, StringRef moduleOutputPath, StringRef sdkModuleOutputPath,
                         const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
-                        clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
+                        const std::vector<std::string> &swiftModuleClangCC1CommandLineArgs,
                         InterfaceSubContextDelegate &delegate,
                         llvm::PrefixMapper *mapper,
                         bool isTestableImport) override;