Merge pull request #6194 from apple/jan_svoboda/20221013-cherry-pick

🍒 Dependency scanning patches

Author: jansvoboda11

clang/include/clang-c/Dependencies.h

@@ -74,11 +74,6 @@ typedef struct {
 
   /**
    * The canonical command line to build this module.
-   *
-   * If getFileDependencies_v3 or later was used to get this dependency, it is
-   * a complete command line. When using getFileDependencies_v2, it excludes
-   * arguments containing modules-related paths:
-   * "-fmodule-file=", "-o", "-fmodule-map-file=".
    */
   CXStringSet *BuildArguments;
 } CXModuleDependency;
@@ -300,16 +295,6 @@ typedef size_t CXModuleLookupOutputCallback(void *Context,
                                             CXOutputKind OutputKind,
                                             char *Output, size_t MaxLen);
 
-/**
- * See \c clang_experimental_DependencyScannerWorker_getFileDependencies_v5.
- */
-CINDEX_LINKAGE CXFileDependencies *
-clang_experimental_DependencyScannerWorker_getFileDependencies_v3(
-    CXDependencyScannerWorker Worker, int argc, const char *const *argv,
-    const char *ModuleName, const char *WorkingDirectory, void *MDCContext,
-    CXModuleDiscoveredCallback *MDC, void *MLOContext,
-    CXModuleLookupOutputCallback *MLO, unsigned Options, CXString *error);
-
 /**
  * See \c clang_experimental_DependencyScannerWorker_getFileDependencies_v5.
  * Returns diagnostics in an unstructured CXString instead of CXDiagnosticSet.

clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

@@ -40,8 +40,14 @@ namespace dependencies {
 using LookupModuleOutputCallback =
     llvm::function_ref<std::string(const ModuleID &, ModuleOutputKind)>;
 
+/// Graph of modular dependencies.
+using ModuleDepsGraph = std::vector<ModuleDeps>;
+
 /// The full dependencies and module graph for a specific input.
-struct FullDependencies {
+struct TranslationUnitDeps {
+  /// The graph of direct and transitive modular dependencies.
+  ModuleDepsGraph ModuleGraph;
+
   /// The identifier of the C++20 module this translation unit exports.
   ///
   /// If the translation unit is not a module then \c ID.ModuleName is empty.
@@ -77,11 +83,6 @@ struct FullDependencies {
   std::vector<std::string> DriverCommandLine;
 };
 
-struct FullDependenciesResult {
-  FullDependencies FullDeps;
-  std::vector<ModuleDeps> DiscoveredModules;
-};
-
 /// The high-level implementation of the dependency discovery tool that runs on
 /// an individual worker thread.
 class DependencyScanningTool {
@@ -93,14 +94,12 @@ class DependencyScanningTool {
 
   /// Print out the dependency information into a string using the dependency
   /// file format that is specified in the options (-MD is the default) and
-  /// return it. If \p ModuleName isn't empty, this function returns the
-  /// dependency information of module \p ModuleName.
+  /// return it.
   ///
   /// \returns A \c StringError with the diagnostic output if clang errors
   /// occurred, dependency file contents otherwise.
   llvm::Expected<std::string>
-  getDependencyFile(const std::vector<std::string> &CommandLine, StringRef CWD,
-                    llvm::Optional<StringRef> ModuleName = None);
+  getDependencyFile(const std::vector<std::string> &CommandLine, StringRef CWD);
 
   /// Collect dependency tree.
   llvm::Expected<llvm::cas::ObjectProxy>
@@ -133,9 +132,8 @@ class DependencyScanningTool {
       DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS,
       bool DiagGenerationAsCompilation);
 
-  /// Collect the full module dependency graph for the input, ignoring any
-  /// modules which have already been seen. If \p ModuleName isn't empty, this
-  /// function returns the full dependency information of module \p ModuleName.
+  /// Given a Clang driver command-line for a translation unit, gather the
+  /// modular dependencies and return the information needed for explicit build.
   ///
   /// \param AlreadySeen This stores modules which have previously been
   ///                    reported. Use the same instance for all calls to this
@@ -147,13 +145,23 @@ class DependencyScanningTool {
   ///                           arguments for dependencies.
   ///
   /// \returns a \c StringError with the diagnostic output if clang errors
-  /// occurred, \c FullDependencies otherwise.
-  llvm::Expected<FullDependenciesResult>
-  getFullDependencies(const std::vector<std::string> &CommandLine,
-                      StringRef CWD, const llvm::StringSet<> &AlreadySeen,
-                      LookupModuleOutputCallback LookupModuleOutput,
-                      llvm::Optional<StringRef> ModuleName = None,
-                      DepscanPrefixMapping PrefixMapping = {});
+  /// occurred, \c TranslationUnitDeps otherwise.
+  llvm::Expected<TranslationUnitDeps>
+  getTranslationUnitDependencies(const std::vector<std::string> &CommandLine,
+                                 StringRef CWD,
+                                 const llvm::StringSet<> &AlreadySeen,
+                                 LookupModuleOutputCallback LookupModuleOutput,
+                                 DepscanPrefixMapping PrefixMapping);
+
+  /// Given a compilation context specified via the Clang driver command-line,
+  /// gather modular dependencies of module with the given name, and return the
+  /// information needed for explicit build.
+  llvm::Expected<ModuleDepsGraph>
+  getModuleDependencies(StringRef ModuleName,
+                        const std::vector<std::string> &CommandLine,
+                        StringRef CWD, const llvm::StringSet<> &AlreadySeen,
+                        LookupModuleOutputCallback LookupModuleOutput,
+                        DepscanPrefixMapping PrefixMapping);
 
   ScanningOutputFormat getScanningFormat() const {
     return Worker.getScanningFormat();
@@ -172,12 +180,6 @@ class DependencyScanningTool {
     return Worker.getOrCreateFileManager();
   }
 
-  llvm::Expected<FullDependenciesResult> getFullDependenciesLegacyDriverCommand(
-      const std::vector<std::string> &CommandLine, StringRef CWD,
-      const llvm::StringSet<> &AlreadySeen,
-      LookupModuleOutputCallback LookupModuleOutput,
-      llvm::Optional<StringRef> ModuleName = None);
-
 private:
   DependencyScanningWorker Worker;
 };
@@ -186,12 +188,10 @@ class FullDependencyConsumer : public DependencyConsumer {
 public:
   FullDependencyConsumer(const llvm::StringSet<> &AlreadySeen,
                          LookupModuleOutputCallback LookupModuleOutput,
-                         bool EagerLoadModules,
                          CachingOnDiskFileSystemPtr CacheFS = nullptr,
                          DepscanPrefixMapping PrefixMapping = {})
       : CacheFS(std::move(CacheFS)), PrefixMapping(std::move(PrefixMapping)),
-        AlreadySeen(AlreadySeen), LookupModuleOutput(LookupModuleOutput),
-        EagerLoadModules(EagerLoadModules) {}
+        AlreadySeen(AlreadySeen), LookupModuleOutput(LookupModuleOutput) {}
 
   llvm::Error initialize(CompilerInstance &ScanInstance,
                          CompilerInvocation &NewInvocation) override;
@@ -239,10 +239,8 @@ class FullDependencyConsumer : public DependencyConsumer {
     return LookupModuleOutput(ID, Kind);
   }
 
-  FullDependenciesResult getFullDependenciesLegacyDriverCommand(
-      const std::vector<std::string> &OriginalCommandLine) const;
-
-  FullDependenciesResult takeFullDependencies();
+  TranslationUnitDeps takeTranslationUnitDeps();
+  ModuleDepsGraph takeModuleGraphDeps();
 
 private:
   std::vector<std::string> Dependencies;
@@ -259,7 +257,6 @@ class FullDependencyConsumer : public DependencyConsumer {
   std::vector<std::string> OutputPaths;
   const llvm::StringSet<> &AlreadySeen;
   LookupModuleOutputCallback LookupModuleOutput;
-  bool EagerLoadModules;
 };
 
 } // end namespace dependencies

clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h

@@ -35,7 +35,7 @@ struct DepscanPrefixMapping;
 
 /// A command-line tool invocation that is part of building a TU.
 ///
-/// \see FullDependencies::Commands.
+/// \see TranslationUnitDeps::Commands.
 struct Command {
   std::string Executable;
   std::vector<std::string> Arguments;

clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

@@ -22,35 +22,13 @@ using namespace tooling;
 using namespace dependencies;
 using llvm::Error;
 
-static std::vector<std::string>
-makeTUCommandLineWithoutPaths(ArrayRef<std::string> OriginalCommandLine) {
-  std::vector<std::string> Args = OriginalCommandLine;
-
-  Args.push_back("-fno-implicit-modules");
-  Args.push_back("-fno-implicit-module-maps");
-
-  // These arguments are unused in explicit compiles.
-  llvm::erase_if(Args, [](StringRef Arg) {
-    if (Arg.consume_front("-fmodules-")) {
-      return Arg.startswith("cache-path=") ||
-             Arg.startswith("prune-interval=") ||
-             Arg.startswith("prune-after=") ||
-             Arg == "validate-once-per-build-session";
-    }
-    return Arg.startswith("-fbuild-session-file=");
-  });
-
-  return Args;
-}
-
 DependencyScanningTool::DependencyScanningTool(
     DependencyScanningService &Service,
     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
     : Worker(Service, std::move(FS)) {}
 
 llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(
-    const std::vector<std::string> &CommandLine, StringRef CWD,
-    llvm::Optional<StringRef> ModuleName) {
+    const std::vector<std::string> &CommandLine, StringRef CWD) {
   /// Prints out all of the gathered dependencies into a string.
   class MakeDependencyPrinterConsumer : public DependencyConsumer {
   public:
@@ -111,8 +89,7 @@ llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(
   };
 
   MakeDependencyPrinterConsumer Consumer;
-  auto Result =
-      Worker.computeDependencies(CWD, CommandLine, Consumer, ModuleName);
+  auto Result = Worker.computeDependencies(CWD, CommandLine, Consumer);
   if (Result)
     return std::move(Result);
   std::string Output;
@@ -144,8 +121,8 @@ class GetDependencyTree : public FullDependencyConsumer {
 
   GetDependencyTree(llvm::cas::CachingOnDiskFileSystem &FS,
                     DepscanPrefixMapping PrefixMapping)
-      : FullDependencyConsumer(AlreadySeen, nullptr, /*EagerModules=*/false,
-                               &FS, std::move(PrefixMapping)),
+      : FullDependencyConsumer(AlreadySeen, nullptr, &FS,
+                               std::move(PrefixMapping)),
         FS(FS) {}
 
 private:
@@ -560,36 +537,32 @@ DependencyScanningTool::getIncludeTreeFromCompilerInvocation(
   return Consumer.getIncludeTree();
 }
 
-llvm::Expected<FullDependenciesResult>
-DependencyScanningTool::getFullDependencies(
+llvm::Expected<TranslationUnitDeps>
+DependencyScanningTool::getTranslationUnitDependencies(
     const std::vector<std::string> &CommandLine, StringRef CWD,
     const llvm::StringSet<> &AlreadySeen,
     LookupModuleOutputCallback LookupModuleOutput,
-    llvm::Optional<StringRef> ModuleName, DepscanPrefixMapping PrefixMapping) {
+    DepscanPrefixMapping PrefixMapping) {
   FullDependencyConsumer Consumer(AlreadySeen, LookupModuleOutput,
-                                  Worker.shouldEagerLoadModules(),
                                   Worker.getCASFS(), std::move(PrefixMapping));
-  llvm::Error Result =
-      Worker.computeDependencies(CWD, CommandLine, Consumer, ModuleName);
+  llvm::Error Result = Worker.computeDependencies(CWD, CommandLine, Consumer);
   if (Result)
     return std::move(Result);
-  return Consumer.takeFullDependencies();
+  return Consumer.takeTranslationUnitDeps();
 }
 
-llvm::Expected<FullDependenciesResult>
-DependencyScanningTool::getFullDependenciesLegacyDriverCommand(
-    const std::vector<std::string> &CommandLine, StringRef CWD,
-    const llvm::StringSet<> &AlreadySeen,
+llvm::Expected<ModuleDepsGraph> DependencyScanningTool::getModuleDependencies(
+    StringRef ModuleName, const std::vector<std::string> &CommandLine,
+    StringRef CWD, const llvm::StringSet<> &AlreadySeen,
     LookupModuleOutputCallback LookupModuleOutput,
-    llvm::Optional<StringRef> ModuleName) {
+    DepscanPrefixMapping PrefixMapping) {
   FullDependencyConsumer Consumer(AlreadySeen, LookupModuleOutput,
-                                  Worker.shouldEagerLoadModules(),
-                                  Worker.getCASFS());
+                                  Worker.getCASFS(), std::move(PrefixMapping));
   llvm::Error Result =
       Worker.computeDependencies(CWD, CommandLine, Consumer, ModuleName);
   if (Result)
     return std::move(Result);
-  return Consumer.getFullDependenciesLegacyDriverCommand(CommandLine);
+  return Consumer.takeModuleGraphDeps();
 }
 
 Error FullDependencyConsumer::initialize(CompilerInstance &ScanInstance,
@@ -733,75 +706,40 @@ Error FullDependencyConsumer::finalizeModuleInvocation(CompilerInvocation &CI,
   return llvm::Error::success();
 }
 
-FullDependenciesResult FullDependencyConsumer::takeFullDependencies() {
-  FullDependenciesResult FDR;
-  FullDependencies &FD = FDR.FullDeps;
+TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() {
+  TranslationUnitDeps TU;
 
-  FD.ID.ContextHash = std::move(ContextHash);
-  FD.FileDeps = std::move(Dependencies);
-  FD.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps);
-  FD.Commands = std::move(Commands);
-  FD.CASFileSystemRootID = CASFileSystemRootID;
+  TU.ID.ContextHash = std::move(ContextHash);
+  TU.FileDeps = std::move(Dependencies);
+  TU.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps);
+  TU.Commands = std::move(Commands);
+  TU.CASFileSystemRootID = CASFileSystemRootID;
 
   for (auto &&M : ClangModuleDeps) {
     auto &MD = M.second;
     if (MD.ImportedByMainFile)
-      FD.ClangModuleDeps.push_back(MD.ID);
+      TU.ClangModuleDeps.push_back(MD.ID);
     // TODO: Avoid handleModuleDependency even being called for modules
     //   we've already seen.
     if (AlreadySeen.count(M.first))
       continue;
-    FDR.DiscoveredModules.push_back(std::move(MD));
+    TU.ModuleGraph.push_back(std::move(MD));
   }
 
-  return FDR;
+  return TU;
 }
 
-FullDependenciesResult
-FullDependencyConsumer::getFullDependenciesLegacyDriverCommand(
-    const std::vector<std::string> &OriginalCommandLine) const {
-  FullDependencies FD;
-
-  FD.DriverCommandLine = makeTUCommandLineWithoutPaths(
-      ArrayRef<std::string>(OriginalCommandLine).slice(1));
-
-  FD.ID.ContextHash = std::move(ContextHash);
-
-  FD.FileDeps.assign(Dependencies.begin(), Dependencies.end());
-
-  for (const PrebuiltModuleDep &PMD : PrebuiltModuleDeps)
-    FD.DriverCommandLine.push_back("-fmodule-file=" + PMD.PCMFile);
+ModuleDepsGraph FullDependencyConsumer::takeModuleGraphDeps() {
+  ModuleDepsGraph ModuleGraph;
 
   for (auto &&M : ClangModuleDeps) {
     auto &MD = M.second;
-    if (MD.ImportedByMainFile) {
-      FD.ClangModuleDeps.push_back(MD.ID);
-      auto PCMPath = LookupModuleOutput(MD.ID, ModuleOutputKind::ModuleFile);
-      if (EagerLoadModules) {
-        FD.DriverCommandLine.push_back("-fmodule-file=" + PCMPath);
-      } else {
-        FD.DriverCommandLine.push_back("-fmodule-map-file=" +
-                                       MD.ClangModuleMapFile);
-        FD.DriverCommandLine.push_back("-fmodule-file=" + MD.ID.ModuleName +
-                                       "=" + PCMPath);
-      }
-    }
-  }
-
-  FD.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps);
-
-  FD.CASFileSystemRootID = CASFileSystemRootID;
-
-  FullDependenciesResult FDR;
-
-  for (auto &&M : ClangModuleDeps) {
     // TODO: Avoid handleModuleDependency even being called for modules
     //   we've already seen.
     if (AlreadySeen.count(M.first))
       continue;
-    FDR.DiscoveredModules.push_back(std::move(M.second));
+    ModuleGraph.push_back(std::move(MD));
   }
 
-  FDR.FullDeps = std::move(FD);
-  return FDR;
+  return ModuleGraph;
 }