[clang][DependencyScanning] Separate clangDependencyScanning and DependencyScanningTool (NFC) (#169962)

This patch is the first of two in refactoring Clang's dependency
scanning tooling to remove its dependency on clangDriver.

It separates Tooling/DependencyScanningTool.cpp from the rest of
clangDependencyScanning and moves clangDependencyScanning out of
clangTooling into its own library. No functional changes are
introduced.

The follow-up patch (#169964) will restrict clangDependencyScanning to
handling only -cc1 command line inputs and will move all functionality
related to handling driver commands into clangTooling.
(Tooling/DependencyScanningTool.cpp).

This is part of a broader effort to support driver-managed builds for
compilations using C++ named modules and/or Clang modules. It is
required for linking the dependency scanning tooling against the driver
without introducing cyclic dependencies, which would otherwise cause
build failures when dynamic linking is enabled.

The RFC for this change can be found here:

https://discourse.llvm.org/t/rfc-new-clangoptions-library-remove-dependency-on-clangdriver-from-clangfrontend-and-flangfrontend/88773?u=naveen-seth

Author: naveen-seth

clang-tools-extra/clangd/ScanningProjectModules.cpp

@@ -8,8 +8,8 @@
 
 #include "ProjectModules.h"
 #include "support/Logger.h"
-#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
-#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
+#include "clang/DependencyScanning/DependencyScanningService.h"
+#include "clang/Tooling/DependencyScanningTool.h"
 
 namespace clang::clangd {
 namespace {
@@ -36,8 +36,8 @@ class ModuleDependencyScanner {
       std::shared_ptr<const clang::tooling::CompilationDatabase> CDB,
       const ThreadsafeFS &TFS)
       : CDB(CDB), TFS(TFS),
-        Service(tooling::dependencies::ScanningMode::CanonicalPreprocessing,
-                tooling::dependencies::ScanningOutputFormat::P1689) {}
+        Service(dependencies::ScanningMode::CanonicalPreprocessing,
+                dependencies::ScanningOutputFormat::P1689) {}
 
   /// The scanned modules dependency information for a specific source file.
   struct ModuleDependencyInfo {
@@ -81,7 +81,7 @@ class ModuleDependencyScanner {
   // Whether the scanner has scanned the project globally.
   bool GlobalScanned = false;
 
-  clang::tooling::dependencies::DependencyScanningService Service;
+  clang::dependencies::DependencyScanningService Service;
 
   // TODO: Add a scanning cache.
 

clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h renamed to clang/include/clang/DependencyScanning/DependencyScannerImpl.h

@@ -1,4 +1,4 @@
-//===- DependencyScannerImpl.h - Implements dependency scanning *- C++ -*--===//
+//===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -9,18 +9,18 @@
 #ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNER_H
 #define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNER_H
 
+#include "clang/DependencyScanning/DependencyScanningFilesystem.h"
+#include "clang/DependencyScanning/ModuleDepCollector.h"
 #include "clang/Driver/Compilation.h"
+#include "clang/Driver/Driver.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Frontend/TextDiagnosticPrinter.h"
 #include "clang/Serialization/ObjectFilePCHContainerReader.h"
-#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
-#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
 
 namespace clang {
 class DiagnosticConsumer;
 
-namespace tooling {
 namespace dependencies {
 class DependencyScanningService;
 class DependencyScanningWorker;
@@ -191,7 +191,6 @@ class CompilerInstanceWithContext {
   llvm::Error handleReturnStatus(bool Success);
 };
 } // namespace dependencies
-} // namespace tooling
 } // namespace clang
 
 #endif

clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h renamed to clang/include/clang/DependencyScanning/DependencyScanningFilesystem.h

@@ -1,13 +1,13 @@
-//===- DependencyScanningFilesystem.h - clang-scan-deps fs ===---*- C++ -*-===//
+//===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H
-#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H
+#ifndef LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H
+#define LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H
 
 #include "clang/Basic/LLVM.h"
 #include "clang/Lex/DependencyDirectivesScanner.h"
@@ -21,7 +21,6 @@
 #include <variant>
 
 namespace clang {
-namespace tooling {
 namespace dependencies {
 
 using DependencyDirectivesTy =
@@ -521,7 +520,6 @@ class DependencyScanningWorkerFilesystem
 };
 
 } // end namespace dependencies
-} // end namespace tooling
 } // end namespace clang
 
-#endif // LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H
+#endif // LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H

clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h renamed to clang/include/clang/DependencyScanning/DependencyScanningService.h

@@ -1,21 +1,20 @@
-//===- DependencyScanningService.h - clang-scan-deps service ===-*- C++ -*-===//
+//===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGSERVICE_H
-#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGSERVICE_H
+#ifndef LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGSERVICE_H
+#define LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGSERVICE_H
 
-#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
-#include "clang/Tooling/DependencyScanning/InProcessModuleCache.h"
+#include "clang/DependencyScanning/DependencyScanningFilesystem.h"
+#include "clang/DependencyScanning/InProcessModuleCache.h"
 #include "llvm/ADT/BitmaskEnum.h"
 #include "llvm/Support/Chrono.h"
 
 namespace clang {
-namespace tooling {
 namespace dependencies {
 
 /// The mode in which the dependency scanner will operate to find the
@@ -125,7 +124,6 @@ class DependencyScanningService {
 };
 
 } // end namespace dependencies
-} // end namespace tooling
 } // end namespace clang
 
-#endif // LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGSERVICE_H
+#endif // LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGSERVICE_H

clang/include/clang/DependencyScanning/DependencyScanningUtils.h

@@ -0,0 +1,170 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGUTILS_H
+#define LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGUTILS_H
+
+#include "clang/DependencyScanning/DependencyScannerImpl.h"
+#include "clang/DependencyScanning/DependencyScanningWorker.h"
+#include "clang/DependencyScanning/ModuleDepCollector.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/MapVector.h"
+#include <string>
+#include <vector>
+
+namespace clang {
+namespace dependencies {
+
+/// Graph of modular dependencies.
+using ModuleDepsGraph = std::vector<clang::dependencies::ModuleDeps>;
+
+/// The full dependencies and module graph for a specific input.
+struct TranslationUnitDeps {
+  /// The graph of direct and transitive modular dependencies.
+  ModuleDepsGraph ModuleGraph;
+
+  /// The identifier of the C++20 module this translation unit exports.
+  ///
+  /// If the translation unit is not a module then \c ID.ModuleName is empty.
+  clang::dependencies::ModuleID ID;
+
+  /// A collection of absolute paths to files that this translation unit
+  /// directly depends on, not including transitive dependencies.
+  std::vector<std::string> FileDeps;
+
+  /// A collection of prebuilt modules this translation unit directly depends
+  /// on, not including transitive dependencies.
+  std::vector<clang::dependencies::PrebuiltModuleDep> PrebuiltModuleDeps;
+
+  /// A list of modules this translation unit directly depends on, not including
+  /// transitive dependencies.
+  ///
+  /// This may include modules with a different context hash when it can be
+  /// determined that the differences are benign for this compilation.
+  std::vector<clang::dependencies::ModuleID> ClangModuleDeps;
+
+  /// A list of module names that are visible to this translation unit. This
+  /// includes both direct and transitive module dependencies.
+  std::vector<std::string> VisibleModules;
+
+  /// A list of the C++20 named modules this translation unit depends on.
+  std::vector<std::string> NamedModuleDeps;
+
+  /// The sequence of commands required to build the translation unit. Commands
+  /// should be executed in order.
+  ///
+  /// FIXME: If we add support for multi-arch builds in clang-scan-deps, we
+  /// should make the dependencies between commands explicit to enable parallel
+  /// builds of each architecture.
+  std::vector<clang::dependencies::Command> Commands;
+
+  /// Deprecated driver command-line. This will be removed in a future version.
+  std::vector<std::string> DriverCommandLine;
+};
+
+class FullDependencyConsumer : public clang::dependencies::DependencyConsumer {
+public:
+  FullDependencyConsumer(
+      const llvm::DenseSet<clang::dependencies::ModuleID> &AlreadySeen)
+      : AlreadySeen(AlreadySeen) {}
+
+  void handleBuildCommand(clang::dependencies::Command Cmd) override {
+    Commands.push_back(std::move(Cmd));
+  }
+
+  void handleDependencyOutputOpts(const DependencyOutputOptions &) override {}
+
+  void handleFileDependency(StringRef File) override {
+    Dependencies.push_back(std::string(File));
+  }
+
+  void handlePrebuiltModuleDependency(
+      clang::dependencies::PrebuiltModuleDep PMD) override {
+    PrebuiltModuleDeps.emplace_back(std::move(PMD));
+  }
+
+  void handleModuleDependency(clang::dependencies::ModuleDeps MD) override {
+    ClangModuleDeps[MD.ID] = std::move(MD);
+  }
+
+  void handleDirectModuleDependency(clang::dependencies::ModuleID ID) override {
+    DirectModuleDeps.push_back(ID);
+  }
+
+  void handleVisibleModule(std::string ModuleName) override {
+    VisibleModules.push_back(ModuleName);
+  }
+
+  void handleContextHash(std::string Hash) override {
+    ContextHash = std::move(Hash);
+  }
+
+  void handleProvidedAndRequiredStdCXXModules(
+      std::optional<clang::dependencies::P1689ModuleInfo> Provided,
+      std::vector<clang::dependencies::P1689ModuleInfo> Requires) override {
+    ModuleName = Provided ? Provided->ModuleName : "";
+    llvm::transform(Requires, std::back_inserter(NamedModuleDeps),
+                    [](const auto &Module) { return Module.ModuleName; });
+  }
+
+  TranslationUnitDeps takeTranslationUnitDeps();
+
+private:
+  std::vector<std::string> Dependencies;
+  std::vector<clang::dependencies::PrebuiltModuleDep> PrebuiltModuleDeps;
+  llvm::MapVector<clang::dependencies::ModuleID,
+                  clang::dependencies::ModuleDeps>
+      ClangModuleDeps;
+  std::string ModuleName;
+  std::vector<std::string> NamedModuleDeps;
+  std::vector<clang::dependencies::ModuleID> DirectModuleDeps;
+  std::vector<std::string> VisibleModules;
+  std::vector<clang::dependencies::Command> Commands;
+  std::string ContextHash;
+  const llvm::DenseSet<clang::dependencies::ModuleID> &AlreadySeen;
+};
+
+/// A callback to lookup module outputs for "-fmodule-file=", "-o" etc.
+using LookupModuleOutputCallback =
+    llvm::function_ref<std::string(const clang::dependencies::ModuleDeps &,
+                                   clang::dependencies::ModuleOutputKind)>;
+
+/// A simple dependency action controller that uses a callback. If no callback
+/// is provided, it is assumed that looking up module outputs is unreachable.
+class CallbackActionController
+    : public clang::dependencies::DependencyActionController {
+public:
+  virtual ~CallbackActionController();
+
+  static std::string
+  lookupUnreachableModuleOutput(const clang::dependencies::ModuleDeps &MD,
+                                clang::dependencies::ModuleOutputKind Kind) {
+    llvm::report_fatal_error("unexpected call to lookupModuleOutput");
+  };
+
+  CallbackActionController(LookupModuleOutputCallback LMO)
+      : LookupModuleOutput(std::move(LMO)) {
+    if (!LookupModuleOutput) {
+      LookupModuleOutput = lookupUnreachableModuleOutput;
+    }
+  }
+
+  std::string
+  lookupModuleOutput(const clang::dependencies::ModuleDeps &MD,
+                     clang::dependencies::ModuleOutputKind Kind) override {
+    return LookupModuleOutput(MD, Kind);
+  }
+
+private:
+  LookupModuleOutputCallback LookupModuleOutput;
+};
+
+} // end namespace dependencies
+} // end namespace clang
+
+#endif // LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGUTILS_H

clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h renamed to clang/include/clang/DependencyScanning/DependencyScanningWorker.h

@@ -1,20 +1,20 @@
-//===- DependencyScanningWorker.h - clang-scan-deps worker ===---*- C++ -*-===//
+//===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGWORKER_H
-#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGWORKER_H
+#ifndef LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGWORKER_H
+#define LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGWORKER_H
 
 #include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/LLVM.h"
+#include "clang/DependencyScanning/DependencyScanningService.h"
+#include "clang/DependencyScanning/ModuleDepCollector.h"
 #include "clang/Frontend/PCHContainerOperations.h"
-#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
-#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBufferRef.h"
@@ -25,7 +25,6 @@ namespace clang {
 
 class DependencyOutputOptions;
 
-namespace tooling {
 namespace dependencies {
 
 class DependencyScanningWorkerFilesystem;
@@ -180,7 +179,6 @@ class DependencyScanningWorker {
 };
 
 } // end namespace dependencies
-} // end namespace tooling
 } // end namespace clang
 
-#endif // LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGWORKER_H
+#endif // LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGWORKER_H

clang/include/clang/Tooling/DependencyScanning/InProcessModuleCache.h renamed to clang/include/clang/DependencyScanning/InProcessModuleCache.h

@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_INPROCESSMODULECACHE_H
-#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_INPROCESSMODULECACHE_H
+#ifndef LLVM_CLANG_DEPENDENCYSCANNING_INPROCESSMODULECACHE_H
+#define LLVM_CLANG_DEPENDENCYSCANNING_INPROCESSMODULECACHE_H
 
 #include "clang/Serialization/ModuleCache.h"
 #include "llvm/ADT/StringMap.h"
@@ -16,8 +16,8 @@
 #include <shared_mutex>
 
 namespace clang {
-namespace tooling {
 namespace dependencies {
+
 struct ModuleCacheEntry {
   std::shared_mutex CompilationMutex;
   std::atomic<std::time_t> Timestamp = 0;
@@ -30,8 +30,8 @@ struct ModuleCacheEntries {
 
 IntrusiveRefCntPtr<ModuleCache>
 makeInProcessModuleCache(ModuleCacheEntries &Entries);
+
 } // namespace dependencies
-} // namespace tooling
 } // namespace clang
 
-#endif
+#endif // LLVM_CLANG_DEPENDENCYSCANNING_INPROCESSMODULECACHE_H