Merge pull request #6481 from benlangmuir/include-tree-libclang-stable

[clang][cas] Cherry-pick include-tree support for full dependencies and libclang

Author: benlangmuir

clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h

@@ -51,6 +51,9 @@ enum class ScanningOutputFormat {
 
   /// This emits the CAS ID of the include tree.
   IncludeTree,
+
+  /// This emits the full dependency graph but with include tree.
+  FullIncludeTree,
 };
 
 /// The dependency scanning service contains shared configuration and state that
@@ -59,6 +62,7 @@ class DependencyScanningService {
 public:
   DependencyScanningService(
       ScanningMode Mode, ScanningOutputFormat Format, CASOptions CASOpts,
+      std::shared_ptr<llvm::cas::ObjectStore> CAS,
       std::shared_ptr<llvm::cas::ActionCache> Cache,
       IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> SharedFS,
       bool OptimizeArgs = false, bool EagerLoadModules = false);
@@ -78,19 +82,19 @@ class DependencyScanningService {
   }
 
   const CASOptions &getCASOpts() const { return CASOpts; }
-  llvm::cas::ActionCache &getCache() const {
-    assert(Cache && "Cache is not initialized");
-    return *Cache;
-  }
+
+  std::shared_ptr<llvm::cas::ObjectStore> getCAS() const { return CAS; }
+  std::shared_ptr<llvm::cas::ActionCache> getCache() const { return Cache; }
 
   llvm::cas::CachingOnDiskFileSystem &getSharedFS() { return *SharedFS; }
 
-  bool useCASScanning() const { return (bool)SharedFS; }
+  bool useCASFS() const { return (bool)SharedFS; }
 
 private:
   const ScanningMode Mode;
   const ScanningOutputFormat Format;
   CASOptions CASOpts;
+  std::shared_ptr<llvm::cas::ObjectStore> CAS;
   std::shared_ptr<llvm::cas::ActionCache> Cache;
   /// Whether to optimize the modules' command-line arguments.
   const bool OptimizeArgs;

clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

@@ -69,7 +69,7 @@ struct TranslationUnitDeps {
   std::vector<ModuleID> ClangModuleDeps;
 
   /// The CASID for input file dependency tree.
-  llvm::Optional<llvm::cas::CASID> CASFileSystemRootID;
+  llvm::Optional<std::string> CASFileSystemRootID;
 
   /// The sequence of commands required to build the translation unit. Commands
   /// should be executed in order.
@@ -180,29 +180,24 @@ class DependencyScanningTool {
     return Worker.getOrCreateFileManager();
   }
 
+  static std::unique_ptr<DependencyActionController>
+  createActionController(DependencyScanningWorker &Worker,
+                         LookupModuleOutputCallback LookupModuleOutput,
+                         DepscanPrefixMapping PrefixMapping);
+
+private:
+  std::unique_ptr<DependencyActionController>
+  createActionController(LookupModuleOutputCallback LookupModuleOutput,
+                         DepscanPrefixMapping PrefixMapping);
+
 private:
   DependencyScanningWorker Worker;
 };
 
 class FullDependencyConsumer : public DependencyConsumer {
 public:
-  FullDependencyConsumer(const llvm::StringSet<> &AlreadySeen,
-                         LookupModuleOutputCallback LookupModuleOutput,
-                         CachingOnDiskFileSystemPtr CacheFS = nullptr,
-                         DepscanPrefixMapping PrefixMapping = {})
-      : CacheFS(std::move(CacheFS)), PrefixMapping(std::move(PrefixMapping)),
-        AlreadySeen(AlreadySeen), LookupModuleOutput(LookupModuleOutput) {}
-
-  llvm::Error initialize(CompilerInstance &ScanInstance,
-                         CompilerInvocation &NewInvocation) override;
-  llvm::Error finalize(CompilerInstance &ScanInstance,
-                       CompilerInvocation &NewInvocation) override;
-  llvm::Error
-  initializeModuleBuild(CompilerInstance &ModuleScanInstance) override;
-  llvm::Error
-  finalizeModuleBuild(CompilerInstance &ModuleScanInstance) override;
-  llvm::Error finalizeModuleInvocation(CompilerInvocation &CI,
-                                       const ModuleDeps &MD) override;
+  FullDependencyConsumer(const llvm::StringSet<> &AlreadySeen)
+      : AlreadySeen(AlreadySeen) {}
 
   void handleBuildCommand(Command Cmd) override {
     Commands.push_back(std::move(Cmd));
@@ -226,17 +221,8 @@ class FullDependencyConsumer : public DependencyConsumer {
     ContextHash = std::move(Hash);
   }
 
-  void handleCASFileSystemRootID(cas::CASID ID) override {
-    CASFileSystemRootID = ID;
-  }
-
-  Optional<cas::CASID> getCASFileSystemRootID() const {
-    return CASFileSystemRootID;
-  }
-
-  std::string lookupModuleOutput(const ModuleID &ID,
-                                 ModuleOutputKind Kind) override {
-    return LookupModuleOutput(ID, Kind);
+  void handleCASFileSystemRootID(std::string ID) override {
+    CASFileSystemRootID = std::move(ID);
   }
 
   TranslationUnitDeps takeTranslationUnitDeps();
@@ -249,13 +235,33 @@ class FullDependencyConsumer : public DependencyConsumer {
       ClangModuleDeps;
   std::vector<Command> Commands;
   std::string ContextHash;
-  CachingOnDiskFileSystemPtr CacheFS;
-  DepscanPrefixMapping PrefixMapping;
-  std::optional<llvm::TreePathPrefixMapper> Mapper;
-  CASOptions CASOpts;
-  Optional<cas::CASID> CASFileSystemRootID;
+  Optional<std::string> CASFileSystemRootID;
   std::vector<std::string> OutputPaths;
   const llvm::StringSet<> &AlreadySeen;
+};
+
+/// A simple dependency action controller that uses a callback. If no callback
+/// is provided, it is assumed that looking up module outputs is unreachable.
+class CallbackActionController : public DependencyActionController {
+public:
+  virtual ~CallbackActionController();
+
+  CallbackActionController(LookupModuleOutputCallback LMO)
+      : LookupModuleOutput(std::move(LMO)) {
+    if (!LookupModuleOutput) {
+      LookupModuleOutput = [](const ModuleID &,
+                              ModuleOutputKind) -> std::string {
+        llvm::report_fatal_error("unexpected call to lookupModuleOutput");
+      };
+    }
+  }
+
+  std::string lookupModuleOutput(const ModuleID &ID,
+                                 ModuleOutputKind Kind) override {
+    return LookupModuleOutput(ID, Kind);
+  }
+
+private:
   LookupModuleOutputCallback LookupModuleOutput;
 };
 

clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h

@@ -45,6 +45,33 @@ class DependencyConsumer {
 public:
   virtual ~DependencyConsumer() {}
 
+  virtual void handleBuildCommand(Command Cmd) {}
+
+  virtual void
+  handleDependencyOutputOpts(const DependencyOutputOptions &Opts) = 0;
+
+  virtual void handleFileDependency(StringRef Filename) = 0;
+
+  virtual void handlePrebuiltModuleDependency(PrebuiltModuleDep PMD) = 0;
+
+  virtual void handleModuleDependency(ModuleDeps MD) = 0;
+
+  virtual void handleContextHash(std::string Hash) = 0;
+
+  virtual void handleCASFileSystemRootID(std::string ID) {}
+
+  virtual void handleIncludeTreeID(std::string ID) {}
+};
+
+/// Dependency scanner callbacks that are used during scanning to influence the
+/// behaviour of the scan - for example, to customize the scanned invocations.
+class DependencyActionController {
+public:
+  virtual ~DependencyActionController();
+
+  virtual std::string lookupModuleOutput(const ModuleID &ID,
+                                         ModuleOutputKind Kind) = 0;
+
   virtual llvm::Error initialize(CompilerInstance &ScanInstance,
                                  CompilerInvocation &NewInvocation) {
     return llvm::Error::success();
@@ -70,65 +97,18 @@ class DependencyConsumer {
     return llvm::Error::success();
   }
 
-  virtual void handleBuildCommand(Command Cmd) = 0;
-
-  virtual void
-  handleDependencyOutputOpts(const DependencyOutputOptions &Opts) = 0;
-
-  virtual void handleFileDependency(StringRef Filename) = 0;
-
-  virtual void handlePrebuiltModuleDependency(PrebuiltModuleDep PMD) = 0;
-
-  virtual void handleModuleDependency(ModuleDeps MD) = 0;
-
-  virtual void handleContextHash(std::string Hash) = 0;
-
-  virtual void handleCASFileSystemRootID(cas::CASID ID) = 0;
-
-  virtual std::string lookupModuleOutput(const ModuleID &ID,
-                                         ModuleOutputKind Kind) = 0;
-};
-
-// FIXME: This may need to merge with \p DependencyConsumer in order to support
-// clang modules for the include-tree.
-class PPIncludeActionsConsumer : public DependencyConsumer {
-public:
-  virtual void enteredInclude(Preprocessor &PP, FileID FID) = 0;
+  virtual void enteredInclude(Preprocessor &PP, FileID FID) {}
 
   virtual void exitedInclude(Preprocessor &PP, FileID IncludedBy,
-                             FileID Include, SourceLocation ExitLoc) = 0;
+                             FileID Include, SourceLocation ExitLoc) {}
 
-  virtual void handleHasIncludeCheck(Preprocessor &PP, bool Result) = 0;
+  virtual void handleHasIncludeCheck(Preprocessor &PP, bool Result) {}
 
   /// FIXME: This is temporary until we eliminate the split between consumers in
   /// \p DependencyScanningTool and collectors in \p DependencyScanningWorker
   /// and have them both in the same file. see FIXME in \p
   /// DependencyScanningAction::runInvocation.
-  virtual const DepscanPrefixMapping &getPrefixMapping() = 0;
-
-protected:
-  void handleBuildCommand(Command) override {}
-  void handleDependencyOutputOpts(const DependencyOutputOptions &Opts) override {
-    llvm::report_fatal_error("unexpected callback for include-tree");
-  }
-  void handleFileDependency(StringRef Filename) override {
-    llvm::report_fatal_error("unexpected callback for include-tree");
-  }
-  void handlePrebuiltModuleDependency(PrebuiltModuleDep PMD) override {
-    llvm::report_fatal_error("unexpected callback for include-tree");
-  }
-  void handleModuleDependency(ModuleDeps MD) override {
-    llvm::report_fatal_error("unexpected callback for include-tree");
-  }
-  void handleContextHash(std::string Hash) override {
-    llvm::report_fatal_error("unexpected callback for include-tree");
-  }
-  void handleCASFileSystemRootID(cas::CASID ID) override {
-    llvm::report_fatal_error("unexpected callback for include-tree");
-  }
-  std::string lookupModuleOutput(const ModuleID &, ModuleOutputKind) override {
-    llvm::report_fatal_error("unexpected callback for include-tree");
-  }
+  virtual const DepscanPrefixMapping *getPrefixMapping() { return nullptr; }
 };
 
 /// An individual dependency scanning worker that is able to run on its own
@@ -152,16 +132,15 @@ class DependencyScanningWorker {
   bool computeDependencies(StringRef WorkingDirectory,
                            const std::vector<std::string> &CommandLine,
                            DependencyConsumer &DepConsumer,
+                           DependencyActionController &Controller,
                            DiagnosticConsumer &DiagConsumer,
                            llvm::Optional<StringRef> ModuleName = None);
   /// \returns A \c StringError with the diagnostic output if clang errors
   /// occurred, success otherwise.
-  llvm::Error computeDependencies(StringRef WorkingDirectory,
-                                  const std::vector<std::string> &CommandLine,
-                                  DependencyConsumer &Consumer,
-                                  llvm::Optional<StringRef> ModuleName = None);
-
-  ScanningOutputFormat getFormat() const { return Format; }
+  llvm::Error computeDependencies(
+      StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
+      DependencyConsumer &Consumer, DependencyActionController &Controller,
+      llvm::Optional<StringRef> ModuleName = None);
 
   /// Scan from a compiler invocation.
   /// If \p DiagGenerationAsCompilation is true it will generate error
@@ -171,15 +150,14 @@ class DependencyScanningWorker {
   void computeDependenciesFromCompilerInvocation(
       std::shared_ptr<CompilerInvocation> Invocation,
       StringRef WorkingDirectory, DependencyConsumer &Consumer,
-      DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS,
-      bool DiagGenerationAsCompilation);
+      DependencyActionController &Controller, DiagnosticConsumer &DiagsConsumer,
+      raw_ostream *VerboseOS, bool DiagGenerationAsCompilation);
 
   ScanningOutputFormat getScanningFormat() const { return Format; }
 
-  llvm::vfs::FileSystem &getRealFS() { return *BaseFS; }
   CachingOnDiskFileSystemPtr getCASFS() { return CacheFS; }
-  bool useCAS() const { return UseCAS; }
   const CASOptions &getCASOpts() const { return CASOpts; }
+  std::shared_ptr<cas::ObjectStore> getCAS() const { return CAS; }
 
   /// If \p DependencyScanningService enabled sharing of \p FileManager this
   /// will return the same instance, otherwise it will create a new one for
@@ -201,16 +179,15 @@ class DependencyScanningWorker {
   ScanningOutputFormat Format;
   /// Whether to optimize the modules' command-line arguments.
   bool OptimizeArgs;
+  /// Whether to set up command-lines to load PCM files eagerly.
+  bool EagerLoadModules;
 
   /// The caching file system.
   CachingOnDiskFileSystemPtr CacheFS;
   /// The CAS Dependency Filesytem. This is not set at the sametime as DepFS;
   llvm::IntrusiveRefCntPtr<DependencyScanningCASFilesystem> DepCASFS;
   CASOptions CASOpts;
-  bool UseCAS;
-
-  /// Whether to set up command-lines to load PCM files eagerly.
-  bool EagerLoadModules;
+  std::shared_ptr<cas::ObjectStore> CAS;
 };
 
 } // end namespace dependencies

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

@@ -28,6 +28,7 @@ namespace clang {
 namespace tooling {
 namespace dependencies {
 
+class DependencyActionController;
 class DependencyConsumer;
 
 /// Modular dependency that has already been built prior to the dependency scan.
@@ -192,6 +193,7 @@ class ModuleDepCollector final : public DependencyCollector {
 public:
   ModuleDepCollector(std::unique_ptr<DependencyOutputOptions> Opts,
                      CompilerInstance &ScanInstance, DependencyConsumer &C,
+                     DependencyActionController &Controller,
                      CompilerInvocation OriginalCI, bool OptimizeArgs,
                      bool EagerLoadModules);
 
@@ -209,6 +211,8 @@ class ModuleDepCollector final : public DependencyCollector {
   CompilerInstance &ScanInstance;
   /// The consumer of collected dependency information.
   DependencyConsumer &Consumer;
+  /// Callbacks for computing dependency information.
+  DependencyActionController &Controller;
   /// Path to the main source file.
   std::string MainFile;
   /// Hash identifying the compilation conditions of the current TU.