[clang] Refactor to remove clangDriver dependency from clangFrontend

This PR removes the clangDriver dependency from clangFrontend.
The goal of this change is to remove dependencies on the Driver.

This refactoring is part of a broader effort to support driver-managed
builds for compilations using C++ named modules and/or Clang modules.
It is required for linking the dependency scanning tooling against the
driver without introducing cyclic dependencies, which would otherwise
cause build failures when dynamic linking is enabled.
In particular, clangFrontend must no longer depend on clangDriver for
this to be possible.

This change is motivated by the following review comment:
llvm#152770 (comment)

Author: naveen-seth

clang-tools-extra/clangd/CompileCommands.cpp

@@ -132,8 +132,8 @@ std::optional<std::string> detectSysroot() {
 
 std::string detectStandardResourceDir() {
   static int StaticForMainAddr; // Just an address in this process.
-  return CompilerInvocation::GetResourcesPath("clangd",
-                                              (void *)&StaticForMainAddr);
+  return driver::Driver::GetResourcesPathForInvocation(
+      "clangd", (void *)&StaticForMainAddr);
 }
 
 // The path passed to argv[0] is important:

clang-tools-extra/clangd/Compiler.cpp

@@ -9,6 +9,7 @@
 #include "Compiler.h"
 #include "support/Logger.h"
 #include "clang/Basic/TargetInfo.h"
+#include "clang/Driver/CreateInvocationFromArgs.h"
 #include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Serialization/PCHContainerOperations.h"

clang/include/clang/Driver/CreateASTUnitFromArgs.h

@@ -0,0 +1,80 @@
+//===-- CreateInvocationFromArgs.h - Create an ASTUnit from Args-*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Utility for creating an ASTUnit from a vector of command line arguments.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_DRIVER_CREATEASTUNITFROMARGS_H
+#define LLVM_CLANG_DRIVER_CREATEASTUNITFROMARGS_H
+
+#include "clang/Frontend/ASTUnit.h"
+
+namespace clang {
+
+/// Create an ASTUnit from a vector of command line arguments, which must
+/// specify exactly one source file.
+///
+/// \param ArgBegin - The beginning of the argument vector.
+///
+/// \param ArgEnd - The end of the argument vector.
+///
+/// \param PCHContainerOps - The PCHContainerOperations to use for loading and
+/// creating modules.
+///
+/// \param Diags - The diagnostics engine to use for reporting errors; its
+/// lifetime is expected to extend past that of the returned ASTUnit.
+///
+/// \param ResourceFilesPath - The path to the compiler resource files.
+///
+/// \param StorePreamblesInMemory - Whether to store PCH in memory. If false,
+/// PCH are stored in temporary files.
+///
+/// \param PreambleStoragePath - The path to a directory, in which to create
+/// temporary PCH files. If empty, the default system temporary directory is
+/// used. This parameter is ignored if \p StorePreamblesInMemory is true.
+///
+/// \param ModuleFormat - If provided, uses the specific module format.
+///
+/// \param ErrAST - If non-null and parsing failed without any AST to return
+/// (e.g. because the PCH could not be loaded), this accepts the ASTUnit
+/// mainly to allow the caller to see the diagnostics.
+///
+/// \param VFS - A llvm::vfs::FileSystem to be used for all file accesses.
+/// Note that preamble is saved to a temporary directory on a RealFileSystem,
+/// so in order for it to be loaded correctly, VFS should have access to
+/// it(i.e., be an overlay over RealFileSystem). RealFileSystem will be used
+/// if \p VFS is nullptr.
+///
+// FIXME: Move OnlyLocalDecls, UseBumpAllocator to setters on the ASTUnit, we
+// shouldn't need to specify them at construction time.
+std::unique_ptr<ASTUnit> CreateASTUnitFromCommandLine(
+    const char **ArgBegin, const char **ArgEnd,
+    std::shared_ptr<PCHContainerOperations> PCHContainerOps,
+    std::shared_ptr<DiagnosticOptions> DiagOpts,
+    IntrusiveRefCntPtr<DiagnosticsEngine> Diags, StringRef ResourceFilesPath,
+    bool StorePreamblesInMemory = false,
+    StringRef PreambleStoragePath = StringRef(), bool OnlyLocalDecls = false,
+    CaptureDiagsKind CaptureDiagnostics = CaptureDiagsKind::None,
+    ArrayRef<ASTUnit::RemappedFile> RemappedFiles = {},
+    bool RemappedFilesKeepOriginalName = true,
+    unsigned PrecompilePreambleAfterNParses = 0,
+    TranslationUnitKind TUKind = TU_Complete,
+    bool CacheCodeCompletionResults = false,
+    bool IncludeBriefCommentsInCodeCompletion = false,
+    bool AllowPCHWithCompilerErrors = false,
+    SkipFunctionBodiesScope SkipFunctionBodies = SkipFunctionBodiesScope::None,
+    bool SingleFileParse = false, bool UserFilesAreVolatile = false,
+    bool ForSerialization = false, bool RetainExcludedConditionalBlocks = false,
+    std::optional<StringRef> ModuleFormat = std::nullopt,
+    std::unique_ptr<ASTUnit> *ErrAST = nullptr,
+    IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = nullptr);
+
+} // namespace clang
+
+#endif // LLVM_CLANG_DRIVER_CREATEASTUNITFROMARGS_H

clang/include/clang/Driver/CreateInvocationFromArgs.h

@@ -0,0 +1,76 @@
+//===--- CreateInvocationFromArgs.h - CompilerInvocation from Args --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Utility for creating a CompilerInvocation from command-line arguments, for
+// tools to use in preparation to parse a file.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_DRIVER_CREATEINVOCATIONFROMARGS_H
+#define LLVM_CLANG_DRIVER_CREATEINVOCATIONFROMARGS_H
+
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/LLVM.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace clang {
+
+class CompilerInvocation;
+class DiagnosticsEngine;
+
+/// Optional inputs to createInvocation.
+struct CreateInvocationOptions {
+  /// Receives diagnostics encountered while parsing command-line flags.
+  /// If not provided, these are printed to stderr.
+  IntrusiveRefCntPtr<DiagnosticsEngine> Diags = nullptr;
+  /// Used e.g. to probe for system headers locations.
+  /// If not provided, the real filesystem is used.
+  /// FIXME: the driver does perform some non-virtualized IO.
+  IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = nullptr;
+  /// Whether to attempt to produce a non-null (possibly incorrect) invocation
+  /// if any errors were encountered.
+  /// By default, always return null on errors.
+  bool RecoverOnError = false;
+  /// Allow the driver to probe the filesystem for PCH files.
+  /// This is used to replace -include with -include-pch in the cc1 args.
+  /// FIXME: ProbePrecompiled=true is a poor, historical default.
+  /// It misbehaves if the PCH file is from GCC, has the wrong version, etc.
+  bool ProbePrecompiled = false;
+  /// If set, the target is populated with the cc1 args produced by the driver.
+  /// This may be populated even if createInvocation returns nullptr.
+  std::vector<std::string> *CC1Args = nullptr;
+};
+
+/// Interpret clang arguments in preparation to parse a file.
+///
+/// This simulates a number of steps Clang takes when its driver is invoked:
+/// - choosing actions (e.g compile + link) to run
+/// - probing the system for settings like standard library locations
+/// - spawning a cc1 subprocess to compile code, with more explicit arguments
+/// - in the cc1 process, assembling those arguments into a CompilerInvocation
+///   which is used to configure the parser
+///
+/// This simulation is lossy, e.g. in some situations one driver run would
+/// result in multiple parses. (Multi-arch, CUDA, ...).
+/// This function tries to select a reasonable invocation that tools should use.
+///
+/// Args[0] should be the driver name, such as "clang" or "/usr/bin/g++".
+/// Absolute path is preferred - this affects searching for system headers.
+///
+/// May return nullptr if an invocation could not be determined.
+/// See CreateInvocationOptions::RecoverOnError to try harder!
+std::unique_ptr<CompilerInvocation>
+createInvocation(ArrayRef<const char *> Args,
+                 CreateInvocationOptions Opts = {});
+
+} // namespace clang
+
+#endif // LLVM_CLANG_DRIVER_CREATEINVOCATIONFROMARGS_H

clang/include/clang/Driver/Driver.h

@@ -411,6 +411,16 @@ class Driver {
   /// the path to clang's resource directory.
   static std::string GetResourcesPath(StringRef BinaryPath);
 
+  /// Get the directory where the compiler headers reside, relative to the
+  /// compiler binary (found by the passed in arguments).
+  ///
+  /// \param Argv0 - The program path (from argv[0]), for finding the builtin
+  /// compiler path.
+  /// \param MainAddr - The address of main (or some other function in the main
+  /// executable), for finding the builtin compiler path.
+  static std::string GetResourcesPathForInvocation(const char *Argv0,
+                                                   void *MainAddr);
+
   Driver(StringRef ClangExecutable, StringRef TargetTriple,
          DiagnosticsEngine &Diags, std::string Title = "clang LLVM compiler",
          IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = nullptr);

clang/include/clang/Frontend/ASTUnit.h

@@ -22,12 +22,14 @@
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetOptions.h"
+#include "clang/Frontend/PrecompiledPreamble.h"
+#include "clang/Frontend/StandaloneDiagnostic.h"
 #include "clang/Lex/HeaderSearchOptions.h"
 #include "clang/Lex/ModuleLoader.h"
 #include "clang/Lex/PreprocessingRecord.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
 #include "clang/Serialization/ASTBitCodes.h"
-#include "clang/Frontend/PrecompiledPreamble.h"
+#include "clang/Serialization/ASTWriter.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
@@ -36,6 +38,7 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator_range.h"
+#include "llvm/Bitstream/BitstreamWriter.h"
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
@@ -88,25 +91,6 @@ enum class CaptureDiagsKind { None, All, AllWithoutNonErrorsFromIncludes };
 
 /// Utility class for loading a ASTContext from an AST file.
 class ASTUnit {
-public:
-  struct StandaloneFixIt {
-    std::pair<unsigned, unsigned> RemoveRange;
-    std::pair<unsigned, unsigned> InsertFromRange;
-    std::string CodeToInsert;
-    bool BeforePreviousInsertions;
-  };
-
-  struct StandaloneDiagnostic {
-    unsigned ID;
-    DiagnosticsEngine::Level Level;
-    std::string Message;
-    std::string Filename;
-    unsigned LocOffset;
-    std::vector<std::pair<unsigned, unsigned>> Ranges;
-    std::vector<StandaloneFixIt> FixIts;
-  };
-
-private:
   std::unique_ptr<LangOptions> LangOpts;
   std::unique_ptr<CodeGenOptions> CodeGenOpts;
   // FIXME: The documentation on \c LoadFrom* member functions states that the
@@ -129,7 +113,15 @@ class ASTUnit {
   bool HadModuleLoaderFatalFailure = false;
   bool StorePreamblesInMemory = false;
 
-  struct ASTWriterData;
+  /// Utility struct for managing ASTWriter and its associated data streams.
+  struct ASTWriterData {
+    SmallString<128> Buffer;
+    llvm::BitstreamWriter Stream;
+    ASTWriter Writer;
+
+    ASTWriterData(ModuleCache &ModCache, const CodeGenOptions &CGOpts)
+        : Stream(Buffer), Writer(Stream, Buffer, ModCache, CGOpts, {}) {}
+  };
   std::unique_ptr<ASTWriterData> WriterData;
 
   FileSystemOptions FileSystemOpts;
@@ -271,11 +263,6 @@ class ASTUnit {
   static void ConfigureDiags(IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
                              ASTUnit &AST, CaptureDiagsKind CaptureDiagnostics);
 
-  void TranslateStoredDiagnostics(FileManager &FileMgr,
-                                  SourceManager &SrcMan,
-                      const SmallVectorImpl<StandaloneDiagnostic> &Diags,
-                            SmallVectorImpl<StoredDiagnostic> &Out);
-
   void clearFileLevelDecls();
 
 public:
@@ -834,65 +821,24 @@ class ASTUnit {
       bool IncludeBriefCommentsInCodeCompletion = false,
       bool UserFilesAreVolatile = false);
 
-  /// LoadFromCommandLine - Create an ASTUnit from a vector of command line
-  /// arguments, which must specify exactly one source file.
-  ///
-  /// \param ArgBegin - The beginning of the argument vector.
-  ///
-  /// \param ArgEnd - The end of the argument vector.
-  ///
-  /// \param PCHContainerOps - The PCHContainerOperations to use for loading and
-  /// creating modules.
-  ///
-  /// \param Diags - The diagnostics engine to use for reporting errors; its
-  /// lifetime is expected to extend past that of the returned ASTUnit.
-  ///
-  /// \param ResourceFilesPath - The path to the compiler resource files.
-  ///
-  /// \param StorePreamblesInMemory - Whether to store PCH in memory. If false,
-  /// PCH are stored in temporary files.
-  ///
-  /// \param PreambleStoragePath - The path to a directory, in which to create
-  /// temporary PCH files. If empty, the default system temporary directory is
-  /// used. This parameter is ignored if \p StorePreamblesInMemory is true.
-  ///
-  /// \param ModuleFormat - If provided, uses the specific module format.
-  ///
-  /// \param ErrAST - If non-null and parsing failed without any AST to return
-  /// (e.g. because the PCH could not be loaded), this accepts the ASTUnit
-  /// mainly to allow the caller to see the diagnostics.
-  ///
-  /// \param VFS - A llvm::vfs::FileSystem to be used for all file accesses.
-  /// Note that preamble is saved to a temporary directory on a RealFileSystem,
-  /// so in order for it to be loaded correctly, VFS should have access to
-  /// it(i.e., be an overlay over RealFileSystem). RealFileSystem will be used
-  /// if \p VFS is nullptr.
-  ///
-  // FIXME: Move OnlyLocalDecls, UseBumpAllocator to setters on the ASTUnit, we
-  // shouldn't need to specify them at construction time.
-  static std::unique_ptr<ASTUnit> LoadFromCommandLine(
+  friend std::unique_ptr<ASTUnit> CreateASTUnitFromCommandLine(
       const char **ArgBegin, const char **ArgEnd,
       std::shared_ptr<PCHContainerOperations> PCHContainerOps,
       std::shared_ptr<DiagnosticOptions> DiagOpts,
       IntrusiveRefCntPtr<DiagnosticsEngine> Diags, StringRef ResourceFilesPath,
-      bool StorePreamblesInMemory = false,
-      StringRef PreambleStoragePath = StringRef(), bool OnlyLocalDecls = false,
-      CaptureDiagsKind CaptureDiagnostics = CaptureDiagsKind::None,
-      ArrayRef<RemappedFile> RemappedFiles = {},
-      bool RemappedFilesKeepOriginalName = true,
-      unsigned PrecompilePreambleAfterNParses = 0,
-      TranslationUnitKind TUKind = TU_Complete,
-      bool CacheCodeCompletionResults = false,
-      bool IncludeBriefCommentsInCodeCompletion = false,
-      bool AllowPCHWithCompilerErrors = false,
-      SkipFunctionBodiesScope SkipFunctionBodies =
-          SkipFunctionBodiesScope::None,
-      bool SingleFileParse = false, bool UserFilesAreVolatile = false,
-      bool ForSerialization = false,
-      bool RetainExcludedConditionalBlocks = false,
-      std::optional<StringRef> ModuleFormat = std::nullopt,
-      std::unique_ptr<ASTUnit> *ErrAST = nullptr,
-      IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = nullptr);
+      bool StorePreamblesInMemory, StringRef PreambleStoragePath,
+      bool OnlyLocalDecls, CaptureDiagsKind CaptureDiagnostics,
+      ArrayRef<ASTUnit::RemappedFile> RemappedFiles,
+      bool RemappedFilesKeepOriginalName,
+      unsigned PrecompilePreambleAfterNParses, TranslationUnitKind TUKind,
+      bool CacheCodeCompletionResults,
+      bool IncludeBriefCommentsInCodeCompletion,
+      bool AllowPCHWithCompilerErrors,
+      SkipFunctionBodiesScope SkipFunctionBodies, bool SingleFileParse,
+      bool UserFilesAreVolatile, bool ForSerialization,
+      bool RetainExcludedConditionalBlocks,
+      std::optional<StringRef> ModuleFormat, std::unique_ptr<ASTUnit> *ErrAST,
+      IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS);
 
   /// Reparse the source files using the same command-line options that
   /// were originally used to produce this translation unit.
@@ -963,6 +909,44 @@ class ASTUnit {
   bool serialize(raw_ostream &OS);
 };
 
+/// Diagnostic consumer that saves each diagnostic it is given.
+class FilterAndStoreDiagnosticConsumer : public DiagnosticConsumer {
+  SmallVectorImpl<StoredDiagnostic> *StoredDiags;
+  SmallVectorImpl<StandaloneDiagnostic> *StandaloneDiags;
+  bool CaptureNonErrorsFromIncludes = true;
+  const LangOptions *LangOpts = nullptr;
+  SourceManager *SourceMgr = nullptr;
+
+public:
+  FilterAndStoreDiagnosticConsumer(
+      SmallVectorImpl<StoredDiagnostic> *StoredDiags,
+      SmallVectorImpl<StandaloneDiagnostic> *StandaloneDiags,
+      bool CaptureNonErrorsFromIncludes);
+
+  void BeginSourceFile(const LangOptions &LangOpts,
+                       const Preprocessor *PP = nullptr) override;
+
+  void HandleDiagnostic(DiagnosticsEngine::Level Level,
+                        const Diagnostic &Info) override;
+};
+
+/// RAII object that optionally captures and filters diagnostics, if
+/// there is no diagnostic client to capture them already.
+class CaptureDroppedDiagnostics {
+  DiagnosticsEngine &Diags;
+  FilterAndStoreDiagnosticConsumer Client;
+  DiagnosticConsumer *PreviousClient = nullptr;
+  std::unique_ptr<DiagnosticConsumer> OwningPreviousClient;
+
+public:
+  CaptureDroppedDiagnostics(
+      CaptureDiagsKind CaptureDiagnostics, DiagnosticsEngine &Diags,
+      SmallVectorImpl<StoredDiagnostic> *StoredDiags,
+      SmallVectorImpl<StandaloneDiagnostic> *StandaloneDiags);
+
+  ~CaptureDroppedDiagnostics();
+};
+
 } // namespace clang
 
 #endif // LLVM_CLANG_FRONTEND_ASTUNIT_H