[clang] Refactor to remove clangDriver dependency from clangFrontend and flangFrontend (llvm#165277)

This removes the dependency on clangDriver from clangFrontend and
flangFrontend.

This refactoring is part of a broader effort to support driver-managed
builds for compilations using C++ named modules and/or Clang modules.
It is required for linking the dependency scanning tooling against the
driver without introducing cyclic dependencies, which would otherwise
cause build failures when dynamic linking is enabled.
In particular, clangFrontend must no longer depend on clangDriver
for this to be possible.

This change was discussed in the following RFC:
https://discourse.llvm.org/t/rfc-new-clangoptions-library-remove-dependency-on-clangdriver-from-clangfrontend-and-flangfrontend/88773

Author: naveen-seth

clang-tools-extra/clangd/CompileCommands.cpp

@@ -132,8 +132,7 @@ std::optional<std::string> detectSysroot() {
 
 std::string detectStandardResourceDir() {
   static int StaticForMainAddr; // Just an address in this process.
-  return CompilerInvocation::GetResourcesPath("clangd",
-                                              (void *)&StaticForMainAddr);
+  return GetResourcesPath("clangd", (void *)&StaticForMainAddr);
 }
 
 // The path passed to argv[0] is important:

clang-tools-extra/clangd/Compiler.cpp

@@ -9,6 +9,7 @@
 #include "Compiler.h"
 #include "support/Logger.h"
 #include "clang/Basic/TargetInfo.h"
+#include "clang/Driver/CreateInvocationFromArgs.h"
 #include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Serialization/PCHContainerOperations.h"

clang/docs/ReleaseNotes.rst

@@ -84,6 +84,8 @@ Potentially Breaking Changes
 - Downstream projects that previously linked only against ``clangDriver`` may
   now (also) need to link against the new ``clangOptions`` library, since
   options-related code has been moved out of the Driver into a separate library.
+- The ``clangFrontend`` library no longer depends on ``clangDriver``, which may
+  break downstream projects that relied on this transitive dependency.
 
 C/C++ Language Potentially Breaking Changes
 -------------------------------------------

clang/include/clang/Driver/CommonArgs.h

@@ -291,16 +291,6 @@ void handleVectorizeLoopsArgs(const llvm::opt::ArgList &Args,
 void handleVectorizeSLPArgs(const llvm::opt::ArgList &Args,
                             llvm::opt::ArgStringList &CmdArgs);
 
-// Parse -mprefer-vector-width=. Return the Value string if well-formed.
-// Otherwise, return an empty string and issue a diagnosic message if needed.
-StringRef parseMPreferVectorWidthOption(clang::DiagnosticsEngine &Diags,
-                                        const llvm::opt::ArgList &Args);
-
-// Parse -mrecip. Return the Value string if well-formed.
-// Otherwise, return an empty string and issue a diagnosic message if needed.
-StringRef parseMRecipOption(clang::DiagnosticsEngine &Diags,
-                            const llvm::opt::ArgList &Args);
-
 // Convert ComplexRangeKind to a string that can be passed as a frontend option.
 std::string complexRangeKindToStr(LangOptions::ComplexRangeKind Range);
 

clang/include/clang/Driver/CreateASTUnitFromArgs.h

@@ -0,0 +1,80 @@
+//===-- CreateInvocationFromArgs.h - Create an ASTUnit from Args-*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Utility for creating an ASTUnit from a vector of command line arguments.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_DRIVER_CREATEASTUNITFROMARGS_H
+#define LLVM_CLANG_DRIVER_CREATEASTUNITFROMARGS_H
+
+#include "clang/Frontend/ASTUnit.h"
+
+namespace clang {
+
+/// Create an ASTUnit from a vector of command line arguments, which must
+/// specify exactly one source file.
+///
+/// \param ArgBegin - The beginning of the argument vector.
+///
+/// \param ArgEnd - The end of the argument vector.
+///
+/// \param PCHContainerOps - The PCHContainerOperations to use for loading and
+/// creating modules.
+///
+/// \param Diags - The diagnostics engine to use for reporting errors; its
+/// lifetime is expected to extend past that of the returned ASTUnit.
+///
+/// \param ResourceFilesPath - The path to the compiler resource files.
+///
+/// \param StorePreamblesInMemory - Whether to store PCH in memory. If false,
+/// PCH are stored in temporary files.
+///
+/// \param PreambleStoragePath - The path to a directory, in which to create
+/// temporary PCH files. If empty, the default system temporary directory is
+/// used. This parameter is ignored if \p StorePreamblesInMemory is true.
+///
+/// \param ModuleFormat - If provided, uses the specific module format.
+///
+/// \param ErrAST - If non-null and parsing failed without any AST to return
+/// (e.g. because the PCH could not be loaded), this accepts the ASTUnit
+/// mainly to allow the caller to see the diagnostics.
+///
+/// \param VFS - A llvm::vfs::FileSystem to be used for all file accesses.
+/// Note that preamble is saved to a temporary directory on a RealFileSystem,
+/// so in order for it to be loaded correctly, VFS should have access to
+/// it(i.e., be an overlay over RealFileSystem). RealFileSystem will be used
+/// if \p VFS is nullptr.
+///
+// FIXME: Move OnlyLocalDecls, UseBumpAllocator to setters on the ASTUnit, we
+// shouldn't need to specify them at construction time.
+std::unique_ptr<ASTUnit> CreateASTUnitFromCommandLine(
+    const char **ArgBegin, const char **ArgEnd,
+    std::shared_ptr<PCHContainerOperations> PCHContainerOps,
+    std::shared_ptr<DiagnosticOptions> DiagOpts,
+    IntrusiveRefCntPtr<DiagnosticsEngine> Diags, StringRef ResourceFilesPath,
+    bool StorePreamblesInMemory = false,
+    StringRef PreambleStoragePath = StringRef(), bool OnlyLocalDecls = false,
+    CaptureDiagsKind CaptureDiagnostics = CaptureDiagsKind::None,
+    ArrayRef<ASTUnit::RemappedFile> RemappedFiles = {},
+    bool RemappedFilesKeepOriginalName = true,
+    unsigned PrecompilePreambleAfterNParses = 0,
+    TranslationUnitKind TUKind = TU_Complete,
+    bool CacheCodeCompletionResults = false,
+    bool IncludeBriefCommentsInCodeCompletion = false,
+    bool AllowPCHWithCompilerErrors = false,
+    SkipFunctionBodiesScope SkipFunctionBodies = SkipFunctionBodiesScope::None,
+    bool SingleFileParse = false, bool UserFilesAreVolatile = false,
+    bool ForSerialization = false, bool RetainExcludedConditionalBlocks = false,
+    std::optional<StringRef> ModuleFormat = std::nullopt,
+    std::unique_ptr<ASTUnit> *ErrAST = nullptr,
+    IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = nullptr);
+
+} // namespace clang
+
+#endif // LLVM_CLANG_DRIVER_CREATEASTUNITFROMARGS_H

clang/include/clang/Driver/CreateInvocationFromArgs.h

@@ -0,0 +1,76 @@
+//===--- CreateInvocationFromArgs.h - CompilerInvocation from Args --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Utility for creating a CompilerInvocation from command-line arguments, for
+// tools to use in preparation to parse a file.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_DRIVER_CREATEINVOCATIONFROMARGS_H
+#define LLVM_CLANG_DRIVER_CREATEINVOCATIONFROMARGS_H
+
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/LLVM.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace clang {
+
+class CompilerInvocation;
+class DiagnosticsEngine;
+
+/// Optional inputs to createInvocation.
+struct CreateInvocationOptions {
+  /// Receives diagnostics encountered while parsing command-line flags.
+  /// If not provided, these are printed to stderr.
+  IntrusiveRefCntPtr<DiagnosticsEngine> Diags = nullptr;
+  /// Used e.g. to probe for system headers locations.
+  /// If not provided, the real filesystem is used.
+  /// FIXME: the driver does perform some non-virtualized IO.
+  IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = nullptr;
+  /// Whether to attempt to produce a non-null (possibly incorrect) invocation
+  /// if any errors were encountered.
+  /// By default, always return null on errors.
+  bool RecoverOnError = false;
+  /// Allow the driver to probe the filesystem for PCH files.
+  /// This is used to replace -include with -include-pch in the cc1 args.
+  /// FIXME: ProbePrecompiled=true is a poor, historical default.
+  /// It misbehaves if the PCH file is from GCC, has the wrong version, etc.
+  bool ProbePrecompiled = false;
+  /// If set, the target is populated with the cc1 args produced by the driver.
+  /// This may be populated even if createInvocation returns nullptr.
+  std::vector<std::string> *CC1Args = nullptr;
+};
+
+/// Interpret clang arguments in preparation to parse a file.
+///
+/// This simulates a number of steps Clang takes when its driver is invoked:
+/// - choosing actions (e.g compile + link) to run
+/// - probing the system for settings like standard library locations
+/// - spawning a cc1 subprocess to compile code, with more explicit arguments
+/// - in the cc1 process, assembling those arguments into a CompilerInvocation
+///   which is used to configure the parser
+///
+/// This simulation is lossy, e.g. in some situations one driver run would
+/// result in multiple parses. (Multi-arch, CUDA, ...).
+/// This function tries to select a reasonable invocation that tools should use.
+///
+/// Args[0] should be the driver name, such as "clang" or "/usr/bin/g++".
+/// Absolute path is preferred - this affects searching for system headers.
+///
+/// May return nullptr if an invocation could not be determined.
+/// See CreateInvocationOptions::RecoverOnError to try harder!
+std::unique_ptr<CompilerInvocation>
+createInvocation(ArrayRef<const char *> Args,
+                 CreateInvocationOptions Opts = {});
+
+} // namespace clang
+
+#endif // LLVM_CLANG_DRIVER_CREATEINVOCATIONFROMARGS_H

clang/include/clang/Driver/Driver.h

@@ -406,11 +406,6 @@ class Driver {
                               SmallString<128> &CrashDiagDir);
 
 public:
-
-  /// Takes the path to a binary that's either in bin/ or lib/ and returns
-  /// the path to clang's resource directory.
-  static std::string GetResourcesPath(StringRef BinaryPath);
-
   Driver(StringRef ClangExecutable, StringRef TargetTriple,
          DiagnosticsEngine &Diags, std::string Title = "clang LLVM compiler",
          IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = nullptr);