[clang][modules-driver] Add initial support for driver-managed module builds

With -fmodules-driver enabled, the Clang driver provides native
support for:

- Named module imports defined in other source files on the command
  line
- Standard library imports (import std; and import std.compat;)
- Clang modules discovered via module map files

Regular translation units can import both Clang modules and C++20
named modules. Importing a Clang module into a C++20 named
module interface unit, or vice versa, is not supported by this patch.

Author: naveen-seth

clang/include/clang/Basic/DiagnosticDriverKinds.td

@@ -587,6 +587,22 @@ def remark_found_cxx20_module_usage : Remark<
 def remark_performing_driver_managed_module_build : Remark<
   "performing driver managed module build">,
   InGroup<ModulesDriver>;
+def remark_std_module_manifest_path : Remark<
+  "using std modules manifest: '%0'">, InGroup<ModulesDriver>;
+def err_failed_parse_modules_manifest_json: Error<
+  "failed to parse the std modules manifest">;
+def err_failed_depdendency_scan : Error<
+  "failed to perform dependency scan">;
+def remark_failed_dependency_scan_for_input : Remark<
+  "dependency scan failed for source input '%0'">,
+  InGroup<ModulesDriver>;
+def err_mod_graph_named_module_redefinition : Error<
+  "duplicate definitions of C++20 named module '%0' in '%1' and '%2'">;
+def  err_building_depdendency_graph : Error<
+  "failed to construct the module dependency graph">;
+def remark_printing_module_graph : Remark<
+  "printing module dependency graph">,
+  InGroup<ModulesDriver>;
 
 def warn_drv_delayed_template_parsing_after_cxx20 : Warning<
   "-fdelayed-template-parsing is deprecated after C++20">,

clang/include/clang/Driver/Driver.h

@@ -135,6 +135,10 @@ class Driver {
   /// interpretation.
   bool ModulesModeCXX20;
 
+  /// Set if the dirver should plan the compilation after scanning module
+  /// dependencies, using the scan results (set by -f(no-)modules-driver.)
+  bool DriverManagedModulesBuild;
+
   /// LTO mode selected via -f(no-)?lto(=.*)? options.
   LTOKind LTOMode;
 
@@ -796,13 +800,6 @@ class Driver {
   /// compilation based on which -f(no-)?lto(=.*)? option occurs last.
   void setLTOMode(const llvm::opt::ArgList &Args);
 
-  /// Scans the leading lines of the C++ source inputs to detect C++20 module
-  /// usage.
-  ///
-  /// \returns True if module usage is detected, false otherwise, or an error on
-  /// read failure.
-  llvm::ErrorOr<bool>
-  ScanInputsForCXX20ModulesUsage(const InputList &Inputs) const;
   /// Retrieves a ToolChain for a particular \p Target triple.
   ///
   /// Will cache ToolChains for the life of the driver object, and create them

clang/include/clang/Driver/Job.h

@@ -221,6 +221,8 @@ class Command {
 
   const char *getExecutable() const { return Executable; }
 
+  llvm::opt::ArgStringList &getArguments() { return Arguments; }
+
   const llvm::opt::ArgStringList &getArguments() const { return Arguments; }
 
   const std::vector<InputInfo> &getInputInfos() const { return InputInfoList; }
@@ -277,6 +279,7 @@ class JobList {
   /// Clear the job list.
   void clear();
 
+  list_type &getJobs() { return Jobs; }
   const list_type &getJobs() const { return Jobs; }
 
   bool empty() const { return Jobs.empty(); }

clang/include/clang/Driver/ModulesDriver.h

@@ -0,0 +1,52 @@
+//===- DependencyScanner.h - Module dependency discovery --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file defines the module dependency graph and dependency-scanning
+/// functionality.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_DRIVER_DEPENDENCYSCANNER_H
+#define LLVM_CLANG_DRIVER_DEPENDENCYSCANNER_H
+
+#include "clang/Driver/Types.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/VirtualFileSystem.h"
+
+namespace clang {
+class DiagnosticsEngine;
+namespace driver {
+class Compilation;
+} // namespace driver
+} // namespace clang
+
+namespace clang::driver::modules {
+
+using InputTy = std::pair<types::ID, const llvm::opt::Arg *>;
+
+using InputList = llvm::SmallVector<InputTy, 16>;
+
+/// Checks whether the -fmodules-driver feature should be implicitly enabled.
+///
+/// When -fmodules-driver is no longer experimental, it should be enabled by
+/// default iff both conditions are met:
+/// (1) there are two or more C++ source inputs; and
+/// (2) at least one input uses C++20 named modules.
+bool shouldEnableModulesDriver(const InputList &Inputs,
+                               llvm::vfs::FileSystem &VFS,
+                               DiagnosticsEngine &Diags);
+
+/// Appends the std and std.compat module inputs.
+bool ensureNamedModuleStdLibraryInputs(Compilation &C, InputList &Inputs);
+
+bool performDriverModuleBuild(Compilation &C, DiagnosticsEngine &Diags);
+
+} // namespace clang::driver::modules
+
+#endif

clang/include/clang/Driver/Options.td

@@ -3302,6 +3302,12 @@ def fmodules_driver : Flag<["-"], "fmodules-driver">,
 def fno_modules_driver : Flag<["-"], "fno-modules-driver">,
   Group<f_Group>, Visibility<[ClangOption]>,
   HelpText<"Disable support for driver managed module builds (experimental)">;
+def fimplicit_import_std : Flag<["-"], "fimplicit-import-std">,
+  Group<f_Group>, Visibility<[ClangOption]>,
+  HelpText<"Implicitly add the std module when discovered in driver managed module builds">;
+def fno_implicit_import_std : Flag<["-"], "fno-implicit-import-std">,
+  Group<f_Group>, Visibility<[ClangOption]>,
+  HelpText<"Don't implicitly add the std module when discovered in driver managed module builds">;
 
 def experimental_modules_reduced_bmi : Flag<["-"], "fexperimental-modules-reduced-bmi">,
   Group<f_Group>, Visibility<[ClangOption, CC1Option]>, Alias<fmodules_reduced_bmi>;

clang/lib/Driver/CMakeLists.txt

@@ -21,6 +21,7 @@ add_clang_library(clangDriver
   Driver.cpp
   DriverOptions.cpp
   Job.cpp
+  ModulesDriver.cpp
   Multilib.cpp
   MultilibBuilder.cpp
   OffloadBundler.cpp
@@ -98,6 +99,7 @@ add_clang_library(clangDriver
 
   LINK_LIBS
   clangBasic
+  clangDependencyScanning
   clangLex
   ${system_libs}
   )

clang/lib/Driver/Driver.cpp

@@ -60,6 +60,7 @@
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/InputInfo.h"
 #include "clang/Driver/Job.h"
+#include "clang/Driver/ModulesDriver.h"
 #include "clang/Driver/Options.h"
 #include "clang/Driver/Phases.h"
 #include "clang/Driver/SanitizerArgs.h"
@@ -1473,33 +1474,6 @@ bool Driver::loadDefaultConfigFiles(llvm::cl::ExpansionContext &ExpCtx) {
   return false;
 }
 
-static bool hasCXXModuleInputType(const Driver::InputList &Inputs) {
-  const auto IsTypeCXXModule = [](const auto &Input) -> bool {
-    const auto TypeID = Input.first;
-    return (TypeID == types::TY_CXXModule);
-  };
-  return llvm::any_of(Inputs, IsTypeCXXModule);
-}
-
-llvm::ErrorOr<bool>
-Driver::ScanInputsForCXX20ModulesUsage(const InputList &Inputs) const {
-  const auto CXXInputs = llvm::make_filter_range(
-      Inputs, [](const auto &Input) { return types::isCXX(Input.first); });
-  for (const auto &Input : CXXInputs) {
-    StringRef Filename = Input.second->getSpelling();
-    auto ErrOrBuffer = VFS->getBufferForFile(Filename);
-    if (!ErrOrBuffer)
-      return ErrOrBuffer.getError();
-    const auto Buffer = std::move(*ErrOrBuffer);
-
-    if (scanInputForCXX20ModulesUsage(Buffer->getBuffer())) {
-      Diags.Report(diag::remark_found_cxx20_module_usage) << Filename;
-      return true;
-    }
-  }
-  return false;
-}
-
 Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
   llvm::PrettyStackTraceString CrashInfo("Compilation construction");
 
@@ -1853,6 +1827,18 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
     }
   }
 
+  if (C->getArgs().hasFlag(options::OPT_fmodules_driver,
+                           options::OPT_fno_modules_driver, false)) {
+    // The detection logic for this is kept here only for diagnostics until
+    // is enabled by default.
+    modules::shouldEnableModulesDriver(Inputs, getVFS(), Diags);
+    Diags.Report(diag::remark_performing_driver_managed_module_build);
+    if (C->getArgs().hasFlag(options::OPT_fimplicit_import_std,
+                             options::OPT_fno_implicit_import_std, true)) {
+      modules::ensureNamedModuleStdLibraryInputs(*C, Inputs);
+    }
+  }
+
   // Populate the tool chains for the offloading devices, if any.
   CreateOffloadingDeviceToolChains(*C, Inputs);
 
@@ -1863,26 +1849,6 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
   else
     BuildActions(*C, C->getArgs(), Inputs, C->getActions());
 
-  if (C->getArgs().hasFlag(options::OPT_fmodules_driver,
-                           options::OPT_fno_modules_driver, false)) {
-    // TODO: When -fmodules-driver is no longer experimental, it should be
-    // enabled by default only if both conditions are met: (1) there are two or
-    // more C++ source inputs; and (2) at least one input uses C++20 named
-    // modules.
-    // The detection logic for this is kept here only for diagnostics until
-    // is enabled by default.
-    bool UsesCXXModules = hasCXXModuleInputType(Inputs);
-    if (!UsesCXXModules) {
-      const auto ErrOrScanResult = ScanInputsForCXX20ModulesUsage(Inputs);
-      if (!ErrOrScanResult) {
-        Diags.Report(diag::err_cannot_open_file)
-            << ErrOrScanResult.getError().message();
-      }
-      UsesCXXModules = *ErrOrScanResult;
-    }
-    Diags.Report(diag::remark_performing_driver_managed_module_build);
-  }
-
   if (CCCPrintPhases) {
     PrintActions(*C);
     return C;
@@ -4236,10 +4202,20 @@ void Driver::handleArguments(Compilation &C, DerivedArgList &Args,
     YcArg = nullptr;
   }
 
-  if (Args.hasArgNoClaim(options::OPT_fmodules_driver))
-    // TODO: Check against all incompatible -fmodules-driver arguments
-    if (!ModulesModeCXX20 && !Args.hasArgNoClaim(options::OPT_fmodules))
-      Args.eraseArg(options::OPT_fmodules_driver);
+  if (Args.hasArgNoClaim(options::OPT_fmodules_driver)) {
+    // HACK: This should be only added for the Standard library jobs, explicitly
+    // created by the modules driver.
+    MakeInputArg(Args, getOpts(),
+                 Args.MakeArgString("-Wno-reserved-module-identifier"));
+    if (Args.hasArg(options::OPT_fmodules)) {
+      Args.eraseArg(options::OPT_fmodules);
+      Arg *Arg = Args.MakeSeparateArg(
+          nullptr, getOpts().getOption(options::OPT_fimplicit_modules),
+          Args.MakeArgString(("-fimplicit-modules")));
+      Arg->claim();
+      Args.append(Arg);
+    }
+  }
 
   Arg *FinalPhaseArg;
   phases::ID FinalPhase = getFinalPhase(Args, &FinalPhaseArg);
@@ -5427,6 +5403,12 @@ void Driver::BuildJobs(Compilation &C) const {
       }
     }
   }
+  if (C.getArgs().hasFlag(options::OPT_fmodules_driver,
+                          options::OPT_fno_modules_driver, false)) {
+    auto Success = modules::performDriverModuleBuild(C, C.getDriver().Diags);
+    if (!Success)
+      return;
+  }
 }
 
 namespace {