[clang][modules-driver] Add dependency scan and dependency graph

This patch is part of a series to support driver-managed module
builds. It adds support for the discovery of module dependencies from
within the driver and for generation of the module dependency graph.

Source inputs provided by the command-line can import modules defined
in the standard library module manifest, which are then scanned on
demand.

The dependency scan and graph support both Clang modules and C++ named
modules. The generated dependency graph can be output in Graphviz DOT
format as a remark.

Author: naveen-seth

clang/include/clang/Basic/DiagnosticDriverKinds.td

@@ -587,6 +587,24 @@ def remark_found_cxx20_module_usage : Remark<
 def remark_performing_driver_managed_module_build : Remark<
   "performing driver managed module build">,
   InGroup<ModulesDriver>;
+def remark_modules_manifest_not_found : Remark<
+  "standard modules manifest file not found; import of standard library "
+  "modules not supported">,
+  InGroup<ModulesDriver>;
+def remark_using_modules_manifest : Remark<
+  "using standard modules manifest file '%0'">,
+   InGroup<ModulesDriver>;
+def err_modules_manifest_failed_parse : Error<
+  "failure while parsing standard modules manifest: '%0'">;
+def err_failed_dependency_scan : Error<
+  "failed to perform dependency scan">;
+def err_mod_graph_named_module_redefinition : Error<
+  "duplicate definitions of C++20 named module '%0' in '%1' and '%2'">;
+def err_building_dependency_graph : Error<
+  "failed to construct the module dependency graph">;
+def remark_printing_module_graph : Remark<
+  "printing module dependency graph">,
+  InGroup<ModulesDriver>;
 
 def warn_drv_delayed_template_parsing_after_cxx20 : Warning<
   "-fdelayed-template-parsing is deprecated after C++20">,

clang/include/clang/Driver/Driver.h

@@ -512,9 +512,6 @@ class Driver {
 
   /// BuildActions - Construct the list of actions to perform for the
   /// given arguments, which are only done for a single architecture.
-  /// If the compilation is an explicit module build, delegates to
-  /// BuildDriverManagedModuleBuildActions. Otherwise, BuildDefaultActions is
-  /// used.
   ///
   /// \param C - The compilation that is being built.
   /// \param Args - The input arguments.
@@ -799,35 +796,6 @@ class Driver {
   /// compilation based on which -f(no-)?lto(=.*)? option occurs last.
   void setLTOMode(const llvm::opt::ArgList &Args);
 
-  /// BuildDefaultActions - Constructs the list of actions to perform
-  /// for the provided arguments, which are only done for a single architecture.
-  ///
-  /// \param C - The compilation that is being built.
-  /// \param Args - The input arguments.
-  /// \param Actions - The list to store the resulting actions onto.
-  void BuildDefaultActions(Compilation &C, llvm::opt::DerivedArgList &Args,
-                           const InputList &Inputs, ActionList &Actions) const;
-
-  /// BuildDriverManagedModuleBuildActions - Performs a dependency
-  /// scan and constructs the list of actions to perform for dependency order
-  /// and the provided arguments. This is only done for a single a architecture.
-  ///
-  /// \param C - The compilation that is being built.
-  /// \param Args - The input arguments.
-  /// \param Actions - The list to store the resulting actions onto.
-  void BuildDriverManagedModuleBuildActions(Compilation &C,
-                                            llvm::opt::DerivedArgList &Args,
-                                            const InputList &Inputs,
-                                            ActionList &Actions) const;
-
-  /// Scans the leading lines of the C++ source inputs to detect C++20 module
-  /// usage.
-  ///
-  /// \returns True if module usage is detected, false otherwise, or an error on
-  /// read failure.
-  llvm::ErrorOr<bool>
-  ScanInputsForCXX20ModulesUsage(const InputList &Inputs) const;
-
   /// Retrieves a ToolChain for a particular \p Target triple.
   ///
   /// Will cache ToolChains for the life of the driver object, and create them

clang/include/clang/Driver/Job.h

@@ -221,6 +221,8 @@ class Command {
 
   const char *getExecutable() const { return Executable; }
 
+  llvm::opt::ArgStringList &getArguments() { return Arguments; }
+
   const llvm::opt::ArgStringList &getArguments() const { return Arguments; }
 
   const std::vector<InputInfo> &getInputInfos() const { return InputInfoList; }
@@ -279,6 +281,9 @@ class JobList {
 
   const list_type &getJobs() const { return Jobs; }
 
+  // Returns and transfers ownership of all jobs, leaving this list empty.
+  list_type takeJobs();
+
   bool empty() const { return Jobs.empty(); }
   size_type size() const { return Jobs.size(); }
   iterator begin() { return Jobs.begin(); }

clang/include/clang/Driver/ModulesDriver.h

@@ -0,0 +1,112 @@
+//===- ModulesDriver.h - Driver managed module builds --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file defines functionality to support driver managed builds for
+/// compilations which use Clang modules or standard C++20 named modules.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_DRIVER_MODULESDRIVER_H
+#define LLVM_CLANG_DRIVER_MODULESDRIVER_H
+
+#include "clang/Driver/Types.h"
+#include "llvm/Support/Error.h"
+
+namespace llvm {
+namespace vfs {
+class FileSystem;
+} // namespace vfs
+namespace opt {
+class Arg;
+} // namespace opt
+} // namespace llvm
+
+namespace clang {
+class DiagnosticsEngine;
+namespace driver {
+class Compilation;
+} // namespace driver
+} // namespace clang
+
+namespace clang::driver::modules {
+
+/// A list of inputs and their types for the given arguments.
+/// Identical to Driver::InputTy.
+using InputTy = std::pair<types::ID, const llvm::opt::Arg *>;
+
+/// A list of inputs and their types for the given arguments.
+/// Identical to Driver::InputList.
+using InputList = llvm::SmallVector<InputTy, 16>;
+
+/// Checks whether the -fmodules-driver feature should be implicitly enabled.
+///
+/// The modules driver should be enabled if both:
+/// 1) the compilation has more than two C++ source inputs; and
+/// 2) any C++ source inputs uses C++20 named modules.
+///
+/// \param Inputs The input list for the compilation being built.
+/// \param VFS The virtual file system to use for all reads.
+/// \param Diags The diagnostics engine used for emitting remarks only.
+///
+/// \returns True if the modules driver should be enabled, false otherwise,
+/// or a llvm::FileError on failure to read a source input.
+llvm::Expected<bool> shouldUseModulesDriver(const InputList &Inputs,
+                                            llvm::vfs::FileSystem &VFS,
+                                            DiagnosticsEngine &Diags);
+
+/// The parsed Standard library module manifest.
+struct StdModuleManifest {
+  struct LocalModuleArgs {
+    std::vector<std::string> SystemIncludeDirs;
+  };
+
+  struct Module {
+    bool IsStdlib = false;
+    std::string LogicalName;
+    std::string SourcePath;
+    std::optional<LocalModuleArgs> LocalArgs;
+  };
+
+  std::vector<Module> ModuleEntries;
+};
+
+/// Reads the Standard library module manifest from the specified path.
+///
+/// All source file paths in the returned manifest are made absolute.
+///
+/// \param StdModuleManifestPath Path to the manifest file.
+/// \param VFS The llvm::vfs::FileSystem to be used for all file accesses.
+///
+/// \returns The parsed manifest on success, a llvm::FileError on failure to
+/// read the manifest, or a llvm::json::ParseError on failure to parse it.
+llvm::Expected<StdModuleManifest>
+readStdModuleManifest(llvm::StringRef StdModuleManifestPath,
+                      llvm::vfs::FileSystem &VFS);
+
+/// Constructs compilation inputs for each module listed in the provided
+/// Standard library module manifest.
+///
+/// \param Manifest The standard modules manifest
+/// \param C The compilation being built.
+/// \param Inputs The input list to which the corresponding input entries are
+/// appended.
+void buildStdModuleManifestInputs(const StdModuleManifest &Manifest,
+                                  Compilation &C, InputList &Inputs);
+
+/// Reorders and builds compilation jobs based on discovered module
+/// dependencies.
+///
+/// \param C The compilation being built.
+/// \param Manifest The standard modules manifest
+void planDriverManagedModuleCompilation(Compilation &C,
+                                        const StdModuleManifest &Manifest);
+
+} // namespace clang::driver::modules
+
+#endif

clang/include/clang/Lex/DependencyDirectivesScanner.h

@@ -140,7 +140,7 @@ void printDependencyDirectivesAsSource(
 /// \param Source The input source buffer.
 ///
 /// \returns true if any C++20 named modules related directive was found.
-bool scanInputForCXX20ModulesUsage(StringRef Source);
+bool scanInputForCXXNamedModulesUsage(StringRef Source);
 
 /// Functor that returns the dependency directives for a given file.
 class DependencyDirectivesGetter {

clang/lib/Driver/CMakeLists.txt

@@ -21,6 +21,7 @@ add_clang_library(clangDriver
   Driver.cpp
   DriverOptions.cpp
   Job.cpp
+  ModulesDriver.cpp
   Multilib.cpp
   MultilibBuilder.cpp
   OffloadBundler.cpp
@@ -99,5 +100,6 @@ add_clang_library(clangDriver
   LINK_LIBS
   clangBasic
   clangLex
+  clangDependencyScanning
   ${system_libs}
   )

clang/lib/Driver/Driver.cpp

@@ -60,6 +60,7 @@
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/InputInfo.h"
 #include "clang/Driver/Job.h"
+#include "clang/Driver/ModulesDriver.h"
 #include "clang/Driver/Options.h"
 #include "clang/Driver/Phases.h"
 #include "clang/Driver/SanitizerArgs.h"
@@ -87,6 +88,7 @@
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/JSON.h"
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/PrettyStackTrace.h"
@@ -1829,6 +1831,53 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
   // Populate the tool chains for the offloading devices, if any.
   CreateOffloadingDeviceToolChains(*C, Inputs);
 
+  modules::StdModuleManifest Manifest;
+  if (C->getArgs().hasFlag(options::OPT_fmodules_driver,
+                           options::OPT_fno_modules_driver, false)) {
+    Diags.Report(diag::remark_performing_driver_managed_module_build);
+    // TODO: Once -fmodules-driver is no longer experimental, move
+    // TODO: The detection logic to implicitly enable -fmodules-driver is kept
+    // here only for diagnostics until the feature is no longer experimental.
+    auto EnableOrErr = modules::shouldUseModulesDriver(Inputs, getVFS(), Diags);
+    if (!EnableOrErr) {
+      llvm::handleAllErrors(
+          EnableOrErr.takeError(), [&](const llvm::FileError &Err) {
+            Diags.Report(diag::err_cannot_open_file)
+                << Err.getFileName() << Err.messageWithoutFileInfo();
+          });
+      return C;
+    }
+
+    // Read the standard modules manifest, and if available, add all discovered
+    // modules to the compilation. Compilation jobs for modules discovered from
+    // the manifest, which are not imported by any other source input, are
+    // pruned later.
+    const auto StdModuleManifestPath =
+        GetStdModuleManifestPath(*C, C->getDefaultToolChain());
+    if (StdModuleManifestPath == "<NOT PRESENT>") {
+      Diags.Report(diag::remark_modules_manifest_not_found);
+    } else {
+      Diags.Report(diag::remark_using_modules_manifest)
+          << StdModuleManifestPath;
+      if (auto ManifestOrErr =
+              modules::readStdModuleManifest(StdModuleManifestPath, getVFS())) {
+        Manifest = std::move(*ManifestOrErr);
+      } else {
+        llvm::handleAllErrors(
+            ManifestOrErr.takeError(),
+            [&](llvm::json::ParseError &Err) {
+              Diags.Report(diag::err_modules_manifest_failed_parse)
+                  << Err.message();
+            },
+            [&](llvm::FileError &Err) {
+              Diags.Report(diag::err_cannot_open_file)
+                  << Err.getFileName() << Err.messageWithoutFileInfo();
+            });
+      }
+    }
+    modules::buildStdModuleManifestInputs(Manifest, *C, Inputs);
+  }
+
   // Construct the list of abstract actions to perform for this compilation. On
   // MachO targets this uses the driver-driver and universal actions.
   if (TC.getTriple().isOSBinFormatMachO())
@@ -1843,6 +1892,11 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
 
   BuildJobs(*C);
 
+  if (C->getArgs().hasFlag(options::OPT_fmodules_driver,
+                           options::OPT_fno_modules_driver, false)) {
+    modules::planDriverManagedModuleCompilation(*C, Manifest);
+  }
+
   return C;
 }
 
@@ -4320,33 +4374,6 @@ void Driver::handleArguments(Compilation &C, DerivedArgList &Args,
   }
 }
 
-static bool hasCXXModuleInputType(const Driver::InputList &Inputs) {
-  const auto IsTypeCXXModule = [](const auto &Input) -> bool {
-    const auto TypeID = Input.first;
-    return (TypeID == types::TY_CXXModule);
-  };
-  return llvm::any_of(Inputs, IsTypeCXXModule);
-}
-
-llvm::ErrorOr<bool>
-Driver::ScanInputsForCXX20ModulesUsage(const InputList &Inputs) const {
-  const auto CXXInputs = llvm::make_filter_range(
-      Inputs, [](const auto &Input) { return types::isCXX(Input.first); });
-  for (const auto &Input : CXXInputs) {
-    StringRef Filename = Input.second->getSpelling();
-    auto ErrOrBuffer = VFS->getBufferForFile(Filename);
-    if (!ErrOrBuffer)
-      return ErrOrBuffer.getError();
-    const auto Buffer = std::move(*ErrOrBuffer);
-
-    if (scanInputForCXX20ModulesUsage(Buffer->getBuffer())) {
-      Diags.Report(diag::remark_found_cxx20_module_usage) << Filename;
-      return true;
-    }
-  }
-  return false;
-}
-
 void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
                           const InputList &Inputs, ActionList &Actions) const {
   llvm::PrettyStackTraceString CrashInfo("Building compilation actions");
@@ -4358,33 +4385,6 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
 
   handleArguments(C, Args, Inputs, Actions);
 
-  if (Args.hasFlag(options::OPT_fmodules_driver,
-                   options::OPT_fno_modules_driver, false)) {
-    // TODO: Move the logic for implicitly enabling explicit-module-builds out
-    // of -fmodules-driver once it is no longer experimental.
-    // Currently, this serves diagnostic purposes only.
-    bool UsesCXXModules = hasCXXModuleInputType(Inputs);
-    if (!UsesCXXModules) {
-      const auto ErrOrScanResult = ScanInputsForCXX20ModulesUsage(Inputs);
-      if (!ErrOrScanResult) {
-        Diags.Report(diag::err_cannot_open_file)
-            << ErrOrScanResult.getError().message();
-        return;
-      }
-      UsesCXXModules = *ErrOrScanResult;
-    }
-    if (UsesCXXModules || Args.hasArg(options::OPT_fmodules))
-      BuildDriverManagedModuleBuildActions(C, Args, Inputs, Actions);
-    return;
-  }
-
-  BuildDefaultActions(C, Args, Inputs, Actions);
-}
-
-void Driver::BuildDefaultActions(Compilation &C, DerivedArgList &Args,
-                                 const InputList &Inputs,
-                                 ActionList &Actions) const {
-
   bool UseNewOffloadingDriver =
       C.isOffloadingHostKind(Action::OFK_OpenMP) ||
       C.isOffloadingHostKind(Action::OFK_SYCL) ||
@@ -4680,12 +4680,6 @@ void Driver::BuildDefaultActions(Compilation &C, DerivedArgList &Args,
   Args.ClaimAllArgs(options::OPT_cl_ignored_Group);
 }
 
-void Driver::BuildDriverManagedModuleBuildActions(
-    Compilation &C, llvm::opt::DerivedArgList &Args, const InputList &Inputs,
-    ActionList &Actions) const {
-  Diags.Report(diag::remark_performing_driver_managed_module_build);
-}
-
 /// Returns the canonical name for the offloading architecture when using a HIP
 /// or CUDA architecture.
 static StringRef getCanonicalArchString(Compilation &C,

clang/lib/Driver/Job.cpp

@@ -453,3 +453,5 @@ void JobList::Print(raw_ostream &OS, const char *Terminator, bool Quote,
 }
 
 void JobList::clear() { Jobs.clear(); }
+
+JobList::list_type JobList::takeJobs() { return std::exchange(Jobs, {}); }