[clang][modules-driver] Add scanner to detect C++20 module presence

This PR is part of a series to natively support C++20 module usage
from the Clang driver (without requiring an external build system).
This introduces a new scanner that detects C++20 module usage in
source files without using the preprocessor or lexer.

For now, it is enabled only with the `-fmodules-driver` flag and
serves solely diagnostic purposes. In the future, the scanner will be
enabled for any (modules-driver compatible) compilation with two or
more inputs, and will help the driver determine whether to implicitly
enable the modules driver.

Since the scanner adds very little overhead, we are also exploring
enabling it for compilations with only a single input.
This approach could allow us to detect `import std` usage in a
single-file compilation, which would then activate the modules driver.
For performance measurements on this, see
https://github.com/naveen-seth/llvm-dev-cxx-modules-check-benchmark.

RFC:
https://discourse.llvm.org/t/rfc-modules-support-simple-c-20-modules-use-from-the-clang-driver-without-a-build-system

Author: naveen-seth

clang/include/clang/Basic/DiagnosticDriverKinds.td

@@ -577,6 +577,19 @@ def err_drv_reduced_module_output_overrided : Warning<
   "please consider use '-fmodule-output=' to specify the output file for reduced BMI explicitly">,
   InGroup<DiagGroup<"reduced-bmi-output-overrided">>;
 
+def remark_fmodules_driver_enabled : Remark<
+  "support for explicit module builds enabled (experimental)">,
+  InGroup<ModulesDriver>;
+def remark_found_cxx20_module_usage : Remark<
+  "found C++20 module usage in file '%0'">,
+  InGroup<ModulesDriver>;
+def remark_performing_explicit_module_build : Remark<
+  "performing explicit module build">,
+  InGroup<ModulesDriver>;
+def warn_modules_driver_unsupported_standard : Warning<
+  "'-fmodules-driver' is not supported before C++20">,
+  InGroup<ModulesDriver>;
+
 def warn_drv_delayed_template_parsing_after_cxx20 : Warning<
   "-fdelayed-template-parsing is deprecated after C++20">,
   InGroup<DiagGroup<"delayed-template-parsing-in-cxx20">>;

clang/include/clang/Basic/DiagnosticGroups.td

@@ -625,6 +625,7 @@ def ModuleConflict : DiagGroup<"module-conflict">;
 def ModuleFileExtension : DiagGroup<"module-file-extension">;
 def ModuleIncludeDirectiveTranslation : DiagGroup<"module-include-translation">;
 def ModuleMap : DiagGroup<"module-map">;
+def ModulesDriver : DiagGroup<"modules-driver">;
 def RoundTripCC1Args : DiagGroup<"round-trip-cc1-args">;
 def NewlineEOF : DiagGroup<"newline-eof">;
 def Nullability : DiagGroup<"nullability">;

clang/include/clang/Driver/Driver.h

@@ -505,6 +505,8 @@ class Driver {
 
   /// BuildActions - Construct the list of actions to perform for the
   /// given arguments, which are only done for a single architecture.
+  /// If the compilation is an explicit module build, delegates to
+  /// BuildExplicitModuleBuildActions. Otherwise, uses BuildDefaultActions.
   ///
   /// \param C - The compilation that is being built.
   /// \param Args - The input arguments.
@@ -790,6 +792,35 @@ class Driver {
   /// compilation based on which -f(no-)?lto(=.*)? option occurs last.
   void setLTOMode(const llvm::opt::ArgList &Args);
 
+  /// BuildDefaultActions - Constructs the list of actions to perform
+  /// for the provided arguments, which are only done for a single architecture.
+  ///
+  /// \param C - The compilation that is being built.
+  /// \param Args - The input arguments.
+  /// \param Actions - The list to store the resulting actions onto.
+  void BuildDefaultActions(Compilation &C, llvm::opt::DerivedArgList &Args,
+                           const InputList &Inputs, ActionList &Actions) const;
+
+  /// BuildExplicitModuleBuildActions - Performs a dependency scan and
+  /// constructs the list of actions to perform for dependency order and
+  /// the provided arguments. This is only done for a single a architecture.
+  ///
+  /// \param C - The compilation that is being built.
+  /// \param Args - The input arguments.
+  /// \param Actions - The list to store the resulting actions onto.
+  void BuildExplicitModuleBuildActions(Compilation &C,
+                                       llvm::opt::DerivedArgList &Args,
+                                       const InputList &Inputs,
+                                       ActionList &Actions) const;
+
+  /// Scans the leading lines of the C++ source inputs to detect C++20 module
+  /// usage.
+  ///
+  /// \returns True if module usage is detected, false otherwise, or an error on
+  /// read failure.
+  llvm::ErrorOr<bool>
+  ScanInputsForCXXModuleUsage(const InputList &Inputs) const;
+
   /// Retrieves a ToolChain for a particular \p Target triple.
   ///
   /// Will cache ToolChains for the life of the driver object, and create them

clang/include/clang/Driver/Options.td

@@ -3240,6 +3240,13 @@ def modules_reduced_bmi : Flag<["-"], "fmodules-reduced-bmi">,
   HelpText<"Generate the reduced BMI">,
   MarshallingInfoFlag<FrontendOpts<"GenReducedBMI">>;
 
+def fmodules_driver : Flag<["-"], "fmodules-driver">,
+  Group<f_Group>, Visibility<[ClangOption]>,
+  HelpText<"Enable support for explicit module builds from the driver (experimental)">;
+def fno_modules_driver : Flag<["-"], "fno-modules-driver">,
+  Group<f_Group>, Visibility<[ClangOption]>,
+  HelpText<"Disable support for explicit module builds from the driver (experimental)">;
+
 def experimental_modules_reduced_bmi : Flag<["-"], "fexperimental-modules-reduced-bmi">,
   Group<f_Group>, Visibility<[ClangOption, CC1Option]>, Alias<modules_reduced_bmi>;
 

clang/lib/Driver/Driver.cpp

@@ -53,6 +53,7 @@
 #include "ToolChains/WebAssembly.h"
 #include "ToolChains/XCore.h"
 #include "ToolChains/ZOS.h"
+#include "clang/Basic/CharInfo.h"
 #include "clang/Basic/DiagnosticDriver.h"
 #include "clang/Basic/TargetID.h"
 #include "clang/Basic/Version.h"
@@ -4285,6 +4286,13 @@ void Driver::handleArguments(Compilation &C, DerivedArgList &Args,
     YcArg = nullptr;
   }
 
+  if (Args.hasArgNoClaim(options::OPT_fmodules_driver))
+    // TODO: Check against all incompatible -fmodules-driver arguments
+    if (!ModulesModeCXX20) {
+      Diag(diag::warn_modules_driver_unsupported_standard);
+      Args.eraseArg(options::OPT_fmodules_driver);
+    }
+
   Arg *FinalPhaseArg;
   phases::ID FinalPhase = getFinalPhase(Args, &FinalPhaseArg);
 
@@ -4403,6 +4411,177 @@ void Driver::handleArguments(Compilation &C, DerivedArgList &Args,
   }
 }
 
+namespace {
+static void skipWhitespace(const char *&Ptr) {
+  while (isWhitespace(*Ptr))
+    ++Ptr;
+}
+
+// Returns the length of EOL, either 0 (no end-of-line), 1 (\n) or 2 (\r\n).
+static unsigned isEOL(const char *Ptr) {
+  if (*Ptr == '\0')
+    return 0;
+  if (*(Ptr + 1) != '\0' && isVerticalWhitespace(Ptr[0]) &&
+      isVerticalWhitespace(Ptr[1]) && Ptr[0] != Ptr[1])
+    return 2;
+  return !!isVerticalWhitespace(Ptr[0]);
+}
+
+static void skipLine(const char *&Ptr) {
+  for (;;) {
+    char LastNonWhitespace = ' ';
+    while (!isVerticalWhitespace(*Ptr) && *Ptr != '\0') {
+      if (!isHorizontalWhitespace(*Ptr))
+        LastNonWhitespace = *Ptr;
+      ++Ptr;
+    }
+
+    const unsigned Len = isEOL(Ptr);
+    if (!Len)
+      return;
+
+    Ptr += Len;
+    if (LastNonWhitespace != '\\')
+      break;
+  }
+}
+
+// Returns the length of a line splice sequence (including trailing
+// whitespace), or 0 if no line splice is found.
+static unsigned isLineSplice(const char *Start) {
+  if (*Start != '\\')
+    return 0;
+
+  const char *Ptr = Start + 1;
+  while (isHorizontalWhitespace(*Ptr))
+    ++Ptr;
+
+  if (unsigned Len = isEOL(Ptr))
+    return Ptr - Start + Len;
+  return 0;
+}
+
+static bool trySkipLineSplice(const char *&Ptr) {
+  if (unsigned Len = isLineSplice(Ptr); Len) {
+    Ptr += Len;
+    return true;
+  }
+  return false;
+}
+
+static bool trySkipDirective(const char *&Ptr) {
+  if (*Ptr != '#')
+    return false;
+
+  ++Ptr;
+  skipLine(Ptr);
+  return true;
+}
+
+static bool trySkipLineComment(const char *&Ptr) {
+  if (Ptr[0] != '/' || Ptr[1] != '/')
+    return false;
+
+  Ptr += 2;
+  skipLine(Ptr);
+  return true;
+}
+
+static bool trySkipBlockComment(const char *&Ptr) {
+  if (Ptr[0] != '/' || Ptr[1] != '*')
+    return false;
+
+  Ptr += 2;
+  while (*Ptr != '\0') {
+    if (Ptr[0] == '*' && Ptr[1] == '/') {
+      Ptr += 2; // '*/'
+      return true;
+    }
+    ++Ptr;
+  }
+  return true;
+}
+
+static bool trySkipComment(const char *&Ptr) {
+  return trySkipLineComment(Ptr) || trySkipBlockComment(Ptr);
+}
+
+// Skipps over comments and (non-module) directives
+static void skipToRelevantCXXModuleText(const char *&Ptr) {
+  while (*Ptr != '\0') {
+    skipWhitespace(Ptr);
+    if (trySkipComment(Ptr) || trySkipDirective(Ptr) || trySkipLineSplice(Ptr))
+      continue;
+    break; // Found relevant text!
+  }
+}
+
+static bool scanBufferForCXXModuleUsage(const llvm::MemoryBuffer &Buffer) {
+  const char *Ptr = Buffer.getBufferStart();
+  skipToRelevantCXXModuleText(Ptr);
+
+  // Check if buffer has enough bytes left to check for the module-related
+  // declaration fragment we want to check without making potentially
+  // memory-mapped buffer load unnecessary pages.
+  constexpr int MinKeywordLength = 6;
+  const char *Begin = Ptr;
+  for (int i = 0; i < MinKeywordLength; ++i) {
+    if (*Ptr == '\0')
+      return false;
+    ++Ptr;
+  }
+  StringRef Text(Begin, MinKeywordLength);
+
+  const bool IsGlobalModule = Text.starts_with("module");
+  if (!IsGlobalModule && !Text.starts_with("import") &&
+      !Text.starts_with("export"))
+    return false;
+
+  // Ensure the keyword has a proper ending and isn't part of a identifier
+  // or namespace. For this we might have to skip comments and line
+  // continuations.
+  while (*Ptr != '\0') {
+    if (isWhitespace(*Ptr) || (IsGlobalModule && *Ptr == ';'))
+      return true;
+    if (trySkipBlockComment(Ptr) || trySkipLineSplice(Ptr))
+      continue;
+    return false;
+  }
+
+  return false;
+}
+
+static bool hasCXXModuleInputType(const Driver::InputList &Inputs) {
+  const auto IsTypeCXXModule = [](const auto &Input) -> bool {
+    const auto TypeID = Input.first;
+    return (TypeID == types::TY_CXXModule);
+  };
+  return llvm::any_of(Inputs, IsTypeCXXModule);
+}
+
+} // anonymous namespace
+
+llvm::ErrorOr<bool>
+Driver::ScanInputsForCXXModuleUsage(const InputList &Inputs) const {
+  const auto CXXInputs = llvm::make_filter_range(
+      Inputs, [](const auto &Input) { return types::isCXX(Input.first); });
+
+  for (const auto &Input : CXXInputs) {
+    StringRef Filename = Input.second->getSpelling();
+    auto ErrOrBuffer = VFS->getBufferForFile(Filename);
+    if (!ErrOrBuffer)
+      return ErrOrBuffer.getError();
+    const auto Buffer = std::move(*ErrOrBuffer);
+
+    if (scanBufferForCXXModuleUsage(*Buffer)) {
+      Diags.Report(diag::remark_found_cxx20_module_usage) << Filename;
+      return true;
+    }
+  }
+
+  return false;
+}
+
 void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
                           const InputList &Inputs, ActionList &Actions) const {
   llvm::PrettyStackTraceString CrashInfo("Building compilation actions");
@@ -4414,6 +4593,34 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
 
   handleArguments(C, Args, Inputs, Actions);
 
+  if (Args.hasFlag(options::OPT_fmodules_driver,
+                   options::OPT_fno_modules_driver, false)) {
+    Diags.Report(diag::remark_fmodules_driver_enabled);
+    // TODO: Move the logic for implicitly enabling explicit-module-builds out
+    // of -fmodules-driver once it is no longer experimental.
+    // Currently, this serves diagnostic purposes only.
+    bool UsesCXXModules = hasCXXModuleInputType(Inputs);
+    if (!UsesCXXModules) {
+      const auto ErrOrScanResult = ScanInputsForCXXModuleUsage(Inputs);
+      if (!ErrOrScanResult) {
+        Diags.Report(diag::err_cannot_open_file)
+            << ErrOrScanResult.getError().message();
+        return;
+      }
+      UsesCXXModules = *ErrOrScanResult;
+    }
+    if (UsesCXXModules)
+      BuildExplicitModuleBuildActions(C, Args, Inputs, Actions);
+    return;
+  }
+
+  Driver::BuildDefaultActions(C, Args, Inputs, Actions);
+}
+
+void Driver::BuildDefaultActions(Compilation &C, DerivedArgList &Args,
+                                 const InputList &Inputs,
+                                 ActionList &Actions) const {
+
   bool UseNewOffloadingDriver =
       C.isOffloadingHostKind(Action::OFK_OpenMP) ||
       C.isOffloadingHostKind(Action::OFK_SYCL) ||
@@ -4693,6 +4900,14 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
   Args.ClaimAllArgs(options::OPT_cl_ignored_Group);
 }
 
+void Driver::BuildExplicitModuleBuildActions(Compilation &C,
+                                             llvm::opt::DerivedArgList &Args,
+                                             const InputList &Inputs,
+                                             ActionList &Actions) const {
+  Diags.Report(diag::remark_performing_explicit_module_build);
+  return;
+}
+
 /// Returns the canonical name for the offloading architecture when using a HIP
 /// or CUDA architecture.
 static StringRef getCanonicalArchString(Compilation &C,