[clang][clang-scan-deps] Add LangOptions::AllowLiteralDigitSeparator and enable it by default for CPlusPlus14 or C23 and set it explicitly by Scanner

Author: tsfn

clang/include/clang/Basic/LangOptions.def

@@ -438,7 +438,7 @@ LANGOPT(CXXAssumptions, 1, 1, NotCompatible, "Enable or disable codegen and comp
 
 LANGOPT(RawStringLiterals, 1, 1, NotCompatible, "Enable or disable raw string literals")
 
-LANGOPT(ScanDeps, 1, 0, NotCompatible, "True if we are scanning by DependencyDirectivesScanner")
+LANGOPT(AllowLiteralDigitSeparator, 1, 0, NotCompatible, "Allow literal digit seperator in source")
 
 ENUM_LANGOPT(StrictFlexArraysLevel, StrictFlexArraysLevelKind, 2,
              StrictFlexArraysLevelKind::Default, NotCompatible,

clang/include/clang/Basic/LangStandard.h

@@ -140,6 +140,9 @@ struct LangStandard {
     return isCPlusPlus11() || (!isCPlusPlus() && isC99() && isGNUMode());
   }
 
+  /// allowLiteralDigitSeparator - Language supports literal digit seperator
+  bool allowLiteralDigitSeparator() const { return isCPlusPlus14() || isC23(); }
+
   /// isGNUMode - Language includes GNU extensions.
   bool isGNUMode() const { return Flags & GNUMode; }
 

clang/lib/Basic/LangOptions.cpp

@@ -128,6 +128,7 @@ void LangOptions::setLangDefaults(LangOptions &Opts, Language Lang,
   Opts.WChar = Std.isCPlusPlus();
   Opts.Digraphs = Std.hasDigraphs();
   Opts.RawStringLiterals = Std.hasRawStringLiterals();
+  Opts.AllowLiteralDigitSeparator = Std.allowLiteralDigitSeparator();
   Opts.NamedLoops = Std.isC2y();
 
   Opts.HLSL = Lang == Language::HLSL;

clang/lib/Lex/DependencyDirectivesScanner.cpp

@@ -74,7 +74,7 @@ struct Scanner {
     LangOpts.ObjC = true;
     LangOpts.LineComment = true;
     LangOpts.RawStringLiterals = true;
-    LangOpts.ScanDeps = true;
+    LangOpts.AllowLiteralDigitSeparator = true;
     // FIXME: we do not enable C11 or C++11, so we are missing u/u8/U"".
     return LangOpts;
   }

clang/lib/Lex/Lexer.cpp

@@ -2087,7 +2087,7 @@ bool Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
   }
 
   // If we have a digit separator, continue.
-  if (C == '\'' && (LangOpts.CPlusPlus14 || LangOpts.C23 || LangOpts.ScanDeps)) {
+  if (C == '\'' && LangOpts.AllowLiteralDigitSeparator) {
     auto [Next, NextSize] = getCharAndSizeNoWarn(CurPtr + Size, LangOpts);
     if (isAsciiIdentifierContinue(Next)) {
       if (!isLexingRawMode())