review: defer handling of numeric constants to the parser -> remove error states

- as commented, using the `NumericLiteralParser` during lexing of
numeric constants falls more under parsing rather than lexing. this
commit removes the use of this completely from the lexer

- this implies that (after moving the register error to parsing) there
are no longer any errors to report during lexing. instead we will just
create an `invalid` or `end_of_stream` token and let the parser provide
an error with more context

- thus we change the api of `ConsumeToken`/`PeekNextToken` so that it no
longer needs to denote an error and will just return a token. this
greatly simplifies the lexer implementation as we no longer need to pass
down the pre-processor

Author: inbelic

clang/include/clang/Basic/DiagnosticLexKinds.td

@@ -1017,12 +1017,4 @@ Error<"'#pragma unsafe_buffer_usage' was not ended">;
 
 def err_pp_pragma_unsafe_buffer_usage_syntax :
 Error<"expected 'begin' or 'end'">;
-
-// HLSL Root Signature Lexing Errors
-let CategoryName = "Root Signature Lexical Issue" in {
-  def err_hlsl_number_literal_overflow :
-    Error<"integer literal '%0' is too large to be represented in a 32-bit integer type">;
-  def err_hlsl_invalid_register_literal: Error<"expected unsigned integer literal as part of register">;
-}
-
 }

clang/include/clang/Parse/HLSLRootSignatureTokenKinds.def

@@ -50,7 +50,6 @@
 #endif
 
 // General Tokens:
-TOK(error)
 TOK(invalid)
 TOK(end_of_stream)
 TOK(int_literal)

clang/include/clang/Parse/ParseHLSLRootSignature.h

@@ -31,15 +31,15 @@ struct RootSignatureToken {
 #include "clang/Parse/HLSLRootSignatureTokenKinds.def"
   };
 
-  Kind Kind = Kind::error;
+  Kind Kind = Kind::invalid;
 
   // Retain the SouceLocation of the token for diagnostics
   clang::SourceLocation TokLoc;
 
-  APValue NumLiteral = APValue();
+  // Retain spelling of an numeric constant to be parsed later
+  StringRef NumSpelling;
 
   // Constructors
-  RootSignatureToken() : TokLoc(SourceLocation()) {}
   RootSignatureToken(clang::SourceLocation TokLoc) : TokLoc(TokLoc) {}
   RootSignatureToken(enum Kind Kind, clang::SourceLocation TokLoc)
       : Kind(Kind), TokLoc(TokLoc) {}
@@ -48,23 +48,15 @@ using TokenKind = enum RootSignatureToken::Kind;
 
 class RootSignatureLexer {
 public:
-  RootSignatureLexer(StringRef Signature, clang::SourceLocation SourceLoc,
-                     clang::Preprocessor &PP)
-      : Buffer(Signature), SourceLoc(SourceLoc), PP(PP) {}
+  RootSignatureLexer(StringRef Signature, clang::SourceLocation SourceLoc)
+      : Buffer(Signature), SourceLoc(SourceLoc) {}
 
-  /// Updates CurToken to the next token. Either it will take the previously
-  /// lexed NextToken, or it will lex a token.
-  ///
-  /// The return value denotes if there was a failure.
-  bool ConsumeToken();
+  /// Consumes and returns the next token.
+  RootSignatureToken ConsumeToken();
 
-  /// Returns the token that comes after CurToken or std::nullopt if an
-  /// error is encountered during lexing of the next token.
-  std::optional<RootSignatureToken> PeekNextToken();
+  /// Returns the token that proceeds CurToken
+  RootSignatureToken PeekNextToken();
 
-  RootSignatureToken GetCurToken() { return CurToken; }
-
-  /// Check if we have reached the end of input
   bool EndOfBuffer() {
     AdvanceBuffer(Buffer.take_while(isspace).size());
     return Buffer.empty();
@@ -74,19 +66,17 @@ class RootSignatureLexer {
   // Internal buffer to iterate over
   StringRef Buffer;
 
-  // Current Token state
-  RootSignatureToken CurToken;
+  // Current peek state
   std::optional<RootSignatureToken> NextToken = std::nullopt;
 
   // Passed down parameters from Sema
   clang::SourceLocation SourceLoc;
-  clang::Preprocessor &PP;
 
-  bool LexNumber(RootSignatureToken &Result);
-  bool LexToken(RootSignatureToken &Result);
+  /// Consumes the buffer and returns the lexed token.
+  RootSignatureToken LexToken();
 
-  // Advance the buffer by the specified number of characters. Updates the
-  // SourceLocation appropriately.
+  /// Advance the buffer by the specified number of characters.
+  /// Updates the SourceLocation appropriately.
   void AdvanceBuffer(unsigned NumCharacters = 1) {
     Buffer = Buffer.drop_front(NumCharacters);
     SourceLoc = SourceLoc.getLocWithOffset(NumCharacters);

clang/lib/Parse/ParseHLSLRootSignature.cpp

@@ -10,45 +10,15 @@ static bool IsNumberChar(char C) {
   return isdigit(C); // integer support
 }
 
-bool RootSignatureLexer::LexNumber(RootSignatureToken &Result) {
-  // Retrieve the possible number
-  StringRef NumSpelling = Buffer.take_while(IsNumberChar);
-
-  // Parse the numeric value and do semantic checks on its specification
-  clang::NumericLiteralParser Literal(NumSpelling, SourceLoc,
-                                      PP.getSourceManager(), PP.getLangOpts(),
-                                      PP.getTargetInfo(), PP.getDiagnostics());
-  if (Literal.hadError)
-    return true; // Error has already been reported so just return
-
-  // Note: if IsNumberChar allows for hexidecimal we will need to turn this
-  // into a diagnostics for potential fixed-point literals
-  assert(Literal.isIntegerLiteral() && "IsNumberChar will only support digits");
-
-  // Retrieve the number value to store into the token
-
-  llvm::APSInt X = llvm::APSInt(32, true);
-  if (Literal.GetIntegerValue(X)) {
-    // Report that the value has overflowed
-    PP.getDiagnostics().Report(Result.TokLoc,
-                               diag::err_hlsl_number_literal_overflow)
-        << NumSpelling;
-    return true;
-  }
-
-  Result.Kind = TokenKind::int_literal;
-  Result.NumLiteral = APValue(X);
-
-  AdvanceBuffer(NumSpelling.size());
-  return false;
-}
-
-bool RootSignatureLexer::LexToken(RootSignatureToken &Result) {
+RootSignatureToken RootSignatureLexer::LexToken() {
   // Discard any leading whitespace
   AdvanceBuffer(Buffer.take_while(isspace).size());
 
+  if (EndOfBuffer())
+    return RootSignatureToken(TokenKind::end_of_stream, SourceLoc);
+
   // Record where this token is in the text for usage in parser diagnostics
-  Result = RootSignatureToken(SourceLoc);
+  RootSignatureToken Result(SourceLoc);
 
   char C = Buffer.front();
 
@@ -58,44 +28,33 @@ bool RootSignatureLexer::LexToken(RootSignatureToken &Result) {
   case Y: {                                                                    \
     Result.Kind = TokenKind::pu_##X;                                           \
     AdvanceBuffer();                                                           \
-    return false;                                                              \
+    return Result;                                                             \
   }
 #include "clang/Parse/HLSLRootSignatureTokenKinds.def"
   default:
     break;
   }
 
-  // Numeric constant
-  if (isdigit(C))
-    return LexNumber(Result);
+  // Integer literal
+  if (isdigit(C)) {
+    Result.Kind = TokenKind::int_literal;
+    Result.NumSpelling = Buffer.take_while(IsNumberChar);
+    AdvanceBuffer(Result.NumSpelling.size());
+    return Result;
+  }
 
   // All following tokens require at least one additional character
   if (Buffer.size() <= 1) {
     Result = RootSignatureToken(TokenKind::invalid, SourceLoc);
-    return false;
+    return Result;
   }
 
   // Peek at the next character to deteremine token type
   char NextC = Buffer[1];
 
   // Registers: [tsub][0-9+]
   if ((C == 't' || C == 's' || C == 'u' || C == 'b') && isdigit(NextC)) {
-    AdvanceBuffer();
-
-    if (LexNumber(Result))
-      return true; // Error parsing number which is already reported
-
-    // Lex number could also parse a float so ensure it was an unsigned int
-    if (Result.Kind != TokenKind::int_literal ||
-        Result.NumLiteral.getInt().isSigned()) {
-      // Return invalid number literal for register error
-      PP.getDiagnostics().Report(Result.TokLoc,
-                                 diag::err_hlsl_invalid_register_literal);
-      return true;
-    }
-
     // Convert character to the register type.
-    // This is done after LexNumber to override the TokenKind
     switch (C) {
     case 'b':
       Result.Kind = TokenKind::bReg;
@@ -112,7 +71,14 @@ bool RootSignatureLexer::LexToken(RootSignatureToken &Result) {
     default:
       llvm_unreachable("Switch for an expected token was not provided");
     }
-    return false;
+
+    AdvanceBuffer();
+
+    // Lex the integer literal
+    Result.NumSpelling = Buffer.take_while(IsNumberChar);
+    AdvanceBuffer(Result.NumSpelling.size());
+
+    return Result;
   }
 
   // Keywords and Enums:
@@ -128,44 +94,26 @@ bool RootSignatureLexer::LexToken(RootSignatureToken &Result) {
   // Then attempt to retreive a string from it
   Result.Kind = Switch.Default(TokenKind::invalid);
   AdvanceBuffer(TokSpelling.size());
-  return false;
+  return Result;
 }
 
-bool RootSignatureLexer::ConsumeToken() {
-  // If we previously peeked then just copy the value over
+RootSignatureToken RootSignatureLexer::ConsumeToken() {
+  // If we previously peeked then just return the previous value over
   if (NextToken && NextToken->Kind != TokenKind::end_of_stream) {
-    // Propogate error up if error was encountered during previous peek
-    if (NextToken->Kind == TokenKind::error)
-      return true;
-    CurToken = *NextToken;
+    RootSignatureToken Result = *NextToken;
     NextToken = std::nullopt;
-    return false;
+    return Result;
   }
-
-  // This will be implicity be true if NextToken->Kind == end_of_stream
-  if (EndOfBuffer()) {
-    CurToken = RootSignatureToken(TokenKind::end_of_stream, SourceLoc);
-    return false;
-  }
-
-  return LexToken(CurToken);
+  return LexToken();
 }
 
-std::optional<RootSignatureToken> RootSignatureLexer::PeekNextToken() {
+RootSignatureToken RootSignatureLexer::PeekNextToken() {
   // Already peeked from the current token
-  if (NextToken.has_value())
-    return NextToken;
-
-  RootSignatureToken Result;
-  if (EndOfBuffer()) {
-    Result = RootSignatureToken(TokenKind::end_of_stream, SourceLoc);
-  } else if (LexToken(Result)) { // propogate lex error up
-    // store error token to prevent further peeking
-    NextToken = RootSignatureToken();
-    return std::nullopt;
-  }
-  NextToken = Result;
-  return Result;
+  if (NextToken)
+    return *NextToken;
+
+  NextToken = LexToken();
+  return *NextToken;
 }
 
 } // namespace hlsl