self-review: define an error token to denote lex errors during peek

- the current use of invalid is not able to distinguish between a lex
error that has been reported to diagnostics and an invalid token that
should be reported during parsing
- an error token is defined as the default token to denote a lex error
that has been reported and shouldn't be handled by the parsing

Author: inbelic

clang/include/clang/Parse/HLSLRootSignatureTokenKinds.def

@@ -50,6 +50,7 @@
 #endif
 
 // General Tokens:
+TOK(error)
 TOK(invalid)
 TOK(end_of_stream)
 TOK(int_literal)

clang/include/clang/Parse/ParseHLSLRootSignature.h

@@ -31,7 +31,7 @@ struct RootSignatureToken {
 #include "clang/Parse/HLSLRootSignatureTokenKinds.def"
   };
 
-  Kind Kind = Kind::invalid;
+  Kind Kind = Kind::error;
 
   // Retain the SouceLocation of the token for diagnostics
   clang::SourceLocation TokLoc;

clang/lib/Parse/ParseHLSLRootSignature.cpp

@@ -71,7 +71,7 @@ bool RootSignatureLexer::LexToken(RootSignatureToken &Result) {
 
   // All following tokens require at least one additional character
   if (Buffer.size() <= 1) {
-    Result = RootSignatureToken(SourceLoc);
+    Result = RootSignatureToken(TokenKind::invalid, SourceLoc);
     return false;
   }
 
@@ -134,6 +134,9 @@ bool RootSignatureLexer::LexToken(RootSignatureToken &Result) {
 bool RootSignatureLexer::ConsumeToken() {
   // If we previously peeked then just copy the value over
   if (NextToken && NextToken->Kind != TokenKind::end_of_stream) {
+    // Propogate error up if error was encountered during previous peek
+    if (NextToken->Kind == TokenKind::error)
+      return true;
     CurToken = *NextToken;
     NextToken = std::nullopt;
     return false;
@@ -156,8 +159,11 @@ std::optional<RootSignatureToken> RootSignatureLexer::PeekNextToken() {
   RootSignatureToken Result;
   if (EndOfBuffer()) {
     Result = RootSignatureToken(TokenKind::end_of_stream, SourceLoc);
-  } else if (LexToken(Result))
-    return std::nullopt; // propogate lex error up
+  } else if (LexToken(Result)) { // propogate lex error up
+    // store error token to prevent further peeking
+    NextToken = RootSignatureToken();
+    return std::nullopt;
+  }
   NextToken = Result;
   return Result;
 }

clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp

@@ -95,7 +95,8 @@ class ParseHLSLRootSignatureTest : public ::testing::Test {
                    SmallVector<hlsl::RootSignatureToken> &Computed,
                    SmallVector<hlsl::TokenKind> &Expected) {
     for (unsigned I = 0, E = Expected.size(); I != E; ++I) {
-      if (Expected[I] == hlsl::TokenKind::invalid ||
+      if (Expected[I] == hlsl::TokenKind::error ||
+          Expected[I] == hlsl::TokenKind::invalid ||
           Expected[I] == hlsl::TokenKind::end_of_stream)
         continue;
       ASSERT_FALSE(Lexer.ConsumeToken());
@@ -282,6 +283,11 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidLexOverflowedNumberTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc, *PP);
 
+  // We will also test that the error can be produced during peek and the Lexer
+  // will correctly just return true on the next ConsumeToken without
+  // reporting another error
+  auto Result = Lexer.PeekNextToken();
+  ASSERT_EQ(std::nullopt, Result);
   ASSERT_TRUE(Lexer.ConsumeToken());
   ASSERT_TRUE(Consumer->IsSatisfied());
 }