review: remove unique end_of_stream lexing error

- instead of reporting the unique end_of_stream error, we can instead
pass up an end_of_stream token and let the unexpected diag reporting
handles this in the parser with more context

Author: inbelic

clang/include/clang/Basic/DiagnosticLexKinds.td

@@ -1025,7 +1025,6 @@ let CategoryName = "Root Signature Lexical Issue" in {
     Error<"integer literal '%0' is too large to be represented in a 32-bit integer type">;
   def err_hlsl_invalid_token: Error<"unable to lex a valid Root Signature token">;
   def err_hlsl_invalid_register_literal: Error<"expected unsigned integer literal as part of register">;
-  def err_hlsl_rootsig_unexpected_eos : Error<"unexpected end to token stream">;
 }
 
 }

clang/lib/Parse/ParseHLSLRootSignature.cpp

@@ -164,10 +164,8 @@ bool RootSignatureLexer::ConsumeToken() {
 
   // This will be implicity be true if NextToken->Kind == end_of_stream
   if (EndOfBuffer()) {
-    // Report unexpected end of tokens error
-    PP.getDiagnostics().Report(SourceLoc,
-                               diag::err_hlsl_rootsig_unexpected_eos);
-    return true;
+    CurToken = RootSignatureToken(TokenKind::end_of_stream, SourceLoc);
+    return false;
   }
 
   return LexToken(CurToken);
@@ -178,14 +176,11 @@ std::optional<RootSignatureToken> RootSignatureLexer::PeekNextToken() {
   if (NextToken.has_value())
     return NextToken;
 
-  if (EndOfBuffer()) {
-    // We have reached the end of the stream, but only error on consume
-    return RootSignatureToken(TokenKind::end_of_stream, SourceLoc);
-  }
-
   RootSignatureToken Result;
-  if (LexToken(Result))
-    return std::nullopt;
+  if (EndOfBuffer()) {
+    Result = RootSignatureToken(TokenKind::end_of_stream, SourceLoc);
+  } else if (LexToken(Result))
+    return std::nullopt; // propogate lex error up
   NextToken = Result;
   return Result;
 }

clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp

@@ -258,27 +258,15 @@ TEST_F(ParseHLSLRootSignatureTest, ValidLexPeekTest) {
   ASSERT_TRUE(Res.has_value());
   ASSERT_EQ(Res->Kind, hlsl::TokenKind::end_of_stream);
 
+  // Ensure no error raised when consuming past end of stream
+  ASSERT_FALSE(Lexer.ConsumeToken());
+  ASSERT_EQ(Lexer.GetCurToken().Kind, hlsl::TokenKind::end_of_stream);
+
   ASSERT_TRUE(Consumer->IsSatisfied());
 }
 
 // Invalid Lexing Tests
 
-TEST_F(ParseHLSLRootSignatureTest, InvalidParseUnexpectedEOSTest) {
-  const llvm::StringLiteral Source = R"cc()cc";
-
-  TrivialModuleLoader ModLoader;
-  auto PP = CreatePP(Source, ModLoader);
-  auto TokLoc = SourceLocation();
-
-  hlsl::RootSignatureLexer Lexer(Source, TokLoc, *PP);
-
-  // Test correct diagnostic produced
-  Consumer->SetExpected(diag::err_hlsl_rootsig_unexpected_eos);
-  ASSERT_TRUE(Lexer.ConsumeToken());
-
-  ASSERT_TRUE(Consumer->IsSatisfied());
-}
-
 TEST_F(ParseHLSLRootSignatureTest, InvalidLexOverflowedNumberTest) {
   // This test will check that the lexing fails due to an integer overflow
   const llvm::StringLiteral Source = R"cc(