Use MacroInvocLexer in AttributeParser

This should make it easier for us to handle attribute meta items of the
form <SimplePath> '=' <Expression> where the expression isn't a literal.
Some low hanging fruit remains here, but I think I should keep this
patch small as I had some trouble debugging it as-is (see:
Rust::Token::as_string vs Rust::Token::get_str vs
Rust::AST::Token::as_string).

gcc/rust/ChangeLog:

	* ast/rust-ast.cc: Include "rust-macro-invoc-lexer.h".
	(AttributeParser::~AttributeParser): Move function definition
	here.
	(AttributeParser::AttributeParser): Likewise and adjust member
	initialization.
	(AttributeParser::parse_meta_item_inner): Handle changes to
	peek_token.
	(AttributeParser::parse_literal): Likewise.
	(AttributeParser::parse_simple_path_segment): Likewise.
	(AttributeParser::parse_meta_item_seq): Handle changes to
	AttributeParser fields.
	(AttributeParser::peek_token): Move function definition here and
	wrap MacroInvocLexer.
	(AttributeParser::skip_token): Likewise.
	* ast/rust-macro.h (class MacroInvocLexer): Forward declare.
	(class Parser): Likewise.
	(AttributeParser::token_stream): Remove field.
	(AttributeParser::stream_pos): Likewise.
	(AttributeParser::lexer): New field.
	(AttributeParser::parser): Likewise.
	(AttributeParser::AttributeParser): Move definition to
	"rust-ast.cc".
	(AttributeParser::~AttributeParser): Likewise.
	(AttributeParser::peek_token): Likewise.
	(AttributeParser::skip_token): Likewise.

Signed-off-by: Owen Avery <[email protected]>

Author: powerboat9

gcc/rust/ast/rust-ast.cc

@@ -33,6 +33,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "rust-operators.h"
 #include "rust-dir-owner.h"
 #include "rust-attribute-values.h"
+#include "rust-macro-invoc-lexer.h"
 
 /* Compilation unit used for various AST-related functions that would make
  * the headers too long if they were defined inline and don't receive any
@@ -3508,6 +3509,17 @@ DelimTokenTree::parse_to_meta_item () const
   return new AttrInputMetaItemContainer (std::move (meta_items));
 }
 
+AttributeParser::AttributeParser (
+  std::vector<std::unique_ptr<Token>> token_stream, int stream_start_pos)
+  : lexer (new MacroInvocLexer (std::move (token_stream))),
+    parser (new Parser<MacroInvocLexer> (*lexer))
+{
+  if (stream_start_pos)
+    lexer->skip_token (stream_start_pos - 1);
+}
+
+AttributeParser::~AttributeParser () {}
+
 std::unique_ptr<MetaItemInner>
 AttributeParser::parse_meta_item_inner ()
 {
@@ -3549,7 +3561,7 @@ AttributeParser::parse_meta_item_inner ()
       return parse_path_meta_item ();
     }
 
-  auto ident = peek_token ()->as_string ();
+  auto ident = peek_token ()->get_str ();
   auto ident_locus = peek_token ()->get_locus ();
 
   if (is_end_meta_item_tok (peek_token (1)->get_id ()))
@@ -3566,17 +3578,14 @@ AttributeParser::parse_meta_item_inner ()
 	  && is_end_meta_item_tok (peek_token (3)->get_id ()))
 	{
 	  // meta name value str syntax
-	  auto &value_tok = peek_token (2);
-	  auto value = value_tok->as_string ();
+	  const_TokenPtr value_tok = peek_token (2);
+	  auto value = value_tok->get_str ();
 	  auto locus = value_tok->get_locus ();
 
 	  skip_token (2);
 
-	  // remove the quotes from the string value
-	  std::string raw_value = unquote_string (std::move (value));
-
 	  return std::unique_ptr<MetaNameValueStr> (
-	    new MetaNameValueStr (ident, ident_locus, std::move (raw_value),
+	    new MetaNameValueStr (ident, ident_locus, std::move (value),
 				  locus));
 	}
       else
@@ -3728,7 +3737,6 @@ AttributeParser::parse_path_meta_item ()
 std::vector<std::unique_ptr<MetaItemInner>>
 AttributeParser::parse_meta_item_seq ()
 {
-  int vec_length = token_stream.size ();
   std::vector<std::unique_ptr<MetaItemInner>> meta_items;
 
   if (peek_token ()->get_id () != LEFT_PAREN)
@@ -3739,7 +3747,8 @@ AttributeParser::parse_meta_item_seq ()
     }
   skip_token ();
 
-  while (stream_pos < vec_length && peek_token ()->get_id () != RIGHT_PAREN)
+  while (peek_token ()->get_id () != END_OF_FILE
+	 && peek_token ()->get_id () != RIGHT_PAREN)
     {
       std::unique_ptr<MetaItemInner> inner = parse_meta_item_inner ();
       if (inner == nullptr)
@@ -3788,33 +3797,32 @@ DelimTokenTree::to_token_stream () const
 Literal
 AttributeParser::parse_literal ()
 {
-  const std::unique_ptr<Token> &tok = peek_token ();
+  const_TokenPtr tok = peek_token ();
   switch (tok->get_id ())
     {
     case CHAR_LITERAL:
       skip_token ();
-      return Literal (tok->as_string (), Literal::CHAR, tok->get_type_hint ());
+      return Literal (tok->get_str (), Literal::CHAR, tok->get_type_hint ());
     case STRING_LITERAL:
       skip_token ();
-      return Literal (tok->as_string (), Literal::STRING,
-		      tok->get_type_hint ());
+      return Literal (tok->get_str (), Literal::STRING, tok->get_type_hint ());
     case BYTE_CHAR_LITERAL:
       skip_token ();
-      return Literal (tok->as_string (), Literal::BYTE, tok->get_type_hint ());
+      return Literal (tok->get_str (), Literal::BYTE, tok->get_type_hint ());
     case BYTE_STRING_LITERAL:
       skip_token ();
-      return Literal (tok->as_string (), Literal::BYTE_STRING,
+      return Literal (tok->get_str (), Literal::BYTE_STRING,
 		      tok->get_type_hint ());
     case RAW_STRING_LITERAL:
       skip_token ();
-      return Literal (tok->as_string (), Literal::RAW_STRING,
+      return Literal (tok->get_str (), Literal::RAW_STRING,
 		      tok->get_type_hint ());
     case INT_LITERAL:
       skip_token ();
-      return Literal (tok->as_string (), Literal::INT, tok->get_type_hint ());
+      return Literal (tok->get_str (), Literal::INT, tok->get_type_hint ());
     case FLOAT_LITERAL:
       skip_token ();
-      return Literal (tok->as_string (), Literal::FLOAT, tok->get_type_hint ());
+      return Literal (tok->get_str (), Literal::FLOAT, tok->get_type_hint ());
     case TRUE_LITERAL:
       skip_token ();
       return Literal ("true", Literal::BOOL, tok->get_type_hint ());
@@ -3872,12 +3880,12 @@ AttributeParser::parse_simple_path ()
 SimplePathSegment
 AttributeParser::parse_simple_path_segment ()
 {
-  const std::unique_ptr<Token> &tok = peek_token ();
+  const_TokenPtr tok = peek_token ();
   switch (tok->get_id ())
     {
     case IDENTIFIER:
       skip_token ();
-      return SimplePathSegment (tok->as_string (), tok->get_locus ());
+      return SimplePathSegment (tok->get_str (), tok->get_locus ());
     case SUPER:
       skip_token ();
       return SimplePathSegment ("super", tok->get_locus ());
@@ -3911,6 +3919,18 @@ AttributeParser::parse_meta_item_lit ()
     new MetaItemLitExpr (std::move (lit_expr)));
 }
 
+const_TokenPtr
+AttributeParser::peek_token (int i)
+{
+  return lexer->peek_token (i);
+}
+
+void
+AttributeParser::skip_token (int i)
+{
+  lexer->skip_token (i);
+}
+
 bool
 AttrInputMetaItemContainer::check_cfg_predicate (const Session &session) const
 {

gcc/rust/ast/rust-macro.h

@@ -27,6 +27,11 @@
 #include "rust-macro-builtins.h"
 
 namespace Rust {
+
+// forward declarations for AttributeParser
+class MacroInvocLexer;
+template <typename ManagedTokenSource> class Parser;
+
 namespace AST {
 
 class MacroFragSpec
@@ -1116,16 +1121,14 @@ struct AttributeParser
 {
 private:
   // TODO: might as well rewrite to use lexer tokens
-  std::vector<std::unique_ptr<Token>> token_stream;
-  int stream_pos;
+  std::unique_ptr<MacroInvocLexer> lexer;
+  std::unique_ptr<Parser<MacroInvocLexer>> parser;
 
 public:
   AttributeParser (std::vector<std::unique_ptr<Token>> token_stream,
-		   int stream_start_pos = 0)
-    : token_stream (std::move (token_stream)), stream_pos (stream_start_pos)
-  {}
+		   int stream_start_pos = 0);
 
-  ~AttributeParser () = default;
+  ~AttributeParser ();
 
   std::vector<std::unique_ptr<MetaItemInner>> parse_meta_item_seq ();
 
@@ -1146,12 +1149,9 @@ struct AttributeParser
   std::unique_ptr<MetaItem> parse_path_meta_item ();
 
   // TODO: should this be const?
-  std::unique_ptr<Token> &peek_token (int i = 0)
-  {
-    return token_stream[stream_pos + i];
-  }
+  const_TokenPtr peek_token (int i = 0);
 
-  void skip_token (int i = 0) { stream_pos += 1 + i; }
+  void skip_token (int i = 0);
 };
 } // namespace AST
 } // namespace Rust