Auto merge of rust-lang#139897 - nnethercote:rm-OpenDelim-CloseDelim, r=petrochenkov

Remove `token::{Open,Close}Delim`

By replacing them with `{Open,Close}{Param,Brace,Bracket,Invisible}`.

PR rust-lang#137902 made `ast::TokenKind` more like `lexer::TokenKind` by
replacing the compound `BinOp{,Eq}(BinOpToken)` variants with fieldless
variants `Plus`, `Minus`, `Star`, etc. This commit does a similar thing
with delimiters. It also makes `ast::TokenKind` more similar to
`parser::TokenType`.

This requires a few new methods:
- `TokenKind::is_{,open_,close_}delim()` replace various kinds of
  pattern matches.
- `Delimiter::as_{open,close}_token_kind` are used to convert
  `Delimiter` values to `TokenKind`.

Despite these additions, it's a net reduction in lines of code. This is
because e.g. `token::OpenParen` is so much shorter than
`token::OpenDelim(Delimiter::Parenthesis)` that many multi-line forms
reduce to single line forms. And many places where the number of lines
doesn't change are still easier to read, just because the names are
shorter, e.g.:
```
-   } else if self.token != token::CloseDelim(Delimiter::Brace) {
+   } else if self.token != token::CloseBrace {
```

r? `@petrochenkov`

Author: bors

compiler/rustc_ast/src/attr/mod.rs

@@ -416,10 +416,7 @@ impl MetaItem {
                 // This path is currently unreachable in the test suite.
                 unreachable!()
             }
-            Some(TokenTree::Token(
-                Token { kind: token::OpenDelim(_) | token::CloseDelim(_), .. },
-                _,
-            )) => {
+            Some(TokenTree::Token(Token { kind, .. }, _)) if kind.is_delim() => {
                 panic!("Should be `AttrTokenTree::Delimited`, not delim tokens: {:?}", tt);
             }
             _ => return None,

compiler/rustc_ast/src/token.rs

@@ -32,6 +32,18 @@ pub enum InvisibleOrigin {
     ProcMacro,
 }
 
+impl InvisibleOrigin {
+    // Should the parser skip these invisible delimiters? Ideally this function
+    // will eventually disappear and no invisible delimiters will be skipped.
+    #[inline]
+    pub fn skip(&self) -> bool {
+        match self {
+            InvisibleOrigin::MetaVar(_) => false,
+            InvisibleOrigin::ProcMacro => true,
+        }
+    }
+}
+
 impl PartialEq for InvisibleOrigin {
     #[inline]
     fn eq(&self, _other: &InvisibleOrigin) -> bool {
@@ -125,8 +137,7 @@ impl Delimiter {
     pub fn skip(&self) -> bool {
         match self {
             Delimiter::Parenthesis | Delimiter::Bracket | Delimiter::Brace => false,
-            Delimiter::Invisible(InvisibleOrigin::MetaVar(_)) => false,
-            Delimiter::Invisible(InvisibleOrigin::ProcMacro) => true,
+            Delimiter::Invisible(origin) => origin.skip(),
         }
     }
 
@@ -140,6 +151,24 @@ impl Delimiter {
             _ => false,
         }
     }
+
+    pub fn as_open_token_kind(&self) -> TokenKind {
+        match *self {
+            Delimiter::Parenthesis => OpenParen,
+            Delimiter::Brace => OpenBrace,
+            Delimiter::Bracket => OpenBracket,
+            Delimiter::Invisible(origin) => OpenInvisible(origin),
+        }
+    }
+
+    pub fn as_close_token_kind(&self) -> TokenKind {
+        match *self {
+            Delimiter::Parenthesis => CloseParen,
+            Delimiter::Brace => CloseBrace,
+            Delimiter::Bracket => CloseBracket,
+            Delimiter::Invisible(origin) => CloseInvisible(origin),
+        }
+    }
 }
 
 // Note that the suffix is *not* considered when deciding the `LitKind` in this
@@ -194,9 +223,9 @@ impl Lit {
         match token.uninterpolate().kind {
             Ident(name, IdentIsRaw::No) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)),
             Literal(token_lit) => Some(token_lit),
-            OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
+            OpenInvisible(InvisibleOrigin::MetaVar(
                 MetaVarKind::Literal | MetaVarKind::Expr { .. },
-            ))) => {
+            )) => {
                 // Unreachable with the current test suite.
                 panic!("from_token metavar");
             }
@@ -426,10 +455,22 @@ pub enum TokenKind {
     Question,
     /// Used by proc macros for representing lifetimes, not generated by lexer right now.
     SingleQuote,
-    /// An opening delimiter (e.g., `{`).
-    OpenDelim(Delimiter),
-    /// A closing delimiter (e.g., `}`).
-    CloseDelim(Delimiter),
+    /// `(`
+    OpenParen,
+    /// `)`
+    CloseParen,
+    /// `{`
+    OpenBrace,
+    /// `}`
+    CloseBrace,
+    /// `[`
+    OpenBracket,
+    /// `]`
+    CloseBracket,
+    /// Invisible opening delimiter, produced by a macro.
+    OpenInvisible(InvisibleOrigin),
+    /// Invisible closing delimiter, produced by a macro.
+    CloseInvisible(InvisibleOrigin),
 
     /* Literals */
     Literal(Lit),
@@ -530,6 +571,37 @@ impl TokenKind {
     pub fn should_end_const_arg(&self) -> bool {
         matches!(self, Gt | Ge | Shr | ShrEq)
     }
+
+    pub fn is_delim(&self) -> bool {
+        self.open_delim().is_some() || self.close_delim().is_some()
+    }
+
+    pub fn open_delim(&self) -> Option<Delimiter> {
+        match *self {
+            OpenParen => Some(Delimiter::Parenthesis),
+            OpenBrace => Some(Delimiter::Brace),
+            OpenBracket => Some(Delimiter::Bracket),
+            OpenInvisible(origin) => Some(Delimiter::Invisible(origin)),
+            _ => None,
+        }
+    }
+
+    pub fn close_delim(&self) -> Option<Delimiter> {
+        match *self {
+            CloseParen => Some(Delimiter::Parenthesis),
+            CloseBrace => Some(Delimiter::Brace),
+            CloseBracket => Some(Delimiter::Bracket),
+            CloseInvisible(origin) => Some(Delimiter::Invisible(origin)),
+            _ => None,
+        }
+    }
+
+    pub fn is_close_delim_or_eof(&self) -> bool {
+        match self {
+            CloseParen | CloseBrace | CloseBracket | CloseInvisible(_) | Eof => true,
+            _ => false,
+        }
+    }
 }
 
 impl Token {
@@ -559,7 +631,8 @@ impl Token {
             | DotDotDot | DotDotEq | Comma | Semi | Colon | PathSep | RArrow | LArrow
             | FatArrow | Pound | Dollar | Question | SingleQuote => true,
 
-            OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
+            OpenParen | CloseParen | OpenBrace | CloseBrace | OpenBracket | CloseBracket
+            | OpenInvisible(_) | CloseInvisible(_) | Literal(..) | DocComment(..) | Ident(..)
             | NtIdent(..) | Lifetime(..) | NtLifetime(..) | Eof => false,
         }
     }
@@ -573,11 +646,12 @@ impl Token {
     /// **NB**: Take care when modifying this function, since it will change
     /// the stable set of tokens that are allowed to match an expr nonterminal.
     pub fn can_begin_expr(&self) -> bool {
-        use Delimiter::*;
         match self.uninterpolate().kind {
             Ident(name, is_raw)              =>
                 ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
-            OpenDelim(Parenthesis | Brace | Bracket) | // tuple, array or block
+            OpenParen                         | // tuple
+            OpenBrace                         | // block
+            OpenBracket                       | // array
             Literal(..)                       | // literal
             Bang                              | // operator not
             Minus                             | // unary minus
@@ -591,12 +665,12 @@ impl Token {
             PathSep                           | // global path
             Lifetime(..)                      | // labeled loop
             Pound                             => true, // expression attributes
-            OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
+            OpenInvisible(InvisibleOrigin::MetaVar(
                 MetaVarKind::Block |
                 MetaVarKind::Expr { .. } |
                 MetaVarKind::Literal |
                 MetaVarKind::Path
-            ))) => true,
+            )) => true,
             _ => false,
         }
     }
@@ -608,8 +682,8 @@ impl Token {
         match &self.uninterpolate().kind {
             // box, ref, mut, and other identifiers (can stricten)
             Ident(..) | NtIdent(..) |
-            OpenDelim(Delimiter::Parenthesis) |  // tuple pattern
-            OpenDelim(Delimiter::Bracket) |      // slice pattern
+            OpenParen |                          // tuple pattern
+            OpenBracket |                        // slice pattern
             And |                                // reference
             Minus |                              // negative literal
             AndAnd |                             // double reference
@@ -620,14 +694,14 @@ impl Token {
             Lt |                                 // path (UFCS constant)
             Shl => true,                         // path (double UFCS)
             Or => matches!(pat_kind, PatWithOr), // leading vert `|` or-pattern
-            OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
+            OpenInvisible(InvisibleOrigin::MetaVar(
                 MetaVarKind::Expr { .. } |
                 MetaVarKind::Literal |
                 MetaVarKind::Meta { .. } |
                 MetaVarKind::Pat(_) |
                 MetaVarKind::Path |
                 MetaVarKind::Ty { .. }
-            ))) => true,
+            )) => true,
             _ => false,
         }
     }
@@ -637,8 +711,8 @@ impl Token {
         match self.uninterpolate().kind {
             Ident(name, is_raw) =>
                 ident_can_begin_type(name, self.span, is_raw), // type name or keyword
-            OpenDelim(Delimiter::Parenthesis) | // tuple
-            OpenDelim(Delimiter::Bracket)     | // array
+            OpenParen                         | // tuple
+            OpenBracket                       | // array
             Bang                              | // never
             Star                              | // raw pointer
             And                               | // reference
@@ -647,10 +721,10 @@ impl Token {
             Lifetime(..)                      | // lifetime bound in trait object
             Lt | Shl                          | // associated path
             PathSep => true,                    // global path
-            OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
+            OpenInvisible(InvisibleOrigin::MetaVar(
                 MetaVarKind::Ty { .. } |
                 MetaVarKind::Path
-            ))) => true,
+            )) => true,
             // For anonymous structs or unions, which only appear in specific positions
             // (type of struct fields or union fields), we don't consider them as regular types
             _ => false,
@@ -660,11 +734,11 @@ impl Token {
     /// Returns `true` if the token can appear at the start of a const param.
     pub fn can_begin_const_arg(&self) -> bool {
         match self.kind {
-            OpenDelim(Delimiter::Brace) | Literal(..) | Minus => true,
+            OpenBrace | Literal(..) | Minus => true,
             Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
-            OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
+            OpenInvisible(InvisibleOrigin::MetaVar(
                 MetaVarKind::Expr { .. } | MetaVarKind::Block | MetaVarKind::Literal,
-            ))) => true,
+            )) => true,
             _ => false,
         }
     }
@@ -711,7 +785,7 @@ impl Token {
         match self.uninterpolate().kind {
             Literal(..) | Minus => true,
             Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
-            OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
+            OpenInvisible(InvisibleOrigin::MetaVar(mv_kind)) => match mv_kind {
                 MetaVarKind::Literal => true,
                 MetaVarKind::Expr { can_begin_literal_maybe_minus, .. } => {
                     can_begin_literal_maybe_minus
@@ -725,7 +799,7 @@ impl Token {
     pub fn can_begin_string_literal(&self) -> bool {
         match self.uninterpolate().kind {
             Literal(..) => true,
-            OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
+            OpenInvisible(InvisibleOrigin::MetaVar(mv_kind)) => match mv_kind {
                 MetaVarKind::Literal => true,
                 MetaVarKind::Expr { can_begin_string_literal, .. } => can_begin_string_literal,
                 _ => false,
@@ -892,7 +966,7 @@ impl Token {
     /// from an expanded metavar?
     pub fn is_metavar_seq(&self) -> Option<MetaVarKind> {
         match self.kind {
-            OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => Some(kind),
+            OpenInvisible(InvisibleOrigin::MetaVar(kind)) => Some(kind),
             _ => None,
         }
     }
@@ -970,7 +1044,8 @@ impl Token {
                 Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | PlusEq | MinusEq | StarEq | SlashEq
                 | PercentEq | CaretEq | AndEq | OrEq | ShlEq | ShrEq | At | DotDotDot | DotDotEq
                 | Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question
-                | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | NtIdent(..)
+                | OpenParen | CloseParen | OpenBrace | CloseBrace | OpenBracket | CloseBracket
+                | OpenInvisible(_) | CloseInvisible(_) | Literal(..) | Ident(..) | NtIdent(..)
                 | Lifetime(..) | NtLifetime(..) | DocComment(..) | Eof,
                 _,
             ) => {

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -770,12 +770,12 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
                 self.bclose(span, empty);
             }
             delim => {
-                let token_str = self.token_kind_to_string(&token::OpenDelim(delim));
+                let token_str = self.token_kind_to_string(&delim.as_open_token_kind());
                 self.word(token_str);
                 self.ibox(0);
                 self.print_tts(tts, convert_dollar_crate);
                 self.end();
-                let token_str = self.token_kind_to_string(&token::CloseDelim(delim));
+                let token_str = self.token_kind_to_string(&delim.as_close_token_kind());
                 self.word(token_str);
             }
         }
@@ -932,14 +932,13 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
             token::RArrow => "->".into(),
             token::LArrow => "<-".into(),
             token::FatArrow => "=>".into(),
-            token::OpenDelim(Delimiter::Parenthesis) => "(".into(),
-            token::CloseDelim(Delimiter::Parenthesis) => ")".into(),
-            token::OpenDelim(Delimiter::Bracket) => "[".into(),
-            token::CloseDelim(Delimiter::Bracket) => "]".into(),
-            token::OpenDelim(Delimiter::Brace) => "{".into(),
-            token::CloseDelim(Delimiter::Brace) => "}".into(),
-            token::OpenDelim(Delimiter::Invisible(_))
-            | token::CloseDelim(Delimiter::Invisible(_)) => "".into(),
+            token::OpenParen => "(".into(),
+            token::CloseParen => ")".into(),
+            token::OpenBracket => "[".into(),
+            token::CloseBracket => "]".into(),
+            token::OpenBrace => "{".into(),
+            token::CloseBrace => "}".into(),
+            token::OpenInvisible(_) | token::CloseInvisible(_) => "".into(),
             token::Pound => "#".into(),
             token::Dollar => "$".into(),
             token::Question => "?".into(),

compiler/rustc_attr_parsing/src/parser.rs

@@ -430,9 +430,7 @@ impl<'a> MetaItemListParserContext<'a> {
                 let span = span.with_hi(segments.last().unwrap().span.hi());
                 Some(AttrPath { segments: segments.into_boxed_slice(), span })
             }
-            TokenTree::Token(Token { kind: token::OpenDelim(_) | token::CloseDelim(_), .. }, _) => {
-                None
-            }
+            TokenTree::Token(Token { kind, .. }, _) if kind.is_delim() => None,
             _ => {
                 // malformed attributes can get here. We can't crash, but somewhere else should've
                 // already warned for this.

compiler/rustc_expand/src/config.rs

@@ -237,10 +237,7 @@ impl<'a> StripUnconfigured<'a> {
                     inner = self.configure_tokens(&inner);
                     Some(AttrTokenTree::Delimited(sp, spacing, delim, inner))
                 }
-                AttrTokenTree::Token(
-                    Token { kind: TokenKind::OpenDelim(_) | TokenKind::CloseDelim(_), .. },
-                    _,
-                ) => {
+                AttrTokenTree::Token(Token { kind, .. }, _) if kind.is_delim() => {
                     panic!("Should be `AttrTokenTree::Delimited`, not delim tokens: {:?}", tree);
                 }
                 AttrTokenTree::Token(token, spacing) => Some(AttrTokenTree::Token(token, spacing)),

compiler/rustc_expand/src/expand.rs

@@ -7,13 +7,12 @@ use std::{iter, mem};
 use rustc_ast as ast;
 use rustc_ast::mut_visit::*;
 use rustc_ast::ptr::P;
-use rustc_ast::token::{self, Delimiter};
 use rustc_ast::tokenstream::TokenStream;
 use rustc_ast::visit::{self, AssocCtxt, Visitor, VisitorResult, try_visit, walk_list};
 use rustc_ast::{
     AssocItemKind, AstNodeWrapper, AttrArgs, AttrStyle, AttrVec, ExprKind, ForeignItemKind,
     HasAttrs, HasNodeId, Inline, ItemKind, MacStmtStyle, MetaItemInner, MetaItemKind, ModKind,
-    NodeId, PatKind, StmtKind, TyKind,
+    NodeId, PatKind, StmtKind, TyKind, token,
 };
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::flat_map_in_place::FlatMapInPlace;
@@ -1004,7 +1003,7 @@ pub fn parse_ast_fragment<'a>(
         AstFragmentKind::Stmts => {
             let mut stmts = SmallVec::new();
             // Won't make progress on a `}`.
-            while this.token != token::Eof && this.token != token::CloseDelim(Delimiter::Brace) {
+            while this.token != token::Eof && this.token != token::CloseBrace {
                 if let Some(stmt) = this.parse_full_stmt(AttemptLocalParseRecovery::Yes)? {
                     stmts.push(stmt);
                 }

compiler/rustc_expand/src/mbe/diagnostics.rs

@@ -1,6 +1,6 @@
 use std::borrow::Cow;
 
-use rustc_ast::token::{self, Delimiter, Token, TokenKind};
+use rustc_ast::token::{self, Token};
 use rustc_ast::tokenstream::TokenStream;
 use rustc_errors::{Applicability, Diag, DiagCtxtHandle, DiagMessage};
 use rustc_macros::Subdiagnostic;
@@ -66,8 +66,8 @@ pub(super) fn failed_to_match_macro(
     }
 
     if let MatcherLoc::Token { token: expected_token } = &remaining_matcher
-        && (matches!(expected_token.kind, TokenKind::OpenDelim(Delimiter::Invisible(_)))
-            || matches!(token.kind, TokenKind::OpenDelim(Delimiter::Invisible(_))))
+        && (matches!(expected_token.kind, token::OpenInvisible(_))
+            || matches!(token.kind, token::OpenInvisible(_)))
     {
         err.note("captured metavariables except for `:tt`, `:ident` and `:lifetime` cannot be compared to other tokens");
         err.note("see <https://doc.rust-lang.org/nightly/reference/macros-by-example.html#forwarding-a-matched-fragment> for more information");