Skip to content

Commit 95e519e

Browse files
nnethercotennethercote
committed
Remove NtIdent and NtLifetime.
The extra span is now recorded in the new `TokenKind::NtIdent` and `TokenKind::NtLifetime`. These both consist of a single token, and so there's no operator precedence problems with inserting them directly into the token stream. The other way to do this would be to wrap the ident/lifetime in invisible delimiters, but there's a lot of code that assumes an interpolated ident/lifetime fits in a single token, and changing all that code to work with invisible delimiters would have been a pain. (Maybe it could be done in a follow-up.) This change might not seem like much of a win, but it's a first step toward the much bigger and long-desired removal of `Nonterminal` and `TokenKind::Interpolated`. That change is big and complex enough that it's worth doing this piece separately. (Indeed, this commit is based on part of a late commit in rust-lang#114647, a prior attempt at that big and complex change.)
1 parent 9a63a42 commit 95e519e

File tree

11 files changed

+131
-104
lines changed

11 files changed

+131
-104
lines changed

‎compiler/rustc_ast/src/ast_traits.rs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,6 @@ impl HasTokens for Nonterminal {
240240
Nonterminal::NtPath(path) => path.tokens(),
241241
Nonterminal::NtVis(vis) => vis.tokens(),
242242
Nonterminal::NtBlock(block) => block.tokens(),
243-
Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
244243
}
245244
}
246245
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
@@ -254,7 +253,6 @@ impl HasTokens for Nonterminal {
254253
Nonterminal::NtPath(path) => path.tokens_mut(),
255254
Nonterminal::NtVis(vis) => vis.tokens_mut(),
256255
Nonterminal::NtBlock(block) => block.tokens_mut(),
257-
Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
258256
}
259257
}
260258
}

‎compiler/rustc_ast/src/mut_visit.rs

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -781,6 +781,12 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
781781
*span = ident.span;
782782
return; // Avoid visiting the span for the second time.
783783
}
784+
token::NtIdent(ident, _is_raw) => {
785+
vis.visit_ident(ident);
786+
}
787+
token::NtLifetime(ident) => {
788+
vis.visit_ident(ident);
789+
}
784790
token::Interpolated(nt) => {
785791
let nt = Lrc::make_mut(nt);
786792
visit_nonterminal(nt, vis);
@@ -832,8 +838,6 @@ fn visit_nonterminal<T: MutVisitor>(nt: &mut token::Nonterminal, vis: &mut T) {
832838
token::NtPat(pat) => vis.visit_pat(pat),
833839
token::NtExpr(expr) => vis.visit_expr(expr),
834840
token::NtTy(ty) => vis.visit_ty(ty),
835-
token::NtIdent(ident, _is_raw) => vis.visit_ident(ident),
836-
token::NtLifetime(ident) => vis.visit_ident(ident),
837841
token::NtLiteral(expr) => vis.visit_expr(expr),
838842
token::NtMeta(item) => {
839843
let AttrItem { path, args, tokens } = item.deref_mut();

‎compiler/rustc_ast/src/token.rs

Lines changed: 32 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -318,11 +318,20 @@ pub enum TokenKind {
318318
/// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
319319
/// treat regular and interpolated identifiers in the same way.
320320
Ident(Symbol, IdentIsRaw),
321+
/// This identifier (and its span) is the identifier passed to the
322+
/// declarative macro. The span in the surrounding `Token` is the span of
323+
/// the `ident` metavariable in the macro's RHS.
324+
NtIdent(Ident, IdentIsRaw),
325+
321326
/// Lifetime identifier token.
322327
/// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
323328
/// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
324329
/// treat regular and interpolated lifetime identifiers in the same way.
325330
Lifetime(Symbol),
331+
/// This identifier (and its span) is the lifetime passed to the
332+
/// declarative macro. The span in the surrounding `Token` is the span of
333+
/// the `lifetime` metavariable in the macro's RHS.
334+
NtLifetime(Ident),
326335

327336
/// An embedded AST node, as produced by a macro. This only exists for
328337
/// historical reasons. We'd like to get rid of it, for multiple reasons.
@@ -444,8 +453,9 @@ impl Token {
444453
/// Note that keywords are also identifiers, so they should use this
445454
/// if they keep spans or perform edition checks.
446455
pub fn uninterpolated_span(&self) -> Span {
447-
match &self.kind {
448-
Interpolated(nt) => nt.use_span(),
456+
match self.kind {
457+
NtIdent(ident, _) | NtLifetime(ident) => ident.span,
458+
Interpolated(ref nt) => nt.use_span(),
449459
_ => self.span,
450460
}
451461
}
@@ -463,7 +473,7 @@ impl Token {
463473
}
464474

465475
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
466-
| Lifetime(..) | Interpolated(..) | Eof => false,
476+
| NtIdent(..) | Lifetime(..) | NtLifetime(..) | Interpolated(..) | Eof => false,
467477
}
468478
}
469479

@@ -613,14 +623,9 @@ impl Token {
613623
/// into the regular identifier or lifetime token it refers to,
614624
/// otherwise returns the original token.
615625
pub fn uninterpolate(&self) -> Cow<'_, Token> {
616-
match &self.kind {
617-
Interpolated(nt) => match &**nt {
618-
NtIdent(ident, is_raw) => {
619-
Cow::Owned(Token::new(Ident(ident.name, *is_raw), ident.span))
620-
}
621-
NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
622-
_ => Cow::Borrowed(self),
623-
},
626+
match self.kind {
627+
NtIdent(ident, is_raw) => Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span)),
628+
NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
624629
_ => Cow::Borrowed(self),
625630
}
626631
}
@@ -629,12 +634,9 @@ impl Token {
629634
#[inline]
630635
pub fn ident(&self) -> Option<(Ident, IdentIsRaw)> {
631636
// We avoid using `Token::uninterpolate` here because it's slow.
632-
match &self.kind {
633-
&Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
634-
Interpolated(nt) => match &**nt {
635-
NtIdent(ident, is_raw) => Some((*ident, *is_raw)),
636-
_ => None,
637-
},
637+
match self.kind {
638+
Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
639+
NtIdent(ident, is_raw) => Some((ident, is_raw)),
638640
_ => None,
639641
}
640642
}
@@ -643,12 +645,9 @@ impl Token {
643645
#[inline]
644646
pub fn lifetime(&self) -> Option<Ident> {
645647
// We avoid using `Token::uninterpolate` here because it's slow.
646-
match &self.kind {
647-
&Lifetime(name) => Some(Ident::new(name, self.span)),
648-
Interpolated(nt) => match &**nt {
649-
NtLifetime(ident) => Some(*ident),
650-
_ => None,
651-
},
648+
match self.kind {
649+
Lifetime(name) => Some(Ident::new(name, self.span)),
650+
NtLifetime(ident) => Some(ident),
652651
_ => None,
653652
}
654653
}
@@ -837,8 +836,10 @@ impl Token {
837836

838837
Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
839838
| DotDotEq | Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar
840-
| Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..)
841-
| Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None,
839+
| Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | NtIdent(..)
840+
| Lifetime(..) | NtLifetime(..) | Interpolated(..) | DocComment(..) | Eof => {
841+
return None;
842+
}
842843
};
843844

844845
Some(Token::new(kind, self.span.to(joint.span)))
@@ -861,9 +862,6 @@ pub enum Nonterminal {
861862
NtPat(P<ast::Pat>),
862863
NtExpr(P<ast::Expr>),
863864
NtTy(P<ast::Ty>),
864-
/// The span is for the identifier argument passed to the macro.
865-
NtIdent(Ident, IdentIsRaw),
866-
NtLifetime(Ident),
867865
NtLiteral(P<ast::Expr>),
868866
/// Stuff inside brackets for attributes
869867
NtMeta(P<ast::AttrItem>),
@@ -958,7 +956,6 @@ impl Nonterminal {
958956
NtPat(pat) => pat.span,
959957
NtExpr(expr) | NtLiteral(expr) => expr.span,
960958
NtTy(ty) => ty.span,
961-
NtIdent(ident, _) | NtLifetime(ident) => ident.span,
962959
NtMeta(attr_item) => attr_item.span(),
963960
NtPath(path) => path.span,
964961
NtVis(vis) => vis.span,
@@ -974,8 +971,6 @@ impl Nonterminal {
974971
NtExpr(..) => "expression",
975972
NtLiteral(..) => "literal",
976973
NtTy(..) => "type",
977-
NtIdent(..) => "identifier",
978-
NtLifetime(..) => "lifetime",
979974
NtMeta(..) => "attribute",
980975
NtPath(..) => "path",
981976
NtVis(..) => "visibility",
@@ -984,18 +979,12 @@ impl Nonterminal {
984979
}
985980

986981
impl PartialEq for Nonterminal {
987-
fn eq(&self, rhs: &Self) -> bool {
988-
match (self, rhs) {
989-
(NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => {
990-
ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs
991-
}
992-
(NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs,
993-
// FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them
994-
// correctly based on data from AST. This will prevent them from matching each other
995-
// in macros. The comparison will become possible only when each nonterminal has an
996-
// attached token stream from which it was parsed.
997-
_ => false,
998-
}
982+
fn eq(&self, _rhs: &Self) -> bool {
983+
// FIXME: Assume that all nonterminals are not equal, we can't compare them
984+
// correctly based on data from AST. This will prevent them from matching each other
985+
// in macros. The comparison will become possible only when each nonterminal has an
986+
// attached token stream from which it was parsed.
987+
false
999988
}
1000989
}
1001990

@@ -1008,12 +997,10 @@ impl fmt::Debug for Nonterminal {
1008997
NtPat(..) => f.pad("NtPat(..)"),
1009998
NtExpr(..) => f.pad("NtExpr(..)"),
1010999
NtTy(..) => f.pad("NtTy(..)"),
1011-
NtIdent(..) => f.pad("NtIdent(..)"),
10121000
NtLiteral(..) => f.pad("NtLiteral(..)"),
10131001
NtMeta(..) => f.pad("NtMeta(..)"),
10141002
NtPath(..) => f.pad("NtPath(..)"),
10151003
NtVis(..) => f.pad("NtVis(..)"),
1016-
NtLifetime(..) => f.pad("NtLifetime(..)"),
10171004
}
10181005
}
10191006
}

‎compiler/rustc_ast/src/tokenstream.rs

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -466,12 +466,6 @@ impl TokenStream {
466466

467467
pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
468468
match nt {
469-
Nonterminal::NtIdent(ident, is_raw) => {
470-
TokenStream::token_alone(token::Ident(ident.name, *is_raw), ident.span)
471-
}
472-
Nonterminal::NtLifetime(ident) => {
473-
TokenStream::token_alone(token::Lifetime(ident.name), ident.span)
474-
}
475469
Nonterminal::NtItem(item) => TokenStream::from_ast(item),
476470
Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
477471
Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => {
@@ -489,11 +483,17 @@ impl TokenStream {
489483
}
490484

491485
fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree {
492-
match &token.kind {
493-
token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = &**nt => {
494-
TokenTree::Token(Token::new(token::Ident(ident.name, *is_raw), ident.span), spacing)
486+
match token.kind {
487+
token::NtIdent(ident, is_raw) => {
488+
TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing)
495489
}
496-
token::Interpolated(nt) => TokenTree::Delimited(
490+
token::NtLifetime(ident) => TokenTree::Delimited(
491+
DelimSpan::from_single(token.span),
492+
DelimSpacing::new(Spacing::JointHidden, spacing),
493+
Delimiter::Invisible,
494+
TokenStream::token_alone(token::Lifetime(ident.name), ident.span),
495+
),
496+
token::Interpolated(ref nt) => TokenTree::Delimited(
497497
DelimSpan::from_single(token.span),
498498
DelimSpacing::new(Spacing::JointHidden, spacing),
499499
Delimiter::Invisible,
@@ -516,7 +516,10 @@ impl TokenStream {
516516
pub fn flattened(&self) -> TokenStream {
517517
fn can_skip(stream: &TokenStream) -> bool {
518518
stream.trees().all(|tree| match tree {
519-
TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)),
519+
TokenTree::Token(token, _) => !matches!(
520+
token.kind,
521+
token::NtIdent(..) | token::NtLifetime(..) | token::Interpolated(..)
522+
),
520523
TokenTree::Delimited(.., inner) => can_skip(inner),
521524
})
522525
}

‎compiler/rustc_ast_pretty/src/pprust/state.rs

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -852,8 +852,6 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
852852
token::NtBlock(e) => self.block_to_string(e),
853853
token::NtStmt(e) => self.stmt_to_string(e),
854854
token::NtPat(e) => self.pat_to_string(e),
855-
&token::NtIdent(e, is_raw) => IdentPrinter::for_ast_ident(e, is_raw.into()).to_string(),
856-
token::NtLifetime(e) => e.to_string(),
857855
token::NtLiteral(e) => self.expr_to_string(e),
858856
token::NtVis(e) => self.vis_to_string(e),
859857
}
@@ -915,10 +913,14 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
915913
token::Literal(lit) => literal_to_string(lit).into(),
916914

917915
/* Name components */
918-
token::Ident(s, is_raw) => {
919-
IdentPrinter::new(s, is_raw.into(), convert_dollar_crate).to_string().into()
916+
token::Ident(name, is_raw) => {
917+
IdentPrinter::new(name, is_raw.into(), convert_dollar_crate).to_string().into()
920918
}
921-
token::Lifetime(s) => s.to_string().into(),
919+
token::NtIdent(ident, is_raw) => {
920+
IdentPrinter::for_ast_ident(ident, is_raw.into()).to_string().into()
921+
}
922+
token::Lifetime(name) => name.to_string().into(),
923+
token::NtLifetime(ident) => ident.name.to_string().into(),
922924

923925
/* Other */
924926
token::DocComment(comment_kind, attr_style, data) => {

‎compiler/rustc_expand/src/config.rs

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -201,10 +201,17 @@ impl<'a> StripUnconfigured<'a> {
201201
inner = self.configure_tokens(&inner);
202202
Some(AttrTokenTree::Delimited(sp, spacing, delim, inner)).into_iter()
203203
}
204-
AttrTokenTree::Token(ref token, _)
205-
if let TokenKind::Interpolated(nt) = &token.kind =>
206-
{
207-
panic!("Nonterminal should have been flattened at {:?}: {:?}", token.span, nt);
204+
AttrTokenTree::Token(
205+
Token {
206+
kind:
207+
TokenKind::NtIdent(..)
208+
| TokenKind::NtLifetime(..)
209+
| TokenKind::Interpolated(..),
210+
..
211+
},
212+
_,
213+
) => {
214+
panic!("Nonterminal should have been flattened: {:?}", tree);
208215
}
209216
AttrTokenTree::Token(token, spacing) => {
210217
Some(AttrTokenTree::Token(token, spacing)).into_iter()

‎compiler/rustc_expand/src/mbe/transcribe.rs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,16 @@ pub(super) fn transcribe<'a>(
261261
// without wrapping them into groups.
262262
maybe_use_metavar_location(cx, &stack, sp, tt, &mut marker)
263263
}
264+
MatchedSingle(ParseNtResult::Ident(ident, is_raw)) => {
265+
marker.visit_span(&mut sp);
266+
let kind = token::NtIdent(*ident, *is_raw);
267+
TokenTree::token_alone(kind, sp)
268+
}
269+
MatchedSingle(ParseNtResult::Lifetime(ident)) => {
270+
marker.visit_span(&mut sp);
271+
let kind = token::NtLifetime(*ident);
272+
TokenTree::token_alone(kind, sp)
273+
}
264274
MatchedSingle(ParseNtResult::Nt(nt)) => {
265275
// Other variables are emitted into the output stream as groups with
266276
// `Delimiter::Invisible` to maintain parsing priorities.

‎compiler/rustc_expand/src/proc_macro_server.rs

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -220,13 +220,28 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
220220
Ident(sym, is_raw) => {
221221
trees.push(TokenTree::Ident(Ident { sym, is_raw: is_raw.into(), span }))
222222
}
223+
NtIdent(ident, is_raw) => trees.push(TokenTree::Ident(Ident {
224+
sym: ident.name,
225+
is_raw: is_raw.into(),
226+
span: ident.span,
227+
})),
228+
223229
Lifetime(name) => {
224230
let ident = symbol::Ident::new(name, span).without_first_quote();
225231
trees.extend([
226232
TokenTree::Punct(Punct { ch: b'\'', joint: true, span }),
227233
TokenTree::Ident(Ident { sym: ident.name, is_raw: false, span }),
228234
]);
229235
}
236+
NtLifetime(ident) => {
237+
let stream = TokenStream::token_alone(token::Lifetime(ident.name), ident.span);
238+
trees.push(TokenTree::Group(Group {
239+
delimiter: pm::Delimiter::None,
240+
stream: Some(stream),
241+
span: DelimSpan::from_single(span),
242+
}))
243+
}
244+
230245
Literal(token::Lit { kind, symbol, suffix }) => {
231246
trees.push(TokenTree::Literal(self::Literal {
232247
kind: FromInternal::from_internal(kind),
@@ -259,14 +274,6 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
259274
}));
260275
}
261276

262-
Interpolated(ref nt) if let NtIdent(ident, is_raw) = &**nt => {
263-
trees.push(TokenTree::Ident(Ident {
264-
sym: ident.name,
265-
is_raw: matches!(is_raw, IdentIsRaw::Yes),
266-
span: ident.span,
267-
}))
268-
}
269-
270277
Interpolated(nt) => {
271278
let stream = TokenStream::from_nonterminal_ast(&nt);
272279
// A hack used to pass AST fragments to attribute and derive

‎compiler/rustc_parse/src/parser/expr.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -724,7 +724,9 @@ impl<'a> Parser<'a> {
724724
/// Returns the span of expr if it was not interpolated, or the span of the interpolated token.
725725
fn interpolated_or_expr_span(&self, expr: &Expr) -> Span {
726726
match self.prev_token.kind {
727-
TokenKind::Interpolated(..) => self.prev_token.span,
727+
TokenKind::NtIdent(..) | TokenKind::NtLifetime(..) | TokenKind::Interpolated(..) => {
728+
self.prev_token.span
729+
}
728730
_ => expr.span,
729731
}
730732
}

‎compiler/rustc_parse/src/parser/mod.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -407,6 +407,8 @@ pub(super) fn token_descr(token: &Token) -> String {
407407
(Some(TokenDescription::Keyword), _) => Some("keyword"),
408408
(Some(TokenDescription::ReservedKeyword), _) => Some("reserved keyword"),
409409
(Some(TokenDescription::DocComment), _) => Some("doc comment"),
410+
(None, TokenKind::NtIdent(..)) => Some("identifier"),
411+
(None, TokenKind::NtLifetime(..)) => Some("lifetime"),
410412
(None, TokenKind::Interpolated(node)) => Some(node.descr()),
411413
(None, _) => None,
412414
};
@@ -1633,5 +1635,9 @@ pub enum FlatToken {
16331635
#[derive(Clone, Debug)]
16341636
pub enum ParseNtResult {
16351637
Tt(TokenTree),
1638+
Ident(Ident, IdentIsRaw),
1639+
Lifetime(Ident),
1640+
1641+
/// This case will eventually be removed, along with `Token::Interpolate`.
16361642
Nt(Lrc<Nonterminal>),
16371643
}

0 commit comments

Comments
 (0)