Skip to content

Commit ae037a8

Browse files
Dylan-DPCDylan-DPC
authored
Rollup merge of rust-lang#95301 - nnethercote:rm-NtTT, r=petrochenkov
Remove `Nonterminal::NtTT`. It's only needed for macro expansion, not as a general element in the AST. This commit removes it, adds `NtOrTt` for the parser and macro expansion cases, and renames the variants in `NamedMatch` to better match the new type. r? `@petrochenkov`
2 parents 0e4524e + 364b908 commit ae037a8

File tree

10 files changed

+52
-55
lines changed

10 files changed

+52
-55
lines changed

‎compiler/rustc_ast/src/ast_like.rs

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@ impl AstLike for crate::token::Nonterminal {
5151
| Nonterminal::NtMeta(_)
5252
| Nonterminal::NtPath(_)
5353
| Nonterminal::NtVis(_)
54-
| Nonterminal::NtTT(_)
5554
| Nonterminal::NtBlock(_)
5655
| Nonterminal::NtIdent(..)
5756
| Nonterminal::NtLifetime(_) => &[],
@@ -67,7 +66,6 @@ impl AstLike for crate::token::Nonterminal {
6766
| Nonterminal::NtMeta(_)
6867
| Nonterminal::NtPath(_)
6968
| Nonterminal::NtVis(_)
70-
| Nonterminal::NtTT(_)
7169
| Nonterminal::NtBlock(_)
7270
| Nonterminal::NtIdent(..)
7371
| Nonterminal::NtLifetime(_) => {}
@@ -84,7 +82,7 @@ impl AstLike for crate::token::Nonterminal {
8482
Nonterminal::NtPath(path) => path.tokens_mut(),
8583
Nonterminal::NtVis(vis) => vis.tokens_mut(),
8684
Nonterminal::NtBlock(block) => block.tokens_mut(),
87-
Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) | Nonterminal::NtTT(..) => None,
85+
Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
8886
}
8987
}
9088
}

‎compiler/rustc_ast/src/mut_visit.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -787,7 +787,6 @@ pub fn visit_interpolated<T: MutVisitor>(nt: &mut token::Nonterminal, vis: &mut
787787
visit_lazy_tts(tokens, vis);
788788
}
789789
token::NtPath(path) => vis.visit_path(path),
790-
token::NtTT(tt) => visit_tt(tt, vis),
791790
token::NtVis(visib) => vis.visit_vis(visib),
792791
}
793792
}

‎compiler/rustc_ast/src/token.rs

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ pub use TokenKind::*;
66

77
use crate::ast;
88
use crate::ptr::P;
9-
use crate::tokenstream::TokenTree;
109

1110
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
1211
use rustc_data_structures::sync::Lrc;
@@ -680,7 +679,6 @@ pub enum Nonterminal {
680679
NtMeta(P<ast::AttrItem>),
681680
NtPath(ast::Path),
682681
NtVis(ast::Visibility),
683-
NtTT(TokenTree),
684682
}
685683

686684
// `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger.
@@ -778,7 +776,6 @@ impl Nonterminal {
778776
NtMeta(attr_item) => attr_item.span(),
779777
NtPath(path) => path.span,
780778
NtVis(vis) => vis.span,
781-
NtTT(tt) => tt.span(),
782779
}
783780
}
784781
}
@@ -790,7 +787,6 @@ impl PartialEq for Nonterminal {
790787
ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs
791788
}
792789
(NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs,
793-
(NtTT(tt_lhs), NtTT(tt_rhs)) => tt_lhs == tt_rhs,
794790
// FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them
795791
// correctly based on data from AST. This will prevent them from matching each other
796792
// in macros. The comparison will become possible only when each nonterminal has an
@@ -813,7 +809,6 @@ impl fmt::Debug for Nonterminal {
813809
NtLiteral(..) => f.pad("NtLiteral(..)"),
814810
NtMeta(..) => f.pad("NtMeta(..)"),
815811
NtPath(..) => f.pad("NtPath(..)"),
816-
NtTT(..) => f.pad("NtTT(..)"),
817812
NtVis(..) => f.pad("NtVis(..)"),
818813
NtLifetime(..) => f.pad("NtLifetime(..)"),
819814
}

‎compiler/rustc_ast_pretty/src/pprust/state.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -714,7 +714,6 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
714714
token::NtIdent(e, is_raw) => IdentPrinter::for_ast_ident(e, is_raw).to_string(),
715715
token::NtLifetime(e) => e.to_string(),
716716
token::NtLiteral(ref e) => self.expr_to_string(e),
717-
token::NtTT(ref tree) => self.tt_to_string(tree),
718717
token::NtVis(ref e) => self.vis_to_string(e),
719718
}
720719
}

‎compiler/rustc_expand/src/mbe/macro_parser.rs

Lines changed: 17 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ crate use ParseResult::*;
7676
use crate::mbe::{self, SequenceRepetition, TokenTree};
7777

7878
use rustc_ast::token::{self, DocComment, Nonterminal, Token};
79-
use rustc_parse::parser::Parser;
79+
use rustc_parse::parser::{NtOrTt, Parser};
8080
use rustc_session::parse::ParseSess;
8181
use rustc_span::symbol::MacroRulesNormalizedIdent;
8282

@@ -275,7 +275,7 @@ pub(super) fn count_names(ms: &[TokenTree]) -> usize {
275275
}
276276

277277
/// `NamedMatch` is a pattern-match result for a single metavar. All
278-
/// `MatchedNtNonTt`s in the `NamedMatch` have the same non-terminal type
278+
/// `MatchedNonterminal`s in the `NamedMatch` have the same non-terminal type
279279
/// (expr, item, etc).
280280
///
281281
/// The in-memory structure of a particular `NamedMatch` represents the match
@@ -306,32 +306,29 @@ pub(super) fn count_names(ms: &[TokenTree]) -> usize {
306306
/// ```rust
307307
/// MatchedSeq([
308308
/// MatchedSeq([
309-
/// MatchedNtNonTt(a),
310-
/// MatchedNtNonTt(b),
311-
/// MatchedNtNonTt(c),
312-
/// MatchedNtNonTt(d),
309+
/// MatchedNonterminal(a),
310+
/// MatchedNonterminal(b),
311+
/// MatchedNonterminal(c),
312+
/// MatchedNonterminal(d),
313313
/// ]),
314314
/// MatchedSeq([
315-
/// MatchedNtNonTt(a),
316-
/// MatchedNtNonTt(b),
317-
/// MatchedNtNonTt(c),
318-
/// MatchedNtNonTt(d),
319-
/// MatchedNtNonTt(e),
315+
/// MatchedNonterminal(a),
316+
/// MatchedNonterminal(b),
317+
/// MatchedNonterminal(c),
318+
/// MatchedNonterminal(d),
319+
/// MatchedNonterminal(e),
320320
/// ])
321321
/// ])
322322
/// ```
323323
#[derive(Debug, Clone)]
324324
crate enum NamedMatch {
325325
MatchedSeq(Lrc<NamedMatchVec>),
326326

327-
// This variant should never hold an `NtTT`. `MatchedNtTt` should be used
328-
// for that case.
329-
MatchedNtNonTt(Lrc<Nonterminal>),
327+
// A metavar match of type `tt`.
328+
MatchedTokenTree(rustc_ast::tokenstream::TokenTree),
330329

331-
// `NtTT` is handled without any cloning when transcribing, unlike other
332-
// nonterminals. Therefore, an `Lrc` isn't helpful and causes unnecessary
333-
// allocations. Hence this separate variant.
334-
MatchedNtTt(rustc_ast::tokenstream::TokenTree),
330+
// A metavar match of any type other than `tt`.
331+
MatchedNonterminal(Lrc<Nonterminal>),
335332
}
336333

337334
/// Takes a slice of token trees `ms` representing a matcher which successfully matched input
@@ -677,8 +674,8 @@ impl<'tt> TtParser<'tt> {
677674
Ok(nt) => nt,
678675
};
679676
let m = match nt {
680-
Nonterminal::NtTT(tt) => MatchedNtTt(tt),
681-
_ => MatchedNtNonTt(Lrc::new(nt)),
677+
NtOrTt::Nt(nt) => MatchedNonterminal(Lrc::new(nt)),
678+
NtOrTt::Tt(tt) => MatchedTokenTree(tt),
682679
};
683680
item.push_match(match_cur, m);
684681
item.idx += 1;

‎compiler/rustc_expand/src/mbe/macro_rules.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ use crate::expand::{ensure_complete_parse, parse_ast_fragment, AstFragment, AstF
44
use crate::mbe;
55
use crate::mbe::macro_check;
66
use crate::mbe::macro_parser::{Error, ErrorReported, Failure, Success, TtParser};
7-
use crate::mbe::macro_parser::{MatchedNtTt, MatchedSeq};
7+
use crate::mbe::macro_parser::{MatchedSeq, MatchedTokenTree};
88
use crate::mbe::transcribe::transcribe;
99

1010
use rustc_ast as ast;
@@ -470,7 +470,7 @@ pub fn compile_declarative_macro(
470470
MatchedSeq(ref s) => s
471471
.iter()
472472
.map(|m| {
473-
if let MatchedNtTt(ref tt) = *m {
473+
if let MatchedTokenTree(ref tt) = *m {
474474
let mut tts = vec![];
475475
mbe::quoted::parse(
476476
tt.clone().into(),
@@ -495,7 +495,7 @@ pub fn compile_declarative_macro(
495495
MatchedSeq(ref s) => s
496496
.iter()
497497
.map(|m| {
498-
if let MatchedNtTt(ref tt) = *m {
498+
if let MatchedTokenTree(ref tt) = *m {
499499
let mut tts = vec![];
500500
mbe::quoted::parse(
501501
tt.clone().into(),

‎compiler/rustc_expand/src/mbe/transcribe.rs

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
use crate::base::ExtCtxt;
2-
use crate::mbe::macro_parser::{MatchedNtNonTt, MatchedNtTt, MatchedSeq, NamedMatch};
2+
use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, MatchedTokenTree, NamedMatch};
33
use crate::mbe::{self, MetaVarExpr};
44
use rustc_ast::mut_visit::{self, MutVisitor};
5-
use rustc_ast::token::{self, Nonterminal, Token, TokenKind};
5+
use rustc_ast::token::{self, Token, TokenKind};
66
use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndSpacing};
77
use rustc_data_structures::fx::FxHashMap;
88
use rustc_data_structures::sync::Lrc;
@@ -234,17 +234,16 @@ pub(super) fn transcribe<'a>(
234234
let ident = MacroRulesNormalizedIdent::new(orignal_ident);
235235
if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) {
236236
match cur_matched {
237-
MatchedNtTt(ref tt) => {
237+
MatchedTokenTree(ref tt) => {
238238
// `tt`s are emitted into the output stream directly as "raw tokens",
239239
// without wrapping them into groups.
240240
let token = tt.clone();
241241
result.push(token.into());
242242
}
243-
MatchedNtNonTt(ref nt) => {
243+
MatchedNonterminal(ref nt) => {
244244
// Other variables are emitted into the output stream as groups with
245245
// `Delimiter::None` to maintain parsing priorities.
246246
// `Interpolated` is currently used for such groups in rustc parser.
247-
debug_assert!(!matches!(**nt, Nonterminal::NtTT(_)));
248247
marker.visit_span(&mut sp);
249248
let token = TokenTree::token(token::Interpolated(nt.clone()), sp);
250249
result.push(token.into());
@@ -312,7 +311,7 @@ fn lookup_cur_matched<'a>(
312311
let mut matched = matched;
313312
for &(idx, _) in repeats {
314313
match matched {
315-
MatchedNtTt(_) | MatchedNtNonTt(_) => break,
314+
MatchedTokenTree(_) | MatchedNonterminal(_) => break,
316315
MatchedSeq(ref ads) => matched = ads.get(idx).unwrap(),
317316
}
318317
}
@@ -402,7 +401,7 @@ fn lockstep_iter_size(
402401
let name = MacroRulesNormalizedIdent::new(name);
403402
match lookup_cur_matched(name, interpolations, repeats) {
404403
Some(matched) => match matched {
405-
MatchedNtTt(_) | MatchedNtNonTt(_) => LockstepIterSize::Unconstrained,
404+
MatchedTokenTree(_) | MatchedNonterminal(_) => LockstepIterSize::Unconstrained,
406405
MatchedSeq(ref ads) => LockstepIterSize::Constraint(ads.len(), name),
407406
},
408407
_ => LockstepIterSize::Unconstrained,
@@ -449,7 +448,7 @@ fn count_repetitions<'a>(
449448
sp: &DelimSpan,
450449
) -> PResult<'a, usize> {
451450
match matched {
452-
MatchedNtTt(_) | MatchedNtNonTt(_) => {
451+
MatchedTokenTree(_) | MatchedNonterminal(_) => {
453452
if declared_lhs_depth == 0 {
454453
return Err(cx.struct_span_err(
455454
sp.entire(),

‎compiler/rustc_parse/src/lib.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -290,7 +290,6 @@ pub fn nt_to_tokenstream(
290290
Nonterminal::NtMeta(ref attr) => convert_tokens(attr.tokens.as_ref()),
291291
Nonterminal::NtPath(ref path) => convert_tokens(path.tokens.as_ref()),
292292
Nonterminal::NtVis(ref vis) => convert_tokens(vis.tokens.as_ref()),
293-
Nonterminal::NtTT(ref tt) => Some(tt.clone().into()),
294293
Nonterminal::NtExpr(ref expr) | Nonterminal::NtLiteral(ref expr) => {
295294
prepend_attrs(&expr.attrs, expr.tokens.as_ref())
296295
}

‎compiler/rustc_parse/src/parser/mod.rs

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ pub use pat::{CommaRecoveryMode, RecoverColon, RecoverComma};
1919
pub use path::PathStyle;
2020

2121
use rustc_ast::ptr::P;
22-
use rustc_ast::token::{self, DelimToken, Token, TokenKind};
22+
use rustc_ast::token::{self, DelimToken, Nonterminal, Token, TokenKind};
2323
use rustc_ast::tokenstream::AttributesData;
2424
use rustc_ast::tokenstream::{self, DelimSpan, Spacing};
2525
use rustc_ast::tokenstream::{TokenStream, TokenTree};
@@ -1507,3 +1507,9 @@ pub enum FlatToken {
15071507
/// handling of replace ranges.
15081508
Empty,
15091509
}
1510+
1511+
#[derive(Debug)]
1512+
pub enum NtOrTt {
1513+
Nt(Nonterminal),
1514+
Tt(TokenTree),
1515+
}

‎compiler/rustc_parse/src/parser/nonterminal.rs

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
use rustc_ast::ptr::P;
2-
use rustc_ast::token::{self, Nonterminal, NonterminalKind, Token};
2+
use rustc_ast::token::{self, NonterminalKind, Token};
33
use rustc_ast::AstLike;
44
use rustc_ast_pretty::pprust;
55
use rustc_errors::PResult;
66
use rustc_span::symbol::{kw, Ident};
77

88
use crate::parser::pat::{CommaRecoveryMode, RecoverColon, RecoverComma};
9-
use crate::parser::{FollowedByType, ForceCollect, Parser, PathStyle};
9+
use crate::parser::{FollowedByType, ForceCollect, NtOrTt, Parser, PathStyle};
1010

1111
impl<'a> Parser<'a> {
1212
/// Checks whether a non-terminal may begin with a particular token.
@@ -85,7 +85,7 @@ impl<'a> Parser<'a> {
8585
NonterminalKind::Lifetime => match token.kind {
8686
token::Lifetime(_) => true,
8787
token::Interpolated(ref nt) => {
88-
matches!(**nt, token::NtLifetime(_) | token::NtTT(_))
88+
matches!(**nt, token::NtLifetime(_))
8989
}
9090
_ => false,
9191
},
@@ -96,14 +96,16 @@ impl<'a> Parser<'a> {
9696
}
9797

9898
/// Parse a non-terminal (e.g. MBE `:pat` or `:ident`).
99-
pub fn parse_nonterminal(&mut self, kind: NonterminalKind) -> PResult<'a, Nonterminal> {
99+
pub fn parse_nonterminal(&mut self, kind: NonterminalKind) -> PResult<'a, NtOrTt> {
100100
// Any `Nonterminal` which stores its tokens (currently `NtItem` and `NtExpr`)
101101
// needs to have them force-captured here.
102102
// A `macro_rules!` invocation may pass a captured item/expr to a proc-macro,
103103
// which requires having captured tokens available. Since we cannot determine
104104
// in advance whether or not a proc-macro will be (transitively) invoked,
105105
// we always capture tokens for any `Nonterminal` which needs them.
106106
let mut nt = match kind {
107+
// Note that TT is treated differently to all the others.
108+
NonterminalKind::TT => return Ok(NtOrTt::Tt(self.parse_token_tree())),
107109
NonterminalKind::Item => match self.parse_item(ForceCollect::Yes)? {
108110
Some(item) => token::NtItem(item),
109111
None => {
@@ -124,9 +126,12 @@ impl<'a> Parser<'a> {
124126
NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr { .. } => {
125127
token::NtPat(self.collect_tokens_no_attrs(|this| match kind {
126128
NonterminalKind::PatParam { .. } => this.parse_pat_no_top_alt(None),
127-
NonterminalKind::PatWithOr { .. } => {
128-
this.parse_pat_allow_top_alt(None, RecoverComma::No, RecoverColon::No, CommaRecoveryMode::EitherTupleOrPipe)
129-
}
129+
NonterminalKind::PatWithOr { .. } => this.parse_pat_allow_top_alt(
130+
None,
131+
RecoverComma::No,
132+
RecoverColon::No,
133+
CommaRecoveryMode::EitherTupleOrPipe,
134+
),
130135
_ => unreachable!(),
131136
})?)
132137
}
@@ -139,9 +144,10 @@ impl<'a> Parser<'a> {
139144
)
140145
}
141146

142-
NonterminalKind::Ty => {
143-
token::NtTy(self.collect_tokens_no_attrs(|this| this.parse_no_question_mark_recover())?)
144-
}
147+
NonterminalKind::Ty => token::NtTy(
148+
self.collect_tokens_no_attrs(|this| this.parse_no_question_mark_recover())?,
149+
),
150+
145151
// this could be handled like a token, since it is one
146152
NonterminalKind::Ident
147153
if let Some((ident, is_raw)) = get_macro_ident(&self.token) =>
@@ -158,7 +164,6 @@ impl<'a> Parser<'a> {
158164
self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))?,
159165
),
160166
NonterminalKind::Meta => token::NtMeta(P(self.parse_attr_item(true)?)),
161-
NonterminalKind::TT => token::NtTT(self.parse_token_tree()),
162167
NonterminalKind::Vis => token::NtVis(
163168
self.collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?,
164169
),
@@ -183,7 +188,7 @@ impl<'a> Parser<'a> {
183188
);
184189
}
185190

186-
Ok(nt)
191+
Ok(NtOrTt::Nt(nt))
187192
}
188193
}
189194

0 commit comments

Comments
 (0)