mbe: Macro fragment fields

compiler/rustc_ast/src/token.rs

@@ -1101,6 +1101,8 @@ pub enum NonterminalKind {
     Path,
     Vis,
     TT,
+    Fn,
+    Adt,
 }
 
 impl NonterminalKind {
@@ -1138,6 +1140,8 @@ impl NonterminalKind {
             sym::path => NonterminalKind::Path,
             sym::vis => NonterminalKind::Vis,
             sym::tt => NonterminalKind::TT,
+            kw::Fn => NonterminalKind::Fn,
+            sym::adt => NonterminalKind::Adt,
             _ => return None,
         })
     }
@@ -1159,6 +1163,8 @@ impl NonterminalKind {
             NonterminalKind::Path => sym::path,
             NonterminalKind::Vis => sym::vis,
             NonterminalKind::TT => sym::tt,
+            NonterminalKind::Fn => kw::Fn,
+            NonterminalKind::Adt => sym::adt,
         }
     }
 }

compiler/rustc_expand/messages.ftl

@@ -142,12 +142,13 @@ expand_mve_extra_tokens =
         *[other] these tokens
     }
 
-expand_mve_missing_paren =
-    expected `(`
-    .label = for this this metavariable expression
+expand_mve_extra_tokens_after_field = unexpected trailing tokens after field
+
+expand_mve_missing_paren_or_dot =
+    expected `(` or `.`
+    .label = after this metavariable expression
     .unexpected = unexpected token
-    .note = metavariable expressions use function-like parentheses syntax
-    .suggestion = try adding parentheses
+    .note = metavariable expressions use parentheses for functions or dot for fields
 
 expand_mve_unrecognized_expr =
     unrecognized metavariable expression
@@ -157,6 +158,8 @@ expand_mve_unrecognized_expr =
 expand_mve_unrecognized_var =
     variable `{$key}` is not recognized in meta-variable expression
 
+expand_mve_expr_has_no_field = expression of type `{$pnr_type}` has no field `{$field}`
+
 expand_non_inline_modules_in_proc_macro_input_are_unstable =
     non-inline modules in proc macro input are unstable
 

compiler/rustc_expand/src/errors.rs

@@ -487,17 +487,23 @@ mod metavar_exprs {
         pub name: String,
     }
 
+    #[derive(Diagnostic)]
+    #[diag(expand_mve_extra_tokens_after_field)]
+    pub(crate) struct MveExtraTokensAfterField {
+        #[primary_span]
+        #[suggestion(code = "", applicability = "machine-applicable")]
+        pub span: Span,
+    }
+
     #[derive(Diagnostic)]
     #[note]
-    #[diag(expand_mve_missing_paren)]
-    pub(crate) struct MveMissingParen {
+    #[diag(expand_mve_missing_paren_or_dot)]
+    pub(crate) struct MveMissingParenOrDot {
         #[primary_span]
         #[label]
         pub ident_span: Span,
         #[label(expand_unexpected)]
         pub unexpected_span: Option<Span>,
-        #[suggestion(code = "( /* ... */ )", applicability = "has-placeholders")]
-        pub insert_span: Option<Span>,
     }
 
     #[derive(Diagnostic)]
@@ -517,6 +523,15 @@ mod metavar_exprs {
         pub span: Span,
         pub key: MacroRulesNormalizedIdent,
     }
+
+    #[derive(Diagnostic)]
+    #[diag(expand_mve_expr_has_no_field)]
+    pub(crate) struct MveExprHasNoField {
+        #[primary_span]
+        pub span: Span,
+        pub pnr_type: &'static str,
+        pub field: Ident,
+    }
 }
 
 #[derive(Diagnostic)]

compiler/rustc_expand/src/mbe.rs

@@ -12,7 +12,7 @@ mod metavar_expr;
 mod quoted;
 mod transcribe;
 
-use metavar_expr::MetaVarExpr;
+use metavar_expr::{MetaVarExpr, MetaVarRecursiveExpr};
 use rustc_ast::token::{Delimiter, NonterminalKind, Token, TokenKind};
 use rustc_ast::tokenstream::{DelimSpacing, DelimSpan};
 use rustc_macros::{Decodable, Encodable};

compiler/rustc_expand/src/mbe/diagnostics.rs

@@ -4,7 +4,7 @@ use rustc_ast::token::{self, Token};
 use rustc_ast::tokenstream::TokenStream;
 use rustc_errors::{Applicability, Diag, DiagCtxtHandle, DiagMessage};
 use rustc_macros::Subdiagnostic;
-use rustc_parse::parser::{Parser, Recovery, token_descr};
+use rustc_parse::parser::{ForceCollect, Parser, Recovery, token_descr};
 use rustc_session::parse::ParseSess;
 use rustc_span::source_map::SourceMap;
 use rustc_span::{DUMMY_SP, ErrorGuaranteed, Ident, Span};
@@ -44,12 +44,14 @@ pub(super) fn failed_to_match_macro(
     // diagnostics.
     let mut tracker = CollectTrackerAndEmitter::new(psess.dcx(), sp);
 
+    // We don't need to collect tokens to process failures, since we'll never expand them.
+    let no = ForceCollect::No;
     let try_success_result = match args {
-        FailedMacro::Func => try_match_macro(psess, name, body, rules, &mut tracker),
+        FailedMacro::Func => try_match_macro(psess, name, body, rules, no, &mut tracker),
         FailedMacro::Attr(attr_args) => {
-            try_match_macro_attr(psess, name, attr_args, body, rules, &mut tracker)
+            try_match_macro_attr(psess, name, attr_args, body, rules, no, &mut tracker)
         }
-        FailedMacro::Derive => try_match_macro_derive(psess, name, body, rules, &mut tracker),
+        FailedMacro::Derive => try_match_macro_derive(psess, name, body, rules, no, &mut tracker),
     };
 
     if try_success_result.is_ok() {
@@ -108,9 +110,12 @@ pub(super) fn failed_to_match_macro(
             let parser = parser_from_cx(psess, body.clone(), Recovery::Allowed);
             let mut tt_parser = TtParser::new(name);
 
-            if let Success(_) =
-                tt_parser.parse_tt(&mut Cow::Borrowed(&parser), lhs, &mut NoopTracker)
-            {
+            if let Success(_) = tt_parser.parse_tt(
+                &mut Cow::Borrowed(&parser),
+                lhs,
+                ForceCollect::No,
+                &mut NoopTracker,
+            ) {
                 if comma_span.is_dummy() {
                     err.note("you might be missing a comma");
                 } else {

compiler/rustc_expand/src/mbe/macro_check.rs

@@ -109,6 +109,7 @@ use rustc_ast::token::{Delimiter, IdentIsRaw, Token, TokenKind};
 use rustc_ast::{DUMMY_NODE_ID, NodeId};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::DecorateDiagCompat;
+use rustc_parse::parser::ForceCollect;
 use rustc_session::lint::builtin::META_VARIABLE_MISUSE;
 use rustc_session::parse::ParseSess;
 use rustc_span::{ErrorGuaranteed, MacroRulesNormalizedIdent, Span, kw};
@@ -193,20 +194,23 @@ struct MacroState<'a> {
 /// - `psess` is used to emit diagnostics and lints
 /// - `node_id` is used to emit lints
 /// - `args`, `lhs`, and `rhs` represent the rule
+/// - `collect` is set to the token collection mode needed for the macro (recursive if any metavar exprs)
 pub(super) fn check_meta_variables(
     psess: &ParseSess,
     node_id: NodeId,
     args: Option<&TokenTree>,
     lhs: &TokenTree,
     rhs: &TokenTree,
+    collect: &mut ForceCollect,
 ) -> Result<(), ErrorGuaranteed> {
     let mut guar = None;
     let mut binders = Binders::default();
+    let ops = &Stack::Empty;
     if let Some(args) = args {
-        check_binders(psess, node_id, args, &Stack::Empty, &mut binders, &Stack::Empty, &mut guar);
+        check_binders(psess, node_id, args, &Stack::Empty, &mut binders, ops, collect, &mut guar);
     }
-    check_binders(psess, node_id, lhs, &Stack::Empty, &mut binders, &Stack::Empty, &mut guar);
-    check_occurrences(psess, node_id, rhs, &Stack::Empty, &binders, &Stack::Empty, &mut guar);
+    check_binders(psess, node_id, lhs, &Stack::Empty, &mut binders, ops, collect, &mut guar);
+    check_occurrences(psess, node_id, rhs, &Stack::Empty, &binders, ops, collect, &mut guar);
     guar.map_or(Ok(()), Err)
 }
 
@@ -220,6 +224,7 @@ pub(super) fn check_meta_variables(
 /// - `macros` is the stack of possible outer macros
 /// - `binders` contains the binders of the LHS
 /// - `ops` is the stack of Kleene operators from the LHS
+/// - `collect` is updated to recursive if we encounter any metadata expressions
 /// - `guar` is set in case of errors
 fn check_binders(
     psess: &ParseSess,
@@ -228,6 +233,7 @@ fn check_binders(
     macros: &Stack<'_, MacroState<'_>>,
     binders: &mut Binders,
     ops: &Stack<'_, KleeneToken>,
+    collect: &mut ForceCollect,
     guar: &mut Option<ErrorGuaranteed>,
 ) {
     match *lhs {
@@ -255,7 +261,7 @@ fn check_binders(
                 binders.insert(name, BinderInfo { span, ops: ops.into() });
             } else {
                 // 3. The meta-variable is bound: This is an occurrence.
-                check_occurrences(psess, node_id, lhs, macros, binders, ops, guar);
+                check_occurrences(psess, node_id, lhs, macros, binders, ops, collect, guar);
             }
         }
         // Similarly, this can only happen when checking a toplevel macro.
@@ -280,13 +286,13 @@ fn check_binders(
         TokenTree::MetaVarExpr(..) => {}
         TokenTree::Delimited(.., ref del) => {
             for tt in &del.tts {
-                check_binders(psess, node_id, tt, macros, binders, ops, guar);
+                check_binders(psess, node_id, tt, macros, binders, ops, collect, guar);
             }
         }
         TokenTree::Sequence(_, ref seq) => {
             let ops = ops.push(seq.kleene);
             for tt in &seq.tts {
-                check_binders(psess, node_id, tt, macros, binders, &ops, guar);
+                check_binders(psess, node_id, tt, macros, binders, &ops, collect, guar);
             }
         }
     }
@@ -316,6 +322,7 @@ fn get_binder_info<'a>(
 /// - `macros` is the stack of possible outer macros
 /// - `binders` contains the binders of the associated LHS
 /// - `ops` is the stack of Kleene operators from the RHS
+/// - `collect` is updated to recursive if we encounter any metadata expressions
 /// - `guar` is set in case of errors
 fn check_occurrences(
     psess: &ParseSess,
@@ -324,6 +331,7 @@ fn check_occurrences(
     macros: &Stack<'_, MacroState<'_>>,
     binders: &Binders,
     ops: &Stack<'_, KleeneToken>,
+    collect: &mut ForceCollect,
     guar: &mut Option<ErrorGuaranteed>,
 ) {
     match *rhs {
@@ -336,17 +344,21 @@ fn check_occurrences(
             check_ops_is_prefix(psess, node_id, macros, binders, ops, span, name);
         }
         TokenTree::MetaVarExpr(dl, ref mve) => {
+            // Require recursive token collection if we have any metadata expressions
+            *collect = ForceCollect::Recursive;
             mve.for_each_metavar((), |_, ident| {
                 let name = MacroRulesNormalizedIdent::new(*ident);
                 check_ops_is_prefix(psess, node_id, macros, binders, ops, dl.entire(), name);
             });
         }
         TokenTree::Delimited(.., ref del) => {
-            check_nested_occurrences(psess, node_id, &del.tts, macros, binders, ops, guar);
+            check_nested_occurrences(psess, node_id, &del.tts, macros, binders, ops, collect, guar);
         }
         TokenTree::Sequence(_, ref seq) => {
             let ops = ops.push(seq.kleene);
-            check_nested_occurrences(psess, node_id, &seq.tts, macros, binders, &ops, guar);
+            check_nested_occurrences(
+                psess, node_id, &seq.tts, macros, binders, &ops, collect, guar,
+            );
         }
     }
 }
@@ -381,6 +393,7 @@ enum NestedMacroState {
 /// - `macros` is the stack of possible outer macros
 /// - `binders` contains the binders of the associated LHS
 /// - `ops` is the stack of Kleene operators from the RHS
+/// - `collect` is updated to recursive if we encounter metadata expressions or nested macros
 /// - `guar` is set in case of errors
 fn check_nested_occurrences(
     psess: &ParseSess,
@@ -389,6 +402,7 @@ fn check_nested_occurrences(
     macros: &Stack<'_, MacroState<'_>>,
     binders: &Binders,
     ops: &Stack<'_, KleeneToken>,
+    collect: &mut ForceCollect,
     guar: &mut Option<ErrorGuaranteed>,
 ) {
     let mut state = NestedMacroState::Empty;
@@ -421,16 +435,26 @@ fn check_nested_occurrences(
             (NestedMacroState::MacroRulesBang, &TokenTree::MetaVar(..)) => {
                 state = NestedMacroState::MacroRulesBangName;
                 // We check that the meta-variable is correctly used.
-                check_occurrences(psess, node_id, tt, macros, binders, ops, guar);
+                check_occurrences(psess, node_id, tt, macros, binders, ops, collect, guar);
             }
             (NestedMacroState::MacroRulesBangName, TokenTree::Delimited(.., del))
             | (NestedMacroState::MacroName, TokenTree::Delimited(.., del))
                 if del.delim == Delimiter::Brace =>
             {
+                // Conservatively assume that we might need recursive tokens, since our parsing in
+                // the face of nested macro definitions is fuzzy.
+                *collect = ForceCollect::Recursive;
                 let macro_rules = state == NestedMacroState::MacroRulesBangName;
                 state = NestedMacroState::Empty;
-                let rest =
-                    check_nested_macro(psess, node_id, macro_rules, &del.tts, &nested_macros, guar);
+                let rest = check_nested_macro(
+                    psess,
+                    node_id,
+                    macro_rules,
+                    &del.tts,
+                    &nested_macros,
+                    collect,
+                    guar,
+                );
                 // If we did not check the whole macro definition, then check the rest as if outside
                 // the macro definition.
                 check_nested_occurrences(
@@ -440,6 +464,7 @@ fn check_nested_occurrences(
                     macros,
                     binders,
                     ops,
+                    collect,
                     guar,
                 );
             }
@@ -452,7 +477,7 @@ fn check_nested_occurrences(
             (NestedMacroState::Macro, &TokenTree::MetaVar(..)) => {
                 state = NestedMacroState::MacroName;
                 // We check that the meta-variable is correctly used.
-                check_occurrences(psess, node_id, tt, macros, binders, ops, guar);
+                check_occurrences(psess, node_id, tt, macros, binders, ops, collect, guar);
             }
             (NestedMacroState::MacroName, TokenTree::Delimited(.., del))
                 if del.delim == Delimiter::Parenthesis =>
@@ -466,6 +491,7 @@ fn check_nested_occurrences(
                     &nested_macros,
                     &mut nested_binders,
                     &Stack::Empty,
+                    collect,
                     guar,
                 );
             }
@@ -480,12 +506,13 @@ fn check_nested_occurrences(
                     &nested_macros,
                     &nested_binders,
                     &Stack::Empty,
+                    collect,
                     guar,
                 );
             }
             (_, tt) => {
                 state = NestedMacroState::Empty;
-                check_occurrences(psess, node_id, tt, macros, binders, ops, guar);
+                check_occurrences(psess, node_id, tt, macros, binders, ops, collect, guar);
             }
         }
     }
@@ -504,13 +531,15 @@ fn check_nested_occurrences(
 /// - `macro_rules` specifies whether the macro is `macro_rules`
 /// - `tts` is checked as a list of (LHS) => {RHS}
 /// - `macros` is the stack of outer macros
+/// - `collect` is passed down through to the macro checking code (but is already recursive)
 /// - `guar` is set in case of errors
 fn check_nested_macro(
     psess: &ParseSess,
     node_id: NodeId,
     macro_rules: bool,
     tts: &[TokenTree],
     macros: &Stack<'_, MacroState<'_>>,
+    collect: &mut ForceCollect,
     guar: &mut Option<ErrorGuaranteed>,
 ) -> usize {
     let n = tts.len();
@@ -528,8 +557,8 @@ fn check_nested_macro(
         let lhs = &tts[i];
         let rhs = &tts[i + 2];
         let mut binders = Binders::default();
-        check_binders(psess, node_id, lhs, macros, &mut binders, &Stack::Empty, guar);
-        check_occurrences(psess, node_id, rhs, macros, &binders, &Stack::Empty, guar);
+        check_binders(psess, node_id, lhs, macros, &mut binders, &Stack::Empty, collect, guar);
+        check_occurrences(psess, node_id, rhs, macros, &binders, &Stack::Empty, collect, guar);
         // Since the last semicolon is optional for `macro_rules` macros and decl_macro are not terminated,
         // we increment our checked position by how many token trees we already checked (the 3
         // above) before checking for the separator.