Make bare underscore token an Ident rather than Punct in proc-macro

Author: kevinmehall

crates/mbe/src/expander/matcher.rs

@@ -710,6 +710,7 @@ fn match_meta_var(kind: &str, input: &mut TtIter) -> ExpandResult<Option<Fragmen
             let tt_result = match kind {
                 "ident" => input
                     .expect_ident()
+                    .and_then(|ident| if ident.text == "_" { Err(()) } else { Ok(ident) })
                     .map(|ident| Some(tt::Leaf::from(ident.clone()).into()))
                     .map_err(|()| err!("expected ident")),
                 "tt" => input.expect_tt().map(Some).map_err(|()| err!()),

crates/mbe/src/parser.rs

@@ -177,16 +177,8 @@ fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Resul
                     Op::Repeat { tokens: MetaTemplate(tokens), separator, kind }
                 }
                 tt::TokenTree::Leaf(leaf) => match leaf {
-                    tt::Leaf::Punct(punct) => {
-                        static UNDERSCORE: SmolStr = SmolStr::new_inline("_");
-
-                        if punct.char != '_' {
-                            return Err(ParseError::Expected("_".to_string()));
-                        }
-                        let name = UNDERSCORE.clone();
-                        let kind = eat_fragment_kind(src, mode)?;
-                        let id = punct.id;
-                        Op::Var { name, kind, id }
+                    tt::Leaf::Punct(_) => {
+                        return Err(ParseError::Expected("ident".to_string()));
                     }
                     tt::Leaf::Ident(ident) if ident.text == "crate" => {
                         // We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path.

crates/mbe/src/subtree_source.rs

@@ -150,6 +150,7 @@ fn convert_ident(ident: &tt::Ident) -> TtToken {
     let kind = match ident.text.as_ref() {
         "true" => T![true],
         "false" => T![false],
+        "_" => UNDERSCORE,
         i if i.starts_with('\'') => LIFETIME_IDENT,
         _ => SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT),
     };

crates/mbe/src/syntax_bridge.rs

@@ -350,7 +350,7 @@ trait TokenConvertor {
             return;
         }
 
-        result.push(if k.is_punct() {
+        result.push(if k.is_punct() && k != UNDERSCORE {
             assert_eq!(range.len(), TextSize::of('.'));
             let delim = match k {
                 T!['('] => Some((tt::DelimiterKind::Parenthesis, T![')'])),
@@ -395,7 +395,9 @@ trait TokenConvertor {
                     {
                         tt::Spacing::Alone
                     }
-                    Some(next) if next.kind().is_punct() => tt::Spacing::Joint,
+                    Some(next) if next.kind().is_punct() && next.kind() != UNDERSCORE => {
+                        tt::Spacing::Joint
+                    }
                     _ => tt::Spacing::Alone,
                 };
                 let char = match token.to_char() {
@@ -415,6 +417,7 @@ trait TokenConvertor {
             let leaf: tt::Leaf = match k {
                 T![true] | T![false] => make_leaf!(Ident),
                 IDENT => make_leaf!(Ident),
+                UNDERSCORE => make_leaf!(Ident),
                 k if k.is_keyword() => make_leaf!(Ident),
                 k if k.is_literal() => make_leaf!(Literal),
                 LIFETIME_IDENT => {

crates/mbe/src/tests/expand.rs

@@ -1079,6 +1079,12 @@ macro_rules! q {
     .assert_expand_items(r#"q![_]"#, r#"0"#);
 }
 
+#[test]
+fn test_underscore_lifetime() {
+    parse_macro(r#"macro_rules! q { ($a:lifetime) => {0}; }"#)
+        .assert_expand_items(r#"q!['_]"#, r#"0"#);
+}
+
 #[test]
 fn test_vertical_bar_with_pat() {
     parse_macro(

crates/proc_macro_srv/src/rustc_server.rs

@@ -805,5 +805,14 @@ mod tests {
         let t2 = TokenStream::from_str("(a);").unwrap();
         assert_eq!(t2.token_trees.len(), 2);
         assert_eq!(t2.token_trees[0], subtree_paren_a);
+
+        let underscore = TokenStream::from_str("_").unwrap();
+        assert_eq!(
+            underscore.token_trees[0],
+            tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
+                text: "_".into(),
+                id: tt::TokenId::unspecified(),
+            }))
+        );
     }
 }