From 5814fa70ca9124b54ba638cf44c508ad6501acde Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Mon, 12 Dec 2022 11:10:00 -0800 Subject: [PATCH 1/4] Add regression test for issue 91 Currently fails with: error: `"1_u32"` is not a valid identifier --> tests/test_expr.rs:56:49 | 56 | [].insert(0, [<1_u $bit>]); | ^^^^^^^^^^^^ ... 63 | vec_insert!(32); | --------------- in this macro invocation | = note: this error originates in the macro `vec_insert` (in Nightly builds, run with -Z macro-backtrace for more info) error[E0425]: cannot find function `vector_insert_32_bit` in this scope --> tests/test_expr.rs:64:5 | 64 | vector_insert_32_bit(2); | ^^^^^^^^^^^^^^^^^^^^ not found in this scope --- tests/test_expr.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/test_expr.rs b/tests/test_expr.rs index 8ad5de9..b135ff3 100644 --- a/tests/test_expr.rs +++ b/tests/test_expr.rs @@ -26,7 +26,7 @@ fn test_repeat() { } #[test] -fn test_literals() { +fn test_literal_to_identifier() { const CONST0: &str = "const0"; let pasted = paste!([]); @@ -45,6 +45,25 @@ fn test_literals() { assert_eq!(pasted, CONST0); } +#[test] +fn test_literal_suffix() { + macro_rules! vec_insert { + ($bit:tt) => { + paste! { + fn [](insert_size: usize) { + let mut [] = Vec::new(); + for _ in 0..insert_size { + [].insert(0, [<1_u $bit>]); + } + } + } + }; + } + + vec_insert!(32); + vector_insert_32_bit(2); +} + #[test] fn test_underscore() { paste! { From 8c15bafbe689df2ffeead7c92de8be499e01e32d Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Mon, 12 Dec 2022 11:22:22 -0800 Subject: [PATCH 2/4] Support Literal as result of paste --- src/lib.rs | 17 ++++++++++++++++- tests/ui/invalid-ident.stderr | 12 ++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 983ba06..31bbf9c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -155,10 +155,13 @@ mod segment; use crate::attr::expand_attr; use crate::error::{Error, Result}; use crate::segment::Segment; -use proc_macro::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree}; +use proc_macro::{ + Delimiter, Group, Ident, LexError, Literal, Punct, Spacing, Span, TokenStream, TokenTree, +}; use std::char; use std::iter; use std::panic; +use std::str::FromStr; #[proc_macro] pub fn paste(input: TokenStream) -> TokenStream { @@ -416,6 +419,18 @@ fn pasted_to_tokens(mut pasted: String, span: Span) -> Result { apostrophe.set_span(span); tokens.extend(iter::once(apostrophe)); pasted.remove(0); + } else if pasted.starts_with(|ch: char| ch.is_ascii_digit()) { + let literal = match panic::catch_unwind(|| Literal::from_str(&pasted)) { + Ok(Ok(literal)) => TokenTree::Literal(literal), + Ok(Err(LexError { .. })) | Err(_) => { + return Err(Error::new( + span, + &format!("`{:?}` is not a valid literal", pasted), + )); + } + }; + tokens.extend(iter::once(literal)); + return Ok(tokens); } let ident = match panic::catch_unwind(|| Ident::new(&pasted, span)) { diff --git a/tests/ui/invalid-ident.stderr b/tests/ui/invalid-ident.stderr index f32d484..28593fb 100644 --- a/tests/ui/invalid-ident.stderr +++ b/tests/ui/invalid-ident.stderr @@ -1,8 +1,12 @@ -error: `"0f"` is not a valid identifier - --> tests/ui/invalid-ident.rs:4:8 +error: expected identifier, found `0f` + --> tests/ui/invalid-ident.rs:3:1 | -4 | fn [<0 f>]() {} - | ^^^^^^^ +3 | / paste! { +4 | | fn [<0 f>]() {} +5 | | } + | |_^ expected identifier + | + = note: this error originates in the macro `paste` (in Nightly builds, run with -Z macro-backtrace for more info) error: `"f\""` is not a valid identifier --> tests/ui/invalid-ident.rs:8:8 From 0ffe86082d82076b354f5f1f1a0d2f5d89619d88 Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Mon, 12 Dec 2022 11:29:35 -0800 Subject: [PATCH 3/4] Simplify literal suffix test --- tests/test_expr.rs | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/test_expr.rs b/tests/test_expr.rs index b135ff3..5ce2549 100644 --- a/tests/test_expr.rs +++ b/tests/test_expr.rs @@ -47,21 +47,13 @@ fn test_literal_to_identifier() { #[test] fn test_literal_suffix() { - macro_rules! vec_insert { + macro_rules! literal { ($bit:tt) => { - paste! { - fn [](insert_size: usize) { - let mut [] = Vec::new(); - for _ in 0..insert_size { - [].insert(0, [<1_u $bit>]); - } - } - } + paste!([<1_u $bit>]) }; } - vec_insert!(32); - vector_insert_32_bit(2); + assert_eq!(literal!(32), 1); } #[test] From b3b77f233dc12ffb24f7a7b1f07494539aae1871 Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Mon, 12 Dec 2022 11:31:51 -0800 Subject: [PATCH 4/4] Gate literal fromstr support on new enough compiler --- build.rs | 33 +++++++++++++++++++++++++++++++++ src/lib.rs | 37 +++++++++++++++++++++---------------- 2 files changed, 54 insertions(+), 16 deletions(-) create mode 100644 build.rs diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..349b5fc --- /dev/null +++ b/build.rs @@ -0,0 +1,33 @@ +use std::env; +use std::process::Command; +use std::str; + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + + let version = match rustc_version() { + Some(version) => version, + None => return, + }; + + if version.minor < 54 { + // https://github.com/rust-lang/rust/pull/84717 + println!("cargo:rustc-cfg=no_literal_fromstr"); + } +} + +struct RustcVersion { + minor: u32, +} + +fn rustc_version() -> Option { + let rustc = env::var_os("RUSTC")?; + let output = Command::new(rustc).arg("--version").output().ok()?; + let version = str::from_utf8(&output.stdout).ok()?; + let mut pieces = version.split('.'); + if pieces.next() != Some("rustc 1") { + return None; + } + let minor = pieces.next()?.parse().ok()?; + Some(RustcVersion { minor }) +} diff --git a/src/lib.rs b/src/lib.rs index 31bbf9c..80c1b98 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -155,13 +155,10 @@ mod segment; use crate::attr::expand_attr; use crate::error::{Error, Result}; use crate::segment::Segment; -use proc_macro::{ - Delimiter, Group, Ident, LexError, Literal, Punct, Spacing, Span, TokenStream, TokenTree, -}; +use proc_macro::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree}; use std::char; use std::iter; use std::panic; -use std::str::FromStr; #[proc_macro] pub fn paste(input: TokenStream) -> TokenStream { @@ -414,23 +411,31 @@ fn parse_bracket_as_segments(input: TokenStream, scope: Span) -> Result Result { let mut tokens = TokenStream::new(); + #[cfg(not(no_literal_fromstr))] + { + use proc_macro::{LexError, Literal}; + use std::str::FromStr; + + if pasted.starts_with(|ch: char| ch.is_ascii_digit()) { + let literal = match panic::catch_unwind(|| Literal::from_str(&pasted)) { + Ok(Ok(literal)) => TokenTree::Literal(literal), + Ok(Err(LexError { .. })) | Err(_) => { + return Err(Error::new( + span, + &format!("`{:?}` is not a valid literal", pasted), + )); + } + }; + tokens.extend(iter::once(literal)); + return Ok(tokens); + } + } + if pasted.starts_with('\'') { let mut apostrophe = TokenTree::Punct(Punct::new('\'', Spacing::Joint)); apostrophe.set_span(span); tokens.extend(iter::once(apostrophe)); pasted.remove(0); - } else if pasted.starts_with(|ch: char| ch.is_ascii_digit()) { - let literal = match panic::catch_unwind(|| Literal::from_str(&pasted)) { - Ok(Ok(literal)) => TokenTree::Literal(literal), - Ok(Err(LexError { .. })) | Err(_) => { - return Err(Error::new( - span, - &format!("`{:?}` is not a valid literal", pasted), - )); - } - }; - tokens.extend(iter::once(literal)); - return Ok(tokens); } let ident = match panic::catch_unwind(|| Ident::new(&pasted, span)) {