Auto merge of rust-lang#87264 - mystor:expand_literal, r=petrochenkov

proc_macro: Add an expand_expr method to TokenStream

This feature is aimed at giving proc macros access to powers similar to those used by builtin macros such as `format_args!` or `concat!`. These macros are able to accept macros in place of string literal parameters, such as the format string, as they perform recursive macro expansion while being expanded.

This can be especially useful in many cases thanks to helper macros like `concat!`, `stringify!` and `include_str!` which are often used to construct string literals at compile-time in user code.

For now, this method only allows expanding macros which produce literals, although more expressions will be supported before the method is stabilized.

In earlier versions of this PR, this method exclusively returned `Literal`, and spans on returned literals were stripped of expansion context before being returned to be as conservative as possible about permission leakage. The method's naming has been generalized to eventually support arbitrary expressions, and the context stripping has been removed (rust-lang#87264 (comment)), which should allow for more general APIs like "format_args_implicits" (rust-lang#67984) to be supported as well.

## API Surface

```rust
impl TokenStream {
    pub fn expand_expr(&self) -> Result<TokenStream, ExpandError>;
}

#[non_exhaustive]
pub struct ExpandError;

impl Debug for ExpandError { ... }
impl Display for ExpandError { ... }
impl Error for ExpandError {}
impl !Send for ExpandError {}
impl !Sync for ExpandError {}
```

Author: bors

proc_macro/src/bridge/mod.rs

@@ -62,6 +62,7 @@ macro_rules! with_api {
                 fn clone($self: &$S::TokenStream) -> $S::TokenStream;
                 fn new() -> $S::TokenStream;
                 fn is_empty($self: &$S::TokenStream) -> bool;
+                fn expand_expr($self: &$S::TokenStream) -> Result<$S::TokenStream, ()>;
                 fn from_str(src: &str) -> $S::TokenStream;
                 fn to_string($self: &$S::TokenStream) -> String;
                 fn from_token_tree(

proc_macro/src/lib.rs

@@ -88,12 +88,6 @@ impl !Sync for TokenStream {}
 #[derive(Debug)]
 pub struct LexError;
 
-impl LexError {
-    fn new() -> Self {
-        LexError
-    }
-}
-
 #[stable(feature = "proc_macro_lexerror_impls", since = "1.44.0")]
 impl fmt::Display for LexError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -109,6 +103,28 @@ impl !Send for LexError {}
 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
 impl !Sync for LexError {}
 
+/// Error returned from `TokenStream::expand_expr`.
+#[unstable(feature = "proc_macro_expand", issue = "90765")]
+#[non_exhaustive]
+#[derive(Debug)]
+pub struct ExpandError;
+
+#[unstable(feature = "proc_macro_expand", issue = "90765")]
+impl fmt::Display for ExpandError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str("macro expansion failed")
+    }
+}
+
+#[unstable(feature = "proc_macro_expand", issue = "90765")]
+impl error::Error for ExpandError {}
+
+#[unstable(feature = "proc_macro_expand", issue = "90765")]
+impl !Send for ExpandError {}
+
+#[unstable(feature = "proc_macro_expand", issue = "90765")]
+impl !Sync for ExpandError {}
+
 impl TokenStream {
     /// Returns an empty `TokenStream` containing no token trees.
     #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
@@ -121,6 +137,24 @@ impl TokenStream {
     pub fn is_empty(&self) -> bool {
         self.0.is_empty()
     }
+
+    /// Parses this `TokenStream` as an expression and attempts to expand any
+    /// macros within it. Returns the expanded `TokenStream`.
+    ///
+    /// Currently only expressions expanding to literals will succeed, although
+    /// this may be relaxed in the future.
+    ///
+    /// NOTE: In error conditions, `expand_expr` may leave macros unexpanded,
+    /// report an error, failing compilation, and/or return an `Err(..)`. The
+    /// specific behavior for any error condition, and what conditions are
+    /// considered errors, is unspecified and may change in the future.
+    #[unstable(feature = "proc_macro_expand", issue = "90765")]
+    pub fn expand_expr(&self) -> Result<TokenStream, ExpandError> {
+        match bridge::client::TokenStream::expand_expr(&self.0) {
+            Ok(stream) => Ok(TokenStream(stream)),
+            Err(_) => Err(ExpandError),
+        }
+    }
 }
 
 /// Attempts to break the string into tokens and parse those tokens into a token stream.
@@ -1211,7 +1245,7 @@ impl FromStr for Literal {
     fn from_str(src: &str) -> Result<Self, LexError> {
         match bridge::client::Literal::from_str(src) {
             Ok(literal) => Ok(Literal(literal)),
-            Err(()) => Err(LexError::new()),
+            Err(()) => Err(LexError),
         }
     }
 }