feat(lexer): Allow including frontmatter with 'tokenize'

Author: epage

crates/parser/src/lexed_str.rs

@@ -11,8 +11,8 @@
 use std::ops;
 
 use rustc_literal_escaper::{
-    EscapeError, Mode, unescape_byte, unescape_byte_str, unescape_c_str, unescape_char,
-    unescape_str,
+    unescape_byte, unescape_byte_str, unescape_c_str, unescape_char, unescape_str, EscapeError,
+    Mode,
 };
 
 use crate::{
@@ -44,7 +44,9 @@ impl<'a> LexedStr<'a> {
 
         // Re-create the tokenizer from scratch every token because `GuardedStrPrefix` is one token in the lexer
         // but we want to split it to two in edition <2024.
-        while let Some(token) = rustc_lexer::tokenize(&text[conv.offset..]).next() {
+        while let Some(token) =
+            rustc_lexer::tokenize(&text[conv.offset..], rustc_lexer::FrontmatterAllowed::No).next()
+        {
             let token_text = &text[conv.offset..][..token.len as usize];
 
             conv.extend_token(&token.kind, token_text);
@@ -58,7 +60,7 @@ impl<'a> LexedStr<'a> {
             return None;
         }
 
-        let token = rustc_lexer::tokenize(text).next()?;
+        let token = rustc_lexer::tokenize(text, rustc_lexer::FrontmatterAllowed::No).next()?;
         if token.len as usize != text.len() {
             return None;
         }

crates/proc-macro-srv/src/server_impl.rs

@@ -121,7 +121,7 @@ pub(super) fn literal_from_str<Span: Copy>(
     use proc_macro::bridge::LitKind;
     use rustc_lexer::{LiteralKind, Token, TokenKind};
 
-    let mut tokens = rustc_lexer::tokenize(s);
+    let mut tokens = rustc_lexer::tokenize(s, rustc_lexer::FrontmatterAllowed::No);
     let minus_or_lit = tokens.next().unwrap_or(Token { kind: TokenKind::Eof, len: 0 });
 
     let lit = if minus_or_lit.kind == TokenKind::Minus {

crates/tt/src/lib.rs

@@ -579,7 +579,7 @@ where
 {
     use rustc_lexer::LiteralKind;
 
-    let token = rustc_lexer::tokenize(text).next_tuple();
+    let token = rustc_lexer::tokenize(text, rustc_lexer::FrontmatterAllowed::No).next_tuple();
     let Some((rustc_lexer::Token {
         kind: rustc_lexer::TokenKind::Literal { kind, suffix_start },
         ..