Fix string literals lexing, parsing and displaying

Author: tyranron

juniper/CHANGELOG.md

@@ -15,6 +15,7 @@ All user visible changes to `juniper` crate will be documented in this file. Thi
 - [September 2025] GraphQL spec: ([#1347])
     - Made `includeDeprecated` argument of `__Type.fields`, `__Type.enumValues`, `__Type.inputFields`, `__Field.args` and `__Directive.args` fields non-`Null`. ([#1348], [graphql/graphql-spec#1142])
     - Made `@deprecated(reason:)` argument non-`Null`. ([#1348], [graphql/graphql-spec#1040])
+- Changed `ScalarToken::String` to contain raw quoted and escaped `StringLiteral` (was unquoted but escaped string before). ([#1349])
 
 ### Added
 
@@ -38,10 +39,12 @@ All user visible changes to `juniper` crate will be documented in this file. Thi
 
 - Incorrect `__Type.specifiedByUrl` field to `__Type.specifiedByURL`. ([#1348])
 - Missing `@specifiedBy(url:)` directive in [SDL] generated by `RootNode::as_sdl()` and `RootNode::as_document()` methods. ([#1348])
+- Incorrect double escaping in `ScalarToken::String` `Display`ing. ([#1349])
 
 [#864]: /../../issues/864
 [#1347]: /../../issues/1347
 [#1348]: /../../pull/1348
+[#1349]: /../../pull/1349
 [graphql/graphql-spec#525]: https://github.com/graphql/graphql-spec/pull/525
 [graphql/graphql-spec#805]: https://github.com/graphql/graphql-spec/pull/805
 [graphql/graphql-spec#825]: https://github.com/graphql/graphql-spec/pull/825

juniper/src/lib.rs

@@ -85,7 +85,7 @@ pub use crate::{
     },
     introspection::IntrospectionFormat,
     macros::helper::subscription::{ExtractTypeFromStream, IntoFieldResult},
-    parser::{ParseError, ScalarToken, Span, Spanning},
+    parser::{ParseError, ScalarToken, Span, Spanning, StringLiteral},
     schema::{
         meta,
         model::{RootNode, SchemaType},

juniper/src/parser/lexer.rs

@@ -1,6 +1,7 @@
 use std::{char, iter::Peekable, ops::Deref, str::CharIndices};
 
 use derive_more::with_trait::{Display, Error};
+//use itertools::Itertools as _;
 
 use crate::parser::{SourcePosition, Spanning};
 
@@ -14,36 +15,34 @@ pub struct Lexer<'a> {
     has_reached_eof: bool,
 }
 
-/// A single scalar value literal
+/// Representation of a raw unparsed scalar value literal.
 ///
 /// This is only used for tagging how the lexer has interpreted a value literal
 #[expect(missing_docs, reason = "self-explanatory")]
 #[derive(Clone, Copy, Debug, Display, Eq, PartialEq)]
 pub enum ScalarToken<'a> {
-    String(StringValue<'a>),
+    String(StringLiteral<'a>),
     Float(&'a str),
     Int(&'a str),
 }
 
-/// Representation of a [String Value].
+/// Representation of a raw unparsed [String Value] literal (with quotes included).
 ///
 /// [String Value]: https://spec.graphql.org/October2021#sec-String-Value
 #[derive(Clone, Copy, Debug, Display, Eq, PartialEq)]
-pub enum StringValue<'a> {
-    /// [Quoted][0] string representation.
+pub enum StringLiteral<'a> {
+    /// [Quoted][0] literal (denoted by single quotes `"`).
     ///
     /// [0]: https://spec.graphql.org/October2021#StringCharacter
-    #[display(r#""{}""#, _0.replace('\\', r"\\").replace('"', r#"\""#))]
     Quoted(&'a str),
 
-    /// [Block][0] string representation.
+    /// [Block][0] literal (denoted by triple quotes `"""`).
     ///
     /// [0]: https://spec.graphql.org/October2021#BlockStringCharacter
-    #[display(r#""""{}""""#, _0.replace(r#"""""#, r#"\""""#))]
     Block(&'a str),
 }
 
-impl Deref for StringValue<'_> {
+impl Deref for StringLiteral<'_> {
     type Target = str;
 
     fn deref(&self) -> &Self::Target {
@@ -115,6 +114,10 @@ pub enum LexerError {
     #[display("Unterminated string literal")]
     UnterminatedString,
 
+    /// An unterminated block string literal was found.
+    #[display("Unterminated block string literal")]
+    UnterminatedBlockString,
+
     /// An unknown character in a string literal was found
     ///
     /// This occurs when an invalid source character is found in a string
@@ -294,8 +297,8 @@ impl<'a> Lexer<'a> {
                     return Ok(Spanning::start_end(
                         &start_pos,
                         &self.position,
-                        Token::Scalar(ScalarToken::String(StringValue::Quoted(
-                            &self.source[start_idx + 1..idx],
+                        Token::Scalar(ScalarToken::String(StringLiteral::Quoted(
+                            &self.source[start_idx..=idx],
                         ))),
                     ));
                 }
@@ -322,6 +325,77 @@ impl<'a> Lexer<'a> {
         ))
     }
 
+    /*
+    fn scan_block_string(&mut self) -> LexerResult<'a> {
+        let start_pos = self.position;
+        let (start_idx, mut start_ch) = self
+            .next_char()
+            .ok_or_else(|| Spanning::zero_width(&self.position, LexerError::UnexpectedEndOfFile))?;
+        if start_ch != '"' {
+            return Err(Spanning::zero_width(
+                &self.position,
+                LexerError::UnterminatedString,
+            ));
+        }
+        for _ in 0..2 {
+            (_, start_ch) = self.next_char().ok_or_else(|| {
+                Spanning::zero_width(&self.position, LexerError::UnexpectedEndOfFile)
+            })?;
+            if start_ch != '"' {
+                return Err(Spanning::zero_width(
+                    &self.position,
+                    LexerError::UnexpectedCharacter(start_ch),
+                ));
+            }
+        }
+
+        let mut quotes = 0;
+        let mut escaped = false;
+        let mut old_pos = self.position;
+        while let Some((idx, ch)) = self.next_char() {
+            match ch {
+                '\\' => escaped = true,
+
+
+                'b' | 'f' | 'n' | 'r' | 't' | '\\' | '/' | '"' if escaped => {
+                    escaped = false;
+                }
+                'u' if escaped => {
+                    self.scan_escaped_unicode(&old_pos)?;
+                    escaped = false;
+                }
+                c if escaped => {
+                    return Err(Spanning::zero_width(
+                        &old_pos,
+                        LexerError::UnknownEscapeSequence(format!("\\{c}")),
+                    ));
+                }
+
+
+
+                '"' if !escaped => {
+                    return Ok(Spanning::start_end(
+                        &start_pos,
+                        &self.position,
+                        Token::Scalar(ScalarToken::String(StringValue::Quoted(
+                            &self.source[start_idx + 1..idx],
+                        ))),
+                    ));
+                }
+
+                _ => {}
+            }
+            old_pos = self.position;
+        }
+
+        Err(Spanning::zero_width(
+            &self.position,
+            LexerError::UnterminatedBlockString,
+        ))
+    }
+
+     */
+
     fn scan_escaped_unicode(
         &mut self,
         start_pos: &SourcePosition,

juniper/src/parser/mod.rs

@@ -13,7 +13,7 @@ mod tests;
 pub use self::document::parse_document_source;
 
 pub use self::{
-    lexer::{Lexer, LexerError, ScalarToken, StringValue, Token},
+    lexer::{Lexer, LexerError, ScalarToken, StringLiteral, Token},
     parser::{OptionParseResult, ParseError, ParseResult, Parser, UnlocatedParseResult},
     utils::{SourcePosition, Span, Spanning},
 };

juniper/src/parser/parser.rs

@@ -1,9 +1,9 @@
-use std::fmt;
+use std::{borrow::Cow, fmt};
 
 use compact_str::{CompactString, format_compact};
 use derive_more::with_trait::{Display, Error};
 
-use crate::parser::{Lexer, LexerError, Spanning, Token};
+use crate::parser::{Lexer, LexerError, ScalarToken, Spanning, StringLiteral, Token};
 
 /// Error while parsing a GraphQL query
 #[derive(Clone, Debug, Display, Eq, Error, PartialEq)]
@@ -199,3 +199,128 @@ impl<'a> Parser<'a> {
         }
     }
 }
+
+impl<'a> StringLiteral<'a> {
+    /// Parses this [`StringLiteral`] returning an unescaped and unquoted string value.
+    ///
+    /// # Errors
+    ///
+    /// If this [`StringLiteral`] is invalid.
+    pub fn parse(self) -> Result<Cow<'a, str>, ParseError> {
+        match self {
+            Self::Quoted(lit) => {
+                if !lit.starts_with('"') {
+                    return Err(ParseError::unexpected_token(Token::Scalar(
+                        ScalarToken::String(self),
+                    )));
+                }
+                if !lit.ends_with('"') {
+                    return Err(ParseError::LexerError(LexerError::UnterminatedString));
+                }
+
+                let unquoted = &lit[1..lit.len() - 1];
+                if !unquoted.contains('\\') {
+                    return Ok(unquoted.into());
+                }
+
+                let mut unescaped = String::with_capacity(unquoted.len());
+                let mut char_iter = unquoted.chars();
+                while let Some(ch) = char_iter.next() {
+                    match ch {
+                        '\\' => match char_iter.next() {
+                            Some('"') => {
+                                unescaped.push('"');
+                            }
+                            Some('/') => {
+                                unescaped.push('/');
+                            }
+                            Some('n') => {
+                                unescaped.push('\n');
+                            }
+                            Some('r') => {
+                                unescaped.push('\r');
+                            }
+                            Some('t') => {
+                                unescaped.push('\t');
+                            }
+                            Some('\\') => {
+                                unescaped.push('\\');
+                            }
+                            Some('f') => {
+                                unescaped.push('\u{000c}');
+                            }
+                            Some('b') => {
+                                unescaped.push('\u{0008}');
+                            }
+                            Some('u') => {
+                                unescaped.push(parse_unicode_codepoint(&mut char_iter)?);
+                            }
+                            Some(s) => {
+                                return Err(ParseError::LexerError(
+                                    LexerError::UnknownEscapeSequence(format!(r"\{s}")),
+                                ));
+                            }
+                            None => {
+                                return Err(ParseError::LexerError(LexerError::UnterminatedString));
+                            }
+                        },
+                        ch => {
+                            unescaped.push(ch);
+                        }
+                    }
+                }
+                Ok(unescaped.into())
+            }
+            Self::Block(_) => todo!(),
+        }
+    }
+}
+
+fn parse_unicode_codepoint<I>(char_iter: &mut I) -> Result<char, ParseError>
+where
+    I: Iterator<Item = char>,
+{
+    let escaped_code_point = char_iter
+        .next()
+        .ok_or_else(|| ParseError::LexerError(LexerError::UnknownEscapeSequence(r"\u".into())))
+        .and_then(|c1| {
+            char_iter
+                .next()
+                .map(|c2| format!("{c1}{c2}"))
+                .ok_or_else(|| {
+                    ParseError::LexerError(LexerError::UnknownEscapeSequence(format!(r"\u{c1}")))
+                })
+        })
+        .and_then(|mut s| {
+            char_iter
+                .next()
+                .ok_or_else(|| {
+                    ParseError::LexerError(LexerError::UnknownEscapeSequence(format!(r"\u{s}")))
+                })
+                .map(|c2| {
+                    s.push(c2);
+                    s
+                })
+        })
+        .and_then(|mut s| {
+            char_iter
+                .next()
+                .ok_or_else(|| {
+                    ParseError::LexerError(LexerError::UnknownEscapeSequence(format!(r"\u{s}")))
+                })
+                .map(|c2| {
+                    s.push(c2);
+                    s
+                })
+        })?;
+    let code_point = u32::from_str_radix(&escaped_code_point, 16).map_err(|_| {
+        ParseError::LexerError(LexerError::UnknownEscapeSequence(format!(
+            r"\u{escaped_code_point}",
+        )))
+    })?;
+    char::from_u32(code_point).ok_or_else(|| {
+        ParseError::LexerError(LexerError::UnknownEscapeSequence(format!(
+            r"\u{escaped_code_point}",
+        )))
+    })
+}