Skip to content

Commit 4b479f0

Browse files
tyranrontyranron
committed
Bootstrap block string in lexer
1 parent 90ecd03 commit 4b479f0

File tree

5 files changed

+80
-22
lines changed

5 files changed

+80
-22
lines changed

‎juniper/src/ast.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -384,6 +384,7 @@ pub enum OperationType {
384384
#[expect(missing_docs, reason = "self-explanatory")]
385385
#[derive(Clone, Debug, PartialEq)]
386386
pub struct Operation<'a, S> {
387+
//pub description: Option<Spanning<&'a str>>,
387388
pub operation_type: OperationType,
388389
pub name: Option<Spanning<&'a str>>,
389390
pub variable_definitions: Option<Spanning<VariableDefinitions<'a, S>>>,

‎juniper/src/parser/lexer.rs‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use std::{char, iter::Peekable, str::CharIndices};
1+
use std::{char, iter::Peekable, ops::Deref, str::CharIndices};
22

33
use derive_more::with_trait::{Display, Error};
44

@@ -20,12 +20,40 @@ pub struct Lexer<'a> {
2020
#[expect(missing_docs, reason = "self-explanatory")]
2121
#[derive(Clone, Copy, Debug, Display, Eq, PartialEq)]
2222
pub enum ScalarToken<'a> {
23-
#[display("\"{}\"", _0.replace('\\', "\\\\").replace('"', "\\\""))]
24-
String(&'a str),
23+
String(StringValue<'a>),
2524
Float(&'a str),
2625
Int(&'a str),
2726
}
2827

28+
/// Representation of a [String Value].
29+
///
30+
/// [String Value]: https://spec.graphql.org/October2021#sec-String-Value
31+
#[derive(Clone, Copy, Debug, Display, Eq, PartialEq)]
32+
pub enum StringValue<'a> {
33+
/// [Quoted][0] string representation.
34+
///
35+
/// [0]: https://spec.graphql.org/October2021#StringCharacter
36+
#[display(r#""{}""#, _0.replace('\\', r"\\").replace('"', r#"\""#))]
37+
Quoted(&'a str),
38+
39+
/// [Block][0] string representation.
40+
///
41+
/// [0]: https://spec.graphql.org/October2021#BlockStringCharacter
42+
#[display(r#""""{}""""#, _0.replace(r#"""""#, r#"\""""#))]
43+
Block(&'a str),
44+
}
45+
46+
impl Deref for StringValue<'_> {
47+
type Target = str;
48+
49+
fn deref(&self) -> &Self::Target {
50+
match self {
51+
Self::Quoted(s) => s,
52+
Self::Block(s) => s,
53+
}
54+
}
55+
}
56+
2957
/// A single token in the input source
3058
#[expect(missing_docs, reason = "self-explanatory")]
3159
#[derive(Clone, Copy, Debug, Display, Eq, PartialEq)]
@@ -266,7 +294,9 @@ impl<'a> Lexer<'a> {
266294
return Ok(Spanning::start_end(
267295
&start_pos,
268296
&self.position,
269-
Token::Scalar(ScalarToken::String(&self.source[start_idx + 1..idx])),
297+
Token::Scalar(ScalarToken::String(StringValue::Quoted(
298+
&self.source[start_idx + 1..idx],
299+
))),
270300
));
271301
}
272302
'\n' | '\r' => {

‎juniper/src/parser/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ mod tests;
1313
pub use self::document::parse_document_source;
1414

1515
pub use self::{
16-
lexer::{Lexer, LexerError, ScalarToken, Token},
16+
lexer::{Lexer, LexerError, ScalarToken, StringValue, Token},
1717
parser::{OptionParseResult, ParseError, ParseResult, Parser, UnlocatedParseResult},
1818
utils::{SourcePosition, Span, Spanning},
1919
};

‎juniper/src/parser/tests/lexer.rs‎

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use crate::parser::{Lexer, LexerError, ScalarToken, SourcePosition, Spanning, Token};
1+
use crate::parser::{Lexer, LexerError, ScalarToken, SourcePosition, Spanning, StringValue, Token};
22

33
fn tokenize_to_vec(s: &str) -> Vec<Spanning<Token<'_>>> {
44
let mut tokens = Vec::new();
@@ -21,6 +21,7 @@ fn tokenize_to_vec(s: &str) -> Vec<Spanning<Token<'_>>> {
2121
tokens
2222
}
2323

24+
#[track_caller]
2425
fn tokenize_single(s: &str) -> Spanning<Token<'_>> {
2526
let mut tokens = tokenize_to_vec(s);
2627

@@ -151,7 +152,7 @@ fn strings() {
151152
Spanning::start_end(
152153
&SourcePosition::new(0, 0, 0),
153154
&SourcePosition::new(8, 0, 8),
154-
Token::Scalar(ScalarToken::String("simple"))
155+
Token::Scalar(ScalarToken::String(StringValue::Quoted("simple")))
155156
)
156157
);
157158

@@ -160,7 +161,7 @@ fn strings() {
160161
Spanning::start_end(
161162
&SourcePosition::new(0, 0, 0),
162163
&SourcePosition::new(15, 0, 15),
163-
Token::Scalar(ScalarToken::String(" white space "))
164+
Token::Scalar(ScalarToken::String(StringValue::Quoted(" white space ")))
164165
)
165166
);
166167

@@ -169,7 +170,7 @@ fn strings() {
169170
Spanning::start_end(
170171
&SourcePosition::new(0, 0, 0),
171172
&SourcePosition::new(10, 0, 10),
172-
Token::Scalar(ScalarToken::String(r#"quote \""#))
173+
Token::Scalar(ScalarToken::String(StringValue::Quoted(r#"quote \""#)))
173174
)
174175
);
175176

@@ -178,7 +179,9 @@ fn strings() {
178179
Spanning::start_end(
179180
&SourcePosition::new(0, 0, 0),
180181
&SourcePosition::new(20, 0, 20),
181-
Token::Scalar(ScalarToken::String(r"escaped \n\r\b\t\f"))
182+
Token::Scalar(ScalarToken::String(StringValue::Quoted(
183+
r"escaped \n\r\b\t\f"
184+
)))
182185
)
183186
);
184187

@@ -187,7 +190,7 @@ fn strings() {
187190
Spanning::start_end(
188191
&SourcePosition::new(0, 0, 0),
189192
&SourcePosition::new(15, 0, 15),
190-
Token::Scalar(ScalarToken::String(r"slashes \\ \/"))
193+
Token::Scalar(ScalarToken::String(StringValue::Quoted(r"slashes \\ \/")))
191194
)
192195
);
193196

@@ -196,7 +199,21 @@ fn strings() {
196199
Spanning::start_end(
197200
&SourcePosition::new(0, 0, 0),
198201
&SourcePosition::new(34, 0, 34),
199-
Token::Scalar(ScalarToken::String(r"unicode \u1234\u5678\u90AB\uCDEF")),
202+
Token::Scalar(ScalarToken::String(StringValue::Quoted(
203+
r"unicode \u1234\u5678\u90AB\uCDEF"
204+
))),
205+
)
206+
);
207+
}
208+
209+
#[test]
210+
fn block_strings() {
211+
assert_eq!(
212+
tokenize_single(r#""""""#),
213+
Spanning::start_end(
214+
&SourcePosition::new(0, 0, 0),
215+
&SourcePosition::new(8, 0, 8),
216+
Token::Scalar(ScalarToken::String(StringValue::Block("".into()))),
200217
)
201218
);
202219
}
@@ -665,13 +682,16 @@ fn display() {
665682
(Token::Scalar(ScalarToken::Int("123")), "123"),
666683
(Token::Scalar(ScalarToken::Float("4.5")), "4.5"),
667684
(
668-
Token::Scalar(ScalarToken::String("some string")),
685+
Token::Scalar(ScalarToken::String(StringValue::Quoted("some string"))),
669686
"\"some string\"",
670687
),
671688
(
672-
Token::Scalar(ScalarToken::String("string with \\ escape and \" quote")),
689+
Token::Scalar(ScalarToken::String(StringValue::Quoted(
690+
"string with \\ escape and \" quote",
691+
))),
673692
"\"string with \\\\ escape and \\\" quote\"",
674693
),
694+
// TODO: tests for block string
675695
(Token::ExclamationMark, "!"),
676696
(Token::Dollar, "$"),
677697
(Token::ParenOpen, "("),

‎juniper/src/types/scalars.rs‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ type String = std::string::String;
6868

6969
mod impl_string_scalar {
7070
use super::*;
71+
use crate::parser::StringValue;
7172

7273
impl<'s, S> FromScalarValue<'s, S> for String
7374
where
@@ -81,7 +82,7 @@ mod impl_string_scalar {
8182
}
8283

8384
pub(super) fn parse_token<S: ScalarValue>(value: ScalarToken<'_>) -> ParseScalarResult<S> {
84-
if let ScalarToken::String(value) = value {
85+
if let ScalarToken::String(StringValue::Quoted(value)) = value {
8586
let mut ret = String::with_capacity(value.len());
8687
let mut char_iter = value.chars();
8788
while let Some(ch) = char_iter.next() {
@@ -128,6 +129,7 @@ mod impl_string_scalar {
128129
}
129130
Ok(ret.into())
130131
} else {
132+
// TODO: Support block string too.
131133
Err(ParseError::unexpected_token(Token::Scalar(value)))
132134
}
133135
}
@@ -571,7 +573,9 @@ mod tests {
571573

572574
#[test]
573575
fn parse_strings() {
574-
fn parse_string(s: &str, expected: &str) {
576+
use crate::parser::StringValue::{self, Quoted};
577+
578+
fn parse_string(s: StringValue<'_>, expected: &str) {
575579
let s =
576580
<String as ParseScalarValue<DefaultScalarValue>>::from_str(ScalarToken::String(s));
577581
assert!(s.is_ok(), "A parsing error occurred: {s:?}");
@@ -580,13 +584,16 @@ mod tests {
580584
assert_eq!(s.unwrap(), expected);
581585
}
582586

583-
parse_string("simple", "simple");
584-
parse_string(" white space ", " white space ");
585-
parse_string(r#"quote \""#, "quote \"");
586-
parse_string(r"escaped \n\r\b\t\f", "escaped \n\r\u{0008}\t\u{000c}");
587-
parse_string(r"slashes \\ \/", "slashes \\ /");
587+
parse_string(Quoted("simple"), "simple");
588+
parse_string(Quoted(" white space "), " white space ");
589+
parse_string(Quoted(r#"quote \""#), "quote \"");
590+
parse_string(
591+
Quoted(r"escaped \n\r\b\t\f"),
592+
"escaped \n\r\u{0008}\t\u{000c}",
593+
);
594+
parse_string(Quoted(r"slashes \\ \/"), "slashes \\ /");
588595
parse_string(
589-
r"unicode \u1234\u5678\u90AB\uCDEF",
596+
Quoted(r"unicode \u1234\u5678\u90AB\uCDEF"),
590597
"unicode \u{1234}\u{5678}\u{90ab}\u{cdef}",
591598
);
592599
}

0 commit comments

Comments
 (0)