From cf843c471f94d3606dd317ec56a3126f1e1c03f6 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 30 Mar 2026 13:41:45 +0530 Subject: [PATCH 1/3] fix: preserve exact text for unquoted string values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scanner was eagerly tokenizing values (breaking on spaces, parsing numbers) instead of treating them as complete text before type inference, as the spec requires (§B.3, §B.4, §B.5). This caused three bugs: - #59: multiple spaces collapsed (`a b` → `a b`) - #60: mixed-type tokens errored (`1 null`, `a 1` in tabular rows) - #61: number formatting lost (`1.0 b` → `1 b`, `1e1 b` → `10 b`) Scanner changes: - Track `last_whitespace_count` and `last_token_text` through scanning - `read_rest_of_line_with_space_info()` returns exact space count - Add `read_until_delimiter_with_space_info()` for tabular cells Parser changes: - `parse_field_value()`: use original token text and exact space count - `parse_tabular_field_value()`: read complete cell text then type-infer - `parse_value_with_depth()`: handle all value token types in root-level concatenation with exact spacing Fixes #59, #60, #61 --- src/decode/parser.rs | 323 ++++++++++++++++++++++++++++++------------ src/decode/scanner.rs | 97 ++++++++++--- 2 files changed, 310 insertions(+), 110 deletions(-) diff --git a/src/decode/parser.rs b/src/decode/parser.rs index 0bcfb8d..ae328df 100644 --- a/src/decode/parser.rs +++ b/src/decode/parser.rs @@ -149,9 +149,40 @@ impl<'a> Parser<'a> { self.advance()?; self.parse_object_with_initial_key(key, depth) } else { + let first_text = self.scanner.last_token_text().to_string(); let val = *i; self.advance()?; - Ok(serde_json::Number::from(val).into()) + // Check if followed by more value tokens on the same line + match &self.current_token { + Token::String(..) + | Token::Integer(..) + | Token::Number(..) + | Token::Bool(..) + | Token::Null => { + let mut accumulated = first_text; + loop { + match &self.current_token { + Token::String(..) + | Token::Integer(..) + | Token::Number(..) + | Token::Bool(..) + | Token::Null => { + let ws = + self.scanner.last_whitespace_count().max(1); + for _ in 0..ws { + accumulated.push(' '); + } + accumulated + .push_str(self.scanner.last_token_text()); + self.advance()?; + } + _ => break, + } + } + Ok(Value::String(accumulated)) + } + _ => Ok(serde_json::Number::from(val).into()), + } } } Token::Number(n) => { @@ -161,17 +192,55 @@ impl<'a> Parser<'a> { self.advance()?; self.parse_object_with_initial_key(key, depth) } else { + let first_text = self.scanner.last_token_text().to_string(); let val = *n; self.advance()?; - // Normalize floats that are actually integers - if val.is_finite() && val.fract() == 0.0 && val.abs() <= i64::MAX as f64 { - Ok(serde_json::Number::from(val as i64).into()) - } else { - Ok(serde_json::Number::from_f64(val) - .ok_or_else(|| { - ToonError::InvalidInput(format!("Invalid number: {val}")) - })? - .into()) + // Check if followed by more value tokens on the same line + match &self.current_token { + Token::String(..) + | Token::Integer(..) + | Token::Number(..) + | Token::Bool(..) + | Token::Null => { + let mut accumulated = first_text; + loop { + match &self.current_token { + Token::String(..) + | Token::Integer(..) + | Token::Number(..) + | Token::Bool(..) + | Token::Null => { + let ws = + self.scanner.last_whitespace_count().max(1); + for _ in 0..ws { + accumulated.push(' '); + } + accumulated + .push_str(self.scanner.last_token_text()); + self.advance()?; + } + _ => break, + } + } + Ok(Value::String(accumulated)) + } + _ => { + // Normalize floats that are actually integers + if val.is_finite() + && val.fract() == 0.0 + && val.abs() <= i64::MAX as f64 + { + Ok(serde_json::Number::from(val as i64).into()) + } else { + Ok(serde_json::Number::from_f64(val) + .ok_or_else(|| { + ToonError::InvalidInput(format!( + "Invalid number: {val}" + )) + })? + .into()) + } + } } } } @@ -197,14 +266,27 @@ impl<'a> Parser<'a> { )); } - // Root-level string value - join consecutive tokens + if matches!(self.current_token, Token::Newline | Token::Eof) { + return Ok(Value::String(first)); + } + // Root-level string value - join consecutive tokens with exact spacing let mut accumulated = first; - while let Token::String(next, _) = &self.current_token { - if !accumulated.is_empty() { - accumulated.push(' '); + loop { + match &self.current_token { + Token::String(..) + | Token::Integer(..) + | Token::Number(..) + | Token::Bool(..) + | Token::Null => { + let ws = self.scanner.last_whitespace_count().max(1); + for _ in 0..ws { + accumulated.push(' '); + } + accumulated.push_str(self.scanner.last_token_text()); + self.advance()?; + } + _ => break, } - accumulated.push_str(next); - self.advance()?; } Ok(Value::String(accumulated)) } @@ -433,9 +515,10 @@ impl<'a> Parser<'a> { self.parse_value_with_depth(depth + 1) } else { // Check if there's more content after the current token - let (rest, had_space) = self.scanner.read_rest_of_line_with_space_info(); + let token_text = self.scanner.last_token_text().to_string(); + let (rest, space_count) = self.scanner.read_rest_of_line_with_space_info(); - let result = if rest.is_empty() { + let result = if rest.is_empty() && space_count == 0 { // Single token - convert directly to avoid redundant parsing match &self.current_token { Token::String(s, _) => Ok(Value::String(s.clone())), @@ -457,28 +540,24 @@ impl<'a> Parser<'a> { _ => Err(self.parse_error_with_context("Unexpected token after colon")), } } else { - // Multi-token value - reconstruct and re-parse as complete string - let mut value_str = String::new(); - - match &self.current_token { - Token::String(s, true) => { - // Quoted strings need quotes preserved for re-parsing - value_str.push('"'); - value_str.push_str(&crate::utils::escape_string(s)); - value_str.push('"'); + // Multi-token value - reconstruct using original token text and re-parse + let mut value_str = match &self.current_token { + Token::String(_, true) => { + // Quoted strings: use last_token_text which includes quotes + token_text.clone() } - Token::String(s, false) => value_str.push_str(s), - Token::Integer(i) => value_str.push_str(&i.to_string()), - Token::Number(n) => value_str.push_str(&n.to_string()), - Token::Bool(b) => value_str.push_str(if *b { "true" } else { "false" }), - Token::Null => value_str.push_str("null"), + Token::String(_, false) + | Token::Integer(_) + | Token::Number(_) + | Token::Bool(_) + | Token::Null => token_text.clone(), _ => { return Err(self.parse_error_with_context("Unexpected token after colon")); } - } + }; - // Only add space if there was whitespace in the original input - if had_space { + // Preserve exact spacing from the original input + for _ in 0..space_count { value_str.push(' '); } value_str.push_str(&rest); @@ -1112,71 +1191,67 @@ impl<'a> Parser<'a> { } fn parse_tabular_field_value(&mut self) -> ToonResult { - match &self.current_token { - Token::Null => { - self.advance()?; - Ok(Value::Null) - } - Token::Bool(b) => { - let val = *b; - self.advance()?; - Ok(Value::Bool(val)) - } - Token::Integer(i) => { - let val = *i; - self.advance()?; - // If followed by string tokens, treat the whole value as a string - if let Token::String(..) = &self.current_token { - let mut accumulated = val.to_string(); - while let Token::String(next, _) = &self.current_token { - accumulated.push(' '); - accumulated.push_str(next); - self.advance()?; - } - Ok(Value::String(accumulated)) - } else { - Ok(Number::from(val).into()) - } - } - Token::Number(n) => { - let val = *n; - self.advance()?; - // If followed by string tokens, treat the whole value as a string - if let Token::String(..) = &self.current_token { - let mut accumulated = val.to_string(); - while let Token::String(next, _) = &self.current_token { - accumulated.push(' '); - accumulated.push_str(next); - self.advance()?; + // Get the original text of the current token + let token_text = self.scanner.last_token_text().to_string(); + + // Read remaining text until delimiter/newline/EOF + let (rest, space_count) = self.scanner.read_until_delimiter_with_space_info(); + + if rest.is_empty() && space_count == 0 { + // Single token — handle as primitive directly + let result = match &self.current_token { + Token::Null => Ok(Value::Null), + Token::Bool(b) => Ok(Value::Bool(*b)), + Token::Integer(i) => Ok(Number::from(*i).into()), + Token::Number(n) => { + let val = *n; + if val.is_finite() && val.fract() == 0.0 && val.abs() <= i64::MAX as f64 { + Ok(Number::from(val as i64).into()) + } else { + Ok(Number::from_f64(val) + .ok_or_else(|| { + ToonError::InvalidInput(format!("Invalid number: {val}")) + })? + .into()) } - Ok(Value::String(accumulated)) - } else if val.is_finite() && val.fract() == 0.0 && val.abs() <= i64::MAX as f64 { - Ok(Number::from(val as i64).into()) - } else { - Ok(Number::from_f64(val) - .ok_or_else(|| ToonError::InvalidInput(format!("Invalid number: {val}")))? - .into()) } + Token::String(s, _) => Ok(Value::String(s.clone())), + _ => Err(self.parse_error_with_context(format!( + "Expected primitive value, found {:?}", + self.current_token + ))), + }; + self.advance()?; + result + } else { + // Multiple tokens — combine original text + spaces + rest, then type-infer + let mut value_str = token_text; + for _ in 0..space_count { + value_str.push(' '); } - Token::String(s, _) => { - // Tabular fields can have multiple string tokens joined with spaces - let mut accumulated = s.clone(); - self.advance()?; + value_str.push_str(&rest); - while let Token::String(next, _) = &self.current_token { - if !accumulated.is_empty() { - accumulated.push(' '); + let token = self.scanner.parse_value_string(&value_str)?; + // Rescan so current_token is positioned at the next delimiter/newline + self.current_token = self.scanner.scan_token()?; + match token { + Token::String(s, _) => Ok(Value::String(s)), + Token::Integer(i) => Ok(Number::from(i).into()), + Token::Number(n) => { + if n.is_finite() && n.fract() == 0.0 && n.abs() <= i64::MAX as f64 { + Ok(Number::from(n as i64).into()) + } else { + Ok(Number::from_f64(n) + .ok_or_else(|| { + ToonError::InvalidInput(format!("Invalid number: {n}")) + })? + .into()) } - accumulated.push_str(next); - self.advance()?; } - - Ok(Value::String(accumulated)) + Token::Bool(b) => Ok(Value::Bool(b)), + Token::Null => Ok(Value::Null), + _ => Err(ToonError::InvalidInput("Unexpected token type".to_string())), } - _ => Err(self.parse_error_with_context(format!( - "Expected primitive value, found {:?}", - self.current_token - ))), } } @@ -1695,7 +1770,7 @@ hello: 0(f)"#; // Issue #56: Array elements starting with a number should be parsed as string // when followed by non-numeric text let result = parse("version1[1]: 1.0 something").unwrap(); - assert_eq!(result["version1"], json!(["1 something"])); + assert_eq!(result["version1"], json!(["1.0 something"])); let result = parse("data[1]: 42 units").unwrap(); assert_eq!(result["data"], json!(["42 units"])); @@ -1707,4 +1782,64 @@ hello: 0(f)"#; let result = parse("nums[1]: 2.75").unwrap(); assert_eq!(result["nums"], json!([2.75])); } + + #[test] + fn test_issue_59_multiple_spaces_preserved() { + // Issue #59: Multiple spaces between words should be preserved + // Field value context + let result = parse("key: a b").unwrap(); + assert_eq!(result["key"], json!("a b")); + + // Tabular cell context + let result = parse("data[2]: a b, c d").unwrap(); + assert_eq!(result["data"], json!(["a b", "c d"])); + + // Root-level value + let result = parse("a b").unwrap(); + assert_eq!(result, json!("a b")); + } + + #[test] + fn test_issue_60_mixed_type_tokens_as_string() { + // Issue #60: "1 null" and "a 1" should parse as strings in tabular rows + // Tabular cell context + let result = parse("data[2]: 1 null, a 1").unwrap(); + assert_eq!(result["data"], json!(["1 null", "a 1"])); + + // Root-level value + let result = parse("1 null").unwrap(); + assert_eq!(result, json!("1 null")); + + let result = parse("a 1").unwrap(); + assert_eq!(result, json!("a 1")); + + // Field value context + let result = parse("key: 1 null").unwrap(); + assert_eq!(result["key"], json!("1 null")); + + let result = parse("key: a 1").unwrap(); + assert_eq!(result["key"], json!("a 1")); + } + + #[test] + fn test_issue_61_number_format_preserved() { + // Issue #61: "1.0 b" should preserve "1.0", not become "1 b" + // Tabular cell context + let result = parse("data[2]: 1.0 b, 1e1 b").unwrap(); + assert_eq!(result["data"], json!(["1.0 b", "1e1 b"])); + + // Field value context + let result = parse("key: 1.0 b").unwrap(); + assert_eq!(result["key"], json!("1.0 b")); + + let result = parse("key: 1e1 b").unwrap(); + assert_eq!(result["key"], json!("1e1 b")); + + // Root-level value + let result = parse("1.0 b").unwrap(); + assert_eq!(result, json!("1.0 b")); + + let result = parse("1e1 b").unwrap(); + assert_eq!(result, json!("1e1 b")); + } } diff --git a/src/decode/scanner.rs b/src/decode/scanner.rs index df9c943..bef103a 100644 --- a/src/decode/scanner.rs +++ b/src/decode/scanner.rs @@ -31,6 +31,8 @@ pub struct Scanner { column: usize, active_delimiter: Option, last_line_indent: usize, + last_whitespace_count: usize, + last_token_text: String, } impl Scanner { @@ -43,6 +45,8 @@ impl Scanner { column: 1, active_delimiter: None, last_line_indent: 0, + last_whitespace_count: 0, + last_token_text: String::new(), } } @@ -120,8 +124,10 @@ impl Scanner { } pub fn skip_whitespace(&mut self) { + self.last_whitespace_count = 0; while let Some(ch) = self.peek() { if ch == ' ' { + self.last_whitespace_count += 1; self.advance(); } else { break; @@ -129,6 +135,14 @@ impl Scanner { } } + pub fn last_whitespace_count(&self) -> usize { + self.last_whitespace_count + } + + pub fn last_token_text(&self) -> &str { + &self.last_token_text + } + /// Scan the next token from the input. pub fn scan_token(&mut self) -> ToonResult { if self.column == 1 { @@ -156,7 +170,7 @@ impl Scanner { self.skip_whitespace(); - match self.peek() { + let token = match self.peek() { None => Ok(Token::Eof), Some('\n') => { self.advance(); @@ -164,22 +178,27 @@ impl Scanner { } Some('[') => { self.advance(); + self.last_token_text = "[".to_string(); Ok(Token::LeftBracket) } Some(']') => { self.advance(); + self.last_token_text = "]".to_string(); Ok(Token::RightBracket) } Some('{') => { self.advance(); + self.last_token_text = "{".to_string(); Ok(Token::LeftBrace) } Some('}') => { self.advance(); + self.last_token_text = "}".to_string(); Ok(Token::RightBrace) } Some(':') => { self.advance(); + self.last_token_text = ":".to_string(); Ok(Token::Colon) } Some('-') => { @@ -187,15 +206,18 @@ impl Scanner { if let Some(ch) = self.peek() { if ch.is_ascii_digit() { let num_str = self.scan_number_string(true)?; + self.last_token_text = num_str.clone(); return self.parse_number(&num_str); } } + self.last_token_text = "-".to_string(); Ok(Token::Dash) } Some(',') => { // Delimiter only when active, otherwise part of unquoted string if matches!(self.active_delimiter, Some(Delimiter::Comma)) { self.advance(); + self.last_token_text = ",".to_string(); Ok(Token::Delimiter(Delimiter::Comma)) } else { self.scan_unquoted_string() @@ -204,6 +226,7 @@ impl Scanner { Some('|') => { if matches!(self.active_delimiter, Some(Delimiter::Pipe)) { self.advance(); + self.last_token_text = "|".to_string(); Ok(Token::Delimiter(Delimiter::Pipe)) } else { self.scan_unquoted_string() @@ -212,6 +235,7 @@ impl Scanner { Some('\t') => { if matches!(self.active_delimiter, Some(Delimiter::Tab)) { self.advance(); + self.last_token_text = "\t".to_string(); Ok(Token::Delimiter(Delimiter::Tab)) } else { self.scan_unquoted_string() @@ -220,10 +244,13 @@ impl Scanner { Some('"') => self.scan_quoted_string(), Some(ch) if ch.is_ascii_digit() => { let num_str = self.scan_number_string(false)?; + self.last_token_text = num_str.clone(); self.parse_number(&num_str) } Some(_) => self.scan_unquoted_string(), - } + }; + + token } fn scan_quoted_string(&mut self) -> ToonResult { @@ -253,6 +280,7 @@ impl Scanner { } else if ch == '\\' { escaped = true; } else if ch == '"' { + self.last_token_text = format!("\"{}\"", crate::utils::escape_string(&value)); return Ok(Token::String(value, true)); } else { value.push(ch); @@ -297,6 +325,7 @@ impl Scanner { value.trim_end().to_string() }; + self.last_token_text = value.clone(); match value.as_str() { "null" => Ok(Token::Null), "true" => Ok(Token::Bool(true)), @@ -380,11 +409,13 @@ impl Scanner { } /// Read the rest of the current line (until newline or EOF). - /// Returns the content with a flag indicating if it started with - /// whitespace. - pub fn read_rest_of_line_with_space_info(&mut self) -> (String, bool) { - let had_leading_space = matches!(self.peek(), Some(' ')); - self.skip_whitespace(); + /// Returns the content with the count of leading spaces consumed. + pub fn read_rest_of_line_with_space_info(&mut self) -> (String, usize) { + let mut space_count = 0; + while let Some(' ') = self.peek() { + space_count += 1; + self.advance(); + } let mut result = String::new(); while let Some(ch) = self.peek() { @@ -395,7 +426,7 @@ impl Scanner { self.advance(); } - (result.trim_end().to_string(), had_leading_space) + (result.trim_end().to_string(), space_count) } /// Read the rest of the current line (until newline or EOF). @@ -403,6 +434,35 @@ impl Scanner { self.read_rest_of_line_with_space_info().0 } + /// Read raw text until the next active delimiter, newline, or EOF. + /// Returns the content with the count of leading spaces consumed. + pub fn read_until_delimiter_with_space_info(&mut self) -> (String, usize) { + let mut space_count = 0; + while let Some(' ') = self.peek() { + space_count += 1; + self.advance(); + } + + let mut result = String::new(); + while let Some(ch) = self.peek() { + if ch == '\n' { + break; + } + if let Some(active) = self.active_delimiter { + if (active == Delimiter::Comma && ch == ',') + || (active == Delimiter::Pipe && ch == '|') + || (active == Delimiter::Tab && ch == '\t') + { + break; + } + } + result.push(ch); + self.advance(); + } + + (result.trim_end().to_string(), space_count) + } + /// Parse a complete value string into a token. pub fn parse_value_string(&self, s: &str) -> ToonResult { let trimmed = s.trim(); @@ -608,24 +668,29 @@ mod tests { #[test] fn test_read_rest_of_line_with_space_info() { let mut scanner = Scanner::new(" world"); - let (content, had_space) = scanner.read_rest_of_line_with_space_info(); + let (content, space_count) = scanner.read_rest_of_line_with_space_info(); assert_eq!(content, "world"); - assert!(had_space); + assert_eq!(space_count, 1); let mut scanner = Scanner::new("world"); - let (content, had_space) = scanner.read_rest_of_line_with_space_info(); + let (content, space_count) = scanner.read_rest_of_line_with_space_info(); assert_eq!(content, "world"); - assert!(!had_space); + assert_eq!(space_count, 0); let mut scanner = Scanner::new("(hello)"); - let (content, had_space) = scanner.read_rest_of_line_with_space_info(); + let (content, space_count) = scanner.read_rest_of_line_with_space_info(); assert_eq!(content, "(hello)"); - assert!(!had_space); + assert_eq!(space_count, 0); let mut scanner = Scanner::new(""); - let (content, had_space) = scanner.read_rest_of_line_with_space_info(); + let (content, space_count) = scanner.read_rest_of_line_with_space_info(); assert_eq!(content, ""); - assert!(!had_space); + assert_eq!(space_count, 0); + + let mut scanner = Scanner::new(" world"); + let (content, space_count) = scanner.read_rest_of_line_with_space_info(); + assert_eq!(content, "world"); + assert_eq!(space_count, 3); } #[test] From f4d0088de1ef14924561dd5a2e58cf4b2721cd17 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 30 Mar 2026 13:44:19 +0530 Subject: [PATCH 2/3] style: cargo fmt --- src/decode/parser.rs | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/decode/parser.rs b/src/decode/parser.rs index ae328df..dde0425 100644 --- a/src/decode/parser.rs +++ b/src/decode/parser.rs @@ -167,13 +167,11 @@ impl<'a> Parser<'a> { | Token::Number(..) | Token::Bool(..) | Token::Null => { - let ws = - self.scanner.last_whitespace_count().max(1); + let ws = self.scanner.last_whitespace_count().max(1); for _ in 0..ws { accumulated.push(' '); } - accumulated - .push_str(self.scanner.last_token_text()); + accumulated.push_str(self.scanner.last_token_text()); self.advance()?; } _ => break, @@ -210,13 +208,11 @@ impl<'a> Parser<'a> { | Token::Number(..) | Token::Bool(..) | Token::Null => { - let ws = - self.scanner.last_whitespace_count().max(1); + let ws = self.scanner.last_whitespace_count().max(1); for _ in 0..ws { accumulated.push(' '); } - accumulated - .push_str(self.scanner.last_token_text()); + accumulated.push_str(self.scanner.last_token_text()); self.advance()?; } _ => break, @@ -226,17 +222,13 @@ impl<'a> Parser<'a> { } _ => { // Normalize floats that are actually integers - if val.is_finite() - && val.fract() == 0.0 - && val.abs() <= i64::MAX as f64 + if val.is_finite() && val.fract() == 0.0 && val.abs() <= i64::MAX as f64 { Ok(serde_json::Number::from(val as i64).into()) } else { Ok(serde_json::Number::from_f64(val) .ok_or_else(|| { - ToonError::InvalidInput(format!( - "Invalid number: {val}" - )) + ToonError::InvalidInput(format!("Invalid number: {val}")) })? .into()) } @@ -1242,9 +1234,7 @@ impl<'a> Parser<'a> { Ok(Number::from(n as i64).into()) } else { Ok(Number::from_f64(n) - .ok_or_else(|| { - ToonError::InvalidInput(format!("Invalid number: {n}")) - })? + .ok_or_else(|| ToonError::InvalidInput(format!("Invalid number: {n}")))? .into()) } } From 77dc8b4ddd573d9921c9c21575f0fd980b0876a1 Mon Sep 17 00:00:00 2001 From: soul Date: Mon, 30 Mar 2026 13:46:08 +0530 Subject: [PATCH 3/3] style: fix clippy warnings Replace loop+match with while-let patterns and remove unnecessary let binding in scan_token. --- src/decode/parser.rs | 78 ++++++++++++++++++------------------------- src/decode/scanner.rs | 6 ++-- 2 files changed, 35 insertions(+), 49 deletions(-) diff --git a/src/decode/parser.rs b/src/decode/parser.rs index dde0425..37c953d 100644 --- a/src/decode/parser.rs +++ b/src/decode/parser.rs @@ -160,22 +160,18 @@ impl<'a> Parser<'a> { | Token::Bool(..) | Token::Null => { let mut accumulated = first_text; - loop { - match &self.current_token { - Token::String(..) - | Token::Integer(..) - | Token::Number(..) - | Token::Bool(..) - | Token::Null => { - let ws = self.scanner.last_whitespace_count().max(1); - for _ in 0..ws { - accumulated.push(' '); - } - accumulated.push_str(self.scanner.last_token_text()); - self.advance()?; - } - _ => break, + while let Token::String(..) + | Token::Integer(..) + | Token::Number(..) + | Token::Bool(..) + | Token::Null = &self.current_token + { + let ws = self.scanner.last_whitespace_count().max(1); + for _ in 0..ws { + accumulated.push(' '); } + accumulated.push_str(self.scanner.last_token_text()); + self.advance()?; } Ok(Value::String(accumulated)) } @@ -201,22 +197,18 @@ impl<'a> Parser<'a> { | Token::Bool(..) | Token::Null => { let mut accumulated = first_text; - loop { - match &self.current_token { - Token::String(..) - | Token::Integer(..) - | Token::Number(..) - | Token::Bool(..) - | Token::Null => { - let ws = self.scanner.last_whitespace_count().max(1); - for _ in 0..ws { - accumulated.push(' '); - } - accumulated.push_str(self.scanner.last_token_text()); - self.advance()?; - } - _ => break, + while let Token::String(..) + | Token::Integer(..) + | Token::Number(..) + | Token::Bool(..) + | Token::Null = &self.current_token + { + let ws = self.scanner.last_whitespace_count().max(1); + for _ in 0..ws { + accumulated.push(' '); } + accumulated.push_str(self.scanner.last_token_text()); + self.advance()?; } Ok(Value::String(accumulated)) } @@ -263,22 +255,18 @@ impl<'a> Parser<'a> { } // Root-level string value - join consecutive tokens with exact spacing let mut accumulated = first; - loop { - match &self.current_token { - Token::String(..) - | Token::Integer(..) - | Token::Number(..) - | Token::Bool(..) - | Token::Null => { - let ws = self.scanner.last_whitespace_count().max(1); - for _ in 0..ws { - accumulated.push(' '); - } - accumulated.push_str(self.scanner.last_token_text()); - self.advance()?; - } - _ => break, + while let Token::String(..) + | Token::Integer(..) + | Token::Number(..) + | Token::Bool(..) + | Token::Null = &self.current_token + { + let ws = self.scanner.last_whitespace_count().max(1); + for _ in 0..ws { + accumulated.push(' '); } + accumulated.push_str(self.scanner.last_token_text()); + self.advance()?; } Ok(Value::String(accumulated)) } diff --git a/src/decode/scanner.rs b/src/decode/scanner.rs index bef103a..b80a450 100644 --- a/src/decode/scanner.rs +++ b/src/decode/scanner.rs @@ -170,7 +170,7 @@ impl Scanner { self.skip_whitespace(); - let token = match self.peek() { + match self.peek() { None => Ok(Token::Eof), Some('\n') => { self.advance(); @@ -248,9 +248,7 @@ impl Scanner { self.parse_number(&num_str) } Some(_) => self.scan_unquoted_string(), - }; - - token + } } fn scan_quoted_string(&mut self) -> ToonResult {