Skip to content

Commit 2bbf4d8

Browse files
authored
fix: fixes bugs and spec alignment (#52)
* fix: treat negative leading-zero tokens as strings per spec v3.0.1 Update spec submodule to latest and fix decoder to treat tokens like "-05" and "-007" as strings rather than parsing them as numbers. The spec now explicitly requires forbidden leading zeros in the integer part to apply to negative numbers as well. * fix: tabular array round-trip when not first field in list item The encoder wrote tabular rows at the wrong depth when the array was a non-first field of a list-item object, and the decoder dropped sibling fields after parsing such arrays because the tabular parser already consumed the trailing newline. * docs: update README spec version references to v3.0
1 parent 31e76ff commit 2bbf4d8

File tree

5 files changed

+30
-7
lines changed

5 files changed

+30
-7
lines changed

README.md

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,13 @@
22

33
[![Crates.io](https://img.shields.io/crates/v/toon-format.svg)](https://crates.io/crates/toon-format)
44
[![Documentation](https://docs.rs/toon-format/badge.svg)](https://docs.rs/toon-format)
5-
[![Spec v2.0](https://img.shields.io/badge/spec-v2.0-brightgreen.svg)](https://github.com/toon-format/spec/blob/main/SPEC.md)
5+
[![Spec v3.0](https://img.shields.io/badge/spec-v3.0-brightgreen.svg)](https://github.com/toon-format/spec/blob/main/SPEC.md)
66
[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE)
77
[![Tests](https://img.shields.io/badge/tests-%20passing-success.svg)]()
88

99
**Token-Oriented Object Notation (TOON)** is a compact, human-readable format designed for passing structured data to Large Language Models with significantly reduced token usage.
1010

11-
This crate provides the official, **spec-compliant Rust implementation** of TOON v2.0 with v1.5 optional features, offering both a library (`toon-format`) and a full-featured command-line tool (`toon`).
11+
This crate provides the official, **spec-compliant Rust implementation** of TOON v3.0, offering both a library (`toon-format`) and a full-featured command-line tool (`toon`).
1212

1313
## Quick Example
1414

@@ -32,8 +32,8 @@ users[2]{id,name}:
3232
## Features
3333

3434
- **Generic API**: Works with any `Serialize`/`Deserialize` type - custom structs, enums, JSON values, and more
35-
- **Spec-Compliant**: Fully compliant with [TOON Specification v2.0](https://github.com/toon-format/spec/blob/main/SPEC.md)
36-
- **v1.5 Optional Features**: Key folding and path expansion
35+
- **Spec-Compliant**: Fully compliant with [TOON Specification v3.0](https://github.com/toon-format/spec/blob/main/SPEC.md)
36+
- **Key Folding & Path Expansion**: Collapse and expand dotted key paths
3737
- **Safe & Performant**: Built with safe, fast Rust
3838
- **Powerful CLI**: Full-featured command-line tool
3939
- **Strict Validation**: Enforces all spec rules (configurable)
@@ -547,7 +547,7 @@ Run with `cargo run --example examples` to see all examples:
547547

548548
## Resources
549549

550-
- 📖 [TOON Specification v2.0](https://github.com/toon-format/spec/blob/main/SPEC.md)
550+
- 📖 [TOON Specification v3.0](https://github.com/toon-format/spec/blob/main/SPEC.md)
551551
- 📦 [Crates.io Package](https://crates.io/crates/toon-format)
552552
- 📚 [API Documentation](https://docs.rs/toon-format)
553553
- 🔧 [Main Repository (JS/TS)](https://github.com/toon-format/toon)

src/decode/parser.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -995,6 +995,13 @@ impl<'a> Parser<'a> {
995995
if !self.options.strict {
996996
self.skip_newlines()?;
997997
}
998+
} else if matches!(self.current_token, Token::String(_, _)) {
999+
// Tabular array parser already consumed the newline
1000+
// and advanced to the next token — check indent
1001+
let current_indent = self.scanner.get_last_line_indent();
1002+
if current_indent != field_indent {
1003+
break;
1004+
}
9981005
} else {
9991006
break;
10001007
}

src/decode/scanner.rs

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -358,6 +358,14 @@ impl Scanner {
358358
}
359359
}
360360

361+
// Negative leading zeros like "-05" are also strings
362+
if s.starts_with("-0") && s.len() > 2 {
363+
let third_char = s.chars().nth(2).unwrap();
364+
if third_char.is_ascii_digit() {
365+
return Ok(Token::String(s.to_string(), false));
366+
}
367+
}
368+
361369
if s.contains('.') || s.contains('e') || s.contains('E') {
362370
if let Ok(f) = s.parse::<f64>() {
363371
Ok(Token::Number(f))
@@ -467,6 +475,14 @@ impl Scanner {
467475
}
468476
}
469477

478+
// Negative leading zeros like "-05" are also strings
479+
if trimmed.starts_with("-0") && trimmed.len() > 2 {
480+
let third_char = trimmed.chars().nth(2).unwrap();
481+
if third_char.is_ascii_digit() {
482+
return Ok(Token::String(trimmed.to_string(), false));
483+
}
484+
}
485+
470486
if trimmed.contains('.') || trimmed.contains('e') || trimmed.contains('E') {
471487
if let Ok(f) = trimmed.parse::<f64>() {
472488
let normalized = if f == -0.0 { 0.0 } else { f };

src/encode/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -620,7 +620,7 @@ fn encode_nested_array(
620620
match value {
621621
Value::Array(arr) => {
622622
writer.write_key(key)?;
623-
write_array(writer, None, arr, depth + 1)?;
623+
write_array(writer, None, arr, depth + 2)?;
624624
}
625625
Value::Object(nested_obj) => {
626626
writer.write_key(key)?;

0 commit comments

Comments
 (0)