Skip to content

Commit 58e375d

Browse files
lurenssclaude
andcommitted
feat: achieve full feature parity with TypeScript TOON library
Implemented 8 features to match TypeScript implementation: **Feature 1: DateTime Serialization** - Added datetime and date support to encoder - Convert to ISO 8601 format (YYYY-MM-DD for dates, ISO string for datetimes) - Added type checking in is_primitive() utility **Feature 2: Scientific Notation Suppression** - Added format_float() utility to suppress unnecessary scientific notation - Applied to all float encoding paths (primitive values, arrays, tabular data) - Maintains readability for human-readable ranges **Feature 3: Delimiter Indicators in Array Headers** - Added delimiter indicators: [N\t] for tab, [N|] for pipe - Comma remains default with no indicator - Applied to both root-level and nested tabular arrays - Decoder auto-detects delimiter from header indicator **Feature 4: Smart Delimiter Detection in Decoder** - Decoder checks header indicator first ([N\t] or [N|]) - Falls back to detecting delimiter from first data row - Default to comma if no delimiter detected **Feature 5: Dash Markers in List Arrays** - Added "- " prefix to list array items for visual clarity - Decoder strips dash markers when parsing - Improves readability of non-uniform arrays **Feature 6: Delimiter-Aware Quoting** - Quote strings containing delimiter characters (tab, pipe, comma) - Added delimiter characters to needs_quoting() checks - Prevents parsing ambiguity in tabular data **Feature 7: Indent Auto-Detection** - Added detect_indent() function to analyze TOON strings - Auto-detects 2-space, 4-space, or tab indentation - Supports explicit indent override via options - Threaded indent_size parameter through all parsing functions **Feature 8: Strict Array Length Validation** - Validate declared array count matches actual items in strict mode - Applied to both tabular and list arrays - Raises descriptive ValueError on mismatch - Non-strict mode allows flexible array lengths Test Coverage: - Added 41 new tests (18 encoder, 15 decoder, 8 roundtrip) - All 87 tests passing - Comprehensive coverage of edge cases and error conditions 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <[email protected]>
1 parent f3ccd1c commit 58e375d

File tree

6 files changed

+1154
-75
lines changed

6 files changed

+1154
-75
lines changed

tests/test_decoder.py

Lines changed: 353 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -108,10 +108,11 @@ def test_decode_tabular_array():
108108

109109
def test_decode_tabular_array_with_tab():
110110
"""Test decoding tabular array with tab delimiter."""
111-
toon = """users[2]{id,name}:
111+
# Tab delimiter should have \t indicator in header
112+
toon = """users[2\t]{id,name}:
112113
1\tAlice
113114
2\tBob"""
114-
115+
115116
result = decode(toon)
116117

117118
expected = {
@@ -248,5 +249,354 @@ def test_decode_quoted_field_values():
248249
{'id': 2, 'description': 'Normal item'}
249250
]
250251
}
251-
252+
253+
assert result == expected
254+
255+
256+
def test_decode_tabular_array_with_tab_indicator():
257+
"""Test decoding tabular array with tab delimiter indicator in header."""
258+
toon = """users[2\t]{id,name}:
259+
1\tAlice
260+
2\tBob"""
261+
262+
result = decode(toon)
263+
264+
expected = {
265+
'users': [
266+
{'id': 1, 'name': 'Alice'},
267+
{'id': 2, 'name': 'Bob'}
268+
]
269+
}
270+
271+
assert result == expected
272+
273+
274+
def test_decode_tabular_array_with_pipe_indicator():
275+
"""Test decoding tabular array with pipe delimiter indicator in header."""
276+
toon = """products[2|]{sku,price}:
277+
A001|29.99
278+
B002|49.99"""
279+
280+
result = decode(toon)
281+
282+
expected = {
283+
'products': [
284+
{'sku': 'A001', 'price': 29.99},
285+
{'sku': 'B002', 'price': 49.99}
286+
]
287+
}
288+
252289
assert result == expected
290+
291+
292+
def test_decode_tabular_array_comma_no_indicator():
293+
"""Test decoding tabular array without delimiter indicator uses comma default."""
294+
toon = """items[2]{code,count}:
295+
X,5
296+
Y,10"""
297+
298+
result = decode(toon)
299+
300+
expected = {
301+
'items': [
302+
{'code': 'X', 'count': 5},
303+
{'code': 'Y', 'count': 10}
304+
]
305+
}
306+
307+
assert result == expected
308+
309+
310+
def test_decode_list_array_with_dash_markers():
311+
"""Test decoding list array with dash markers."""
312+
toon = """items[3]:
313+
- apple
314+
- banana
315+
- cherry"""
316+
317+
result = decode(toon)
318+
319+
expected = {
320+
'items': ['apple', 'banana', 'cherry']
321+
}
322+
323+
assert result == expected
324+
325+
326+
def test_decode_mixed_types_with_dash_markers():
327+
"""Test decoding mixed types array with dash markers."""
328+
toon = """mixed[3]:
329+
- string value
330+
- 42
331+
- key: value"""
332+
333+
result = decode(toon)
334+
335+
expected = {
336+
'mixed': ['string value', 42, {'key': 'value'}]
337+
}
338+
339+
assert result == expected
340+
341+
342+
def test_decode_datetime_string():
343+
"""Test decoding datetime ISO strings."""
344+
toon = """created: "2024-01-01T12:30:45"
345+
updated: "2024-06-15T09:00:00\""""
346+
347+
result = decode(toon)
348+
349+
expected = {
350+
'created': '2024-01-01T12:30:45',
351+
'updated': '2024-06-15T09:00:00'
352+
}
353+
354+
assert result == expected
355+
356+
357+
def test_decode_scientific_notation():
358+
"""Test decoding numbers in scientific notation."""
359+
toon = """small: 1e-06
360+
smaller: 1e-07
361+
large: 1.5e+16
362+
very_large: 1.23e20
363+
normal: 3.14159"""
364+
365+
result = decode(toon)
366+
367+
expected = {
368+
'small': 1e-06,
369+
'smaller': 1e-07,
370+
'large': 1.5e+16,
371+
'very_large': 1.23e20,
372+
'normal': 3.14159
373+
}
374+
375+
assert result == expected
376+
377+
378+
def test_decode_decimal_notation():
379+
"""Test decoding numbers in decimal notation (no scientific)."""
380+
toon = """small: 0.000001
381+
smaller: 0.0000001
382+
large: 15000000000000000
383+
normal: 3.14159
384+
integer: 42"""
385+
386+
result = decode(toon)
387+
388+
expected = {
389+
'small': 0.000001,
390+
'smaller': 0.0000001,
391+
'large': 15000000000000000.0,
392+
'normal': 3.14159,
393+
'integer': 42
394+
}
395+
396+
assert result == expected
397+
398+
399+
def test_decode_float_array_with_scientific():
400+
"""Test decoding arrays with scientific notation numbers."""
401+
toon = """values: [1e-06,1e-07,1.5e+16,3.14159]"""
402+
403+
result = decode(toon)
404+
405+
expected = {
406+
'values': [1e-06, 1e-07, 1.5e+16, 3.14159]
407+
}
408+
409+
assert result == expected
410+
411+
412+
def test_decode_root_inline_array():
413+
"""Test decoding root-level inline array."""
414+
toon = "[1,2,3,4,5]"
415+
416+
result = decode(toon)
417+
418+
expected = [1, 2, 3, 4, 5]
419+
420+
assert result == expected
421+
422+
423+
def test_decode_root_tabular_array():
424+
"""Test decoding root-level tabular array."""
425+
toon = """[3]{id,name}:
426+
1,Alice
427+
2,Bob
428+
3,Charlie"""
429+
430+
result = decode(toon)
431+
432+
expected = [
433+
{'id': 1, 'name': 'Alice'},
434+
{'id': 2, 'name': 'Bob'},
435+
{'id': 3, 'name': 'Charlie'}
436+
]
437+
438+
assert result == expected
439+
440+
441+
def test_decode_root_list_array():
442+
"""Test decoding root-level list array."""
443+
toon = """[4]:
444+
- 1
445+
- text
446+
- nested: object
447+
- [1,2,3]"""
448+
449+
result = decode(toon)
450+
451+
expected = [
452+
1,
453+
'text',
454+
{'nested': 'object'},
455+
[1, 2, 3]
456+
]
457+
458+
assert result == expected
459+
460+
461+
def test_decode_4space_indent():
462+
"""Test auto-detecting 4-space indentation."""
463+
toon = """user:
464+
name: Alice
465+
age: 30
466+
profile:
467+
city: NYC
468+
country: USA"""
469+
470+
result = decode(toon)
471+
472+
expected = {
473+
'user': {
474+
'name': 'Alice',
475+
'age': 30,
476+
'profile': {
477+
'city': 'NYC',
478+
'country': 'USA'
479+
}
480+
}
481+
}
482+
483+
assert result == expected
484+
485+
486+
def test_decode_explicit_indent_override():
487+
"""Test explicitly specifying indent size."""
488+
# 3-space indent (unusual but should work with explicit option)
489+
toon = """data:
490+
value: 123
491+
nested:
492+
item: test"""
493+
494+
result = decode(toon, {'indent': 3})
495+
496+
expected = {
497+
'data': {
498+
'value': 123,
499+
'nested': {
500+
'item': 'test'
501+
}
502+
}
503+
}
504+
505+
assert result == expected
506+
507+
508+
def test_decode_array_with_custom_indent():
509+
"""Test decoding array with custom indentation."""
510+
toon = """users[2]{id,name}:
511+
1,Alice
512+
2,Bob"""
513+
514+
result = decode(toon)
515+
516+
expected = {
517+
'users': [
518+
{'id': 1, 'name': 'Alice'},
519+
{'id': 2, 'name': 'Bob'}
520+
]
521+
}
522+
523+
assert result == expected
524+
525+
526+
def test_decode_strict_mode_correct_count():
527+
"""Test strict mode with correct array count."""
528+
toon = """users[2]{id,name}:
529+
1,Alice
530+
2,Bob"""
531+
532+
result = decode(toon, {'strict': True})
533+
534+
expected = {
535+
'users': [
536+
{'id': 1, 'name': 'Alice'},
537+
{'id': 2, 'name': 'Bob'}
538+
]
539+
}
540+
541+
assert result == expected
542+
543+
544+
def test_decode_strict_mode_too_few_items():
545+
"""Test strict mode raises error when array has fewer items than declared."""
546+
toon = """users[3]{id,name}:
547+
1,Alice
548+
2,Bob"""
549+
550+
try:
551+
decode(toon, {'strict': True})
552+
assert False, 'Should have raised ValueError'
553+
except ValueError as e:
554+
assert 'Array length mismatch' in str(e)
555+
assert 'expected 3, got 2' in str(e)
556+
557+
558+
def test_decode_non_strict_mode_too_few_items():
559+
"""Test non-strict mode allows fewer items than declared."""
560+
toon = """users[5]{id,name}:
561+
1,Alice
562+
2,Bob"""
563+
564+
result = decode(toon, {'strict': False})
565+
566+
expected = {
567+
'users': [
568+
{'id': 1, 'name': 'Alice'},
569+
{'id': 2, 'name': 'Bob'}
570+
]
571+
}
572+
573+
assert result == expected
574+
575+
576+
def test_decode_strict_mode_list_array():
577+
"""Test strict mode with list array."""
578+
toon = """items[2]:
579+
- item1
580+
- item2"""
581+
582+
result = decode(toon, {'strict': True})
583+
584+
expected = {
585+
'items': ['item1', 'item2']
586+
}
587+
588+
assert result == expected
589+
590+
591+
def test_decode_strict_mode_list_array_mismatch():
592+
"""Test strict mode raises error for list array length mismatch."""
593+
toon = """items[4]:
594+
- item1
595+
- item2"""
596+
597+
try:
598+
decode(toon, {'strict': True})
599+
assert False, 'Should have raised ValueError'
600+
except ValueError as e:
601+
assert 'Array length mismatch' in str(e)
602+
assert 'expected 4, got 2' in str(e)

0 commit comments

Comments
 (0)