CRAYON/test_readme_examples.py at main · Electroiscoding/CRAYON · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
"""
Test all code examples from README.md to ensure they work correctly.
"""
import sys
import os

# Add paths
sys.path.insert(0, os.path.join(os.getcwd(), "build", "lib.win-amd64-cpython-313"))
sys.path.insert(0, os.path.join(os.getcwd(), "src"))

print("=" * 70)
print("TESTING README CODE EXAMPLES")
print("=" * 70)
print()

# Test 1: Quick Start Example
print("[TEST 1] Quick Start - Load Profile and Tokenize")
print("-" * 70)
try:
    from crayon.core.vocabulary import CrayonVocab

    vocab = CrayonVocab(device="auto")
    vocab.load_profile("lite")

    # Tokenize specialized syntax
    code_snippet = "fn main() { println!(\"Hello, World!\"); }"
    tokens = vocab.tokenize(code_snippet)

    # Check if decode works
    try:
        decoded = vocab.decode(tokens)
        print(f"✓ Tokenize: {code_snippet}")
        print(f"✓ Tokens: {tokens}")
        print(f"✓ Decoded: {decoded}")
        print("✓ TEST PASSED")
    except AttributeError:
        print(f"⚠ WARNING: vocab.decode() not implemented yet")
        print(f"✓ Tokenize works: {tokens}")
        print("✓ TEST PARTIALLY PASSED")
except Exception as e:
    print(f"✗ TEST FAILED: {e}")
    import traceback
    traceback.print_exc()

print()

# Test 2: Load different profiles
print("[TEST 2] Load Different Profiles")
print("-" * 70)
for profile_name in ["lite", "standard"]:
    try:
        vocab = CrayonVocab(device="auto")
        vocab.load_profile(profile_name)
        print(f"✓ Loaded '{profile_name}' profile")
    except Exception as e:
        print(f"✗ Failed to load '{profile_name}': {e}")

print()

# Test 3: DAT Builder Example
print("[TEST 3] Compile Vocabulary to DAT Format")
print("-" * 70)
try:
    from crayon.c_ext.dat_builder import DATBuilder
    import json
    import tempfile

    # Use a small test vocab
    test_vocab = ["hello", "world", "test", "python"]

    # Compile to DAT
    builder = DATBuilder()
    builder.build(test_vocab)

    # Save to temp file
    dat_path = os.path.join(tempfile.gettempdir(), "test_readme.dat")
    builder.save(dat_path)

    print(f"✓ Built DAT with {builder.size} nodes")
    print(f"✓ Saved to {dat_path}")

    os.unlink(dat_path)
    print("✓ TEST PASSED")
except Exception as e:
    print(f"✗ TEST FAILED: {e}")
    import traceback
    traceback.print_exc()

print()

# Test 4: Direct C++ Engine Access
print("[TEST 4] Direct C++ Engine Access")
print("-" * 70)
try:
    import mmap
    from crayon.c_ext import crayon_fast
    from crayon.c_ext.dat_builder import DATBuilder
    import tempfile

    # Build a small DAT
    test_vocab = ["the", "quick", "brown", "fox"]
    builder = DATBuilder()
    builder.build(test_vocab)

    dat_path = os.path.join(tempfile.gettempdir(), "test_engine.dat")
    builder.save(dat_path)

    # Zero-copy load via mmap
    with open(dat_path, "rb") as f:
        mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
        size = crayon_fast.load_dat(mm)

    # Ultra-fast tokenization
    tokens = crayon_fast.tokenize("the quick brown fox")

    print(f"✓ Loaded DAT: {size} nodes")
    print(f"✓ Tokenized: {tokens}")

    os.unlink(dat_path)
    print("✓ TEST PASSED")
except Exception as e:
    print(f"✗ TEST FAILED: {e}")
    import traceback
    traceback.print_exc()

print()
print("=" * 70)
print("README CODE TESTS COMPLETE")
print("=" * 70)