fix(ci): resolve pyproject.toml license error and fix wheel tests

- Fixed project.license format in pyproject.toml (PEP 621)
- Added backward compatibility for fast_mode and longest_match in CrayonVocab
- Removed manual sys.path manipulation in unit tests to support wheel testing
- Synchronized metadata across setup.py and pyproject.toml

Author: Electroiscoding

pyproject.toml

@@ -8,7 +8,7 @@ version = "4.3.0"
 description = "Omni-Backend Tokenizer - CPU (AVX2/512), CUDA (NVIDIA), ROCm (AMD) with automatic hardware detection"
 readme = "README.md"
 requires-python = ">=3.10"
-license = "MIT"
+license = {file = "LICENSE"}
 authors = [
   {name = "Xerv Research Engineering Division", email = "engineering@xerv.ai"}
 ]

src/crayon/core/vocabulary.py

@@ -1034,6 +1034,38 @@ def is_profile_loaded(self) -> bool:
         """Check if a profile is currently loaded."""
         return self._profile_loaded
 
+    @property
+    def fast_mode(self) -> bool:
+        """Check if running in high-performance mode (C++ backend)."""
+        return self.device in ("cpu", "cuda", "rocm") and (self._cpu_backend is not None or self._gpu_backend is not None)
+
+    def longest_match(self, text: str, pos: int = 0) -> Tuple[int, int]:
+        """
+        Find the longest matching token at the given position (Compatibility Mode).
+        
+        Note: This is slower than tokenize() as it creates a substring.
+        """
+        if pos >= len(text):
+            return self.unk_token_id, 0
+            
+        # Optimization: We only need to check a reasonable window
+        # The longest token is rarely more than 100 characters.
+        window = text[pos : pos + 128]
+        tokens = self.tokenize(window)
+        
+        if not tokens:
+            return self.unk_token_id, 1
+            
+        # Get the first token ID
+        first_id = tokens[0]
+        
+        # Get its length from id_to_token
+        if 0 <= first_id < len(self._idx_to_str):
+            token_str = self._idx_to_str[first_id]
+            return first_id, len(token_str)
+        else:
+            return self.unk_token_id, 1
+
 
 # ============================================================================
 # CONVENIENCE FUNCTIONS

tests/test_c_ext.py

@@ -7,8 +7,6 @@
 import json
 from pathlib import Path
 
-# Add src to path for imports
-sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
 
 try:
     from crayon.c_ext import crayon_cpu, crayon_trainer, crayon_compiler

tests/test_core.py

@@ -3,8 +3,6 @@
 import sys
 from pathlib import Path
 
-# Add src to path for imports
-sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
 
 from crayon.core.vocabulary import CrayonVocab
 from crayon.core.primitives import TokenMetadata

tests/test_throughput.py

@@ -1,3 +1,4 @@
+
 import unittest
 import time
 from crayon.core.vocabulary import CrayonVocab
@@ -29,32 +30,26 @@ def test_throughput_target(self):
 
         print(f"Throughput Test: {throughput:,.0f} tokens/sec")
 
-        # We should at least achieve baseline performance
+        # We should at least achieve baseline performance (10k is very conservative for C++ engine)
         self.assertGreater(throughput, 10000, "Throughput fell below minimum acceptable threshold")
 
-    def test_c_extension_performance_boost(self):
-        """Test that C extension provides performance improvement."""
-        if not self.vocab._c_ext_available:
-            self.skipTest("C extension not available")
-        
-        # Measure Python fallback
-        self.vocab._c_ext_available = False
-        original_trie = self.vocab._c_trie
-        self.vocab._c_trie = None
-        
-        start = time.perf_counter()
-        for _ in range(3):
-            _ = self.vocab.tokenize(self.text)
-        python_time = time.perf_counter() - start
-        
-        # Restore C extension
-        self.vocab._c_ext_available = True
-        self.vocab._c_trie = original_trie
-        
+    def test_engine_performance_boost(self):
+        """Test that the engine provides reasonable performance."""
+        # In V4, 'fast_mode' is the default if compiled.
+        # We check by seeing if it's using the C++ backend.
+        info = self.vocab.get_info()
+        is_fast = info["backend"].endswith("_extension")
+        
+        if not is_fast:
+             self.skipTest("C++ extension not available, can't test boost")
+             
         start = time.perf_counter()
         for _ in range(3):
             _ = self.vocab.tokenize(self.text)
         c_time = time.perf_counter() - start
 
-        print(f"Python time: {python_time:.3f}s, C time: {c_time:.3f}s")
-        # C extension should be at least comparable (may not always be faster due to Python overhead)
+        print(f"C++ Engine time: {c_time:.3f}s")
+        self.assertGreater(len(self.vocab.tokenize(self.text)), 0)
+
+if __name__ == "__main__":
+    unittest.main()