feat: integrate fastokens BPE tokenizer backend

Cargo.toml

@@ -46,6 +46,7 @@ dynamo-mocker = { path = "lib/mocker", version = "1.0.0" }
 dynamo-kv-router = { path = "lib/kv-router", version = "1.0.0", features = ["metrics"] }
 dynamo-async-openai = { path = "lib/async-openai", version = "1.0.0", features = ["byot"] }
 dynamo-parsers = { path = "lib/parsers", version = "1.0.0" }
+fastokens = { git = "https://github.com/Atero-ai/fastokens", version = "0.1.0" }
 
 # kvbm
 kvbm-common = { path = "lib/kvbm-common", version = "0.1.0" }

components/src/dynamo/frontend/frontend_args.py

@@ -76,6 +76,7 @@ class FrontendConfig(KvRouterConfigBase):
     enable_streaming_tool_dispatch: bool
     enable_streaming_reasoning_dispatch: bool
     preprocess_workers: int
+    tokenizer_backend: str
 
     def validate(self) -> None:
         if bool(self.tls_cert_path) ^ bool(self.tls_key_path):  # ^ is XOR
@@ -424,3 +425,16 @@ def add_arguments(self, parser) -> None:
             ),
             arg_type=int,
         )
+
+        add_argument(
+            g,
+            flag_name="--dyn-tokenizer-backend",
+            env_var="DYN_TOKENIZER_BACKEND",
+            default="default",
+            dest="tokenizer_backend",
+            help=(
+                "Tokenizer backend for BPE models: 'default' (HuggingFace tokenizers library) "
+                "or 'fasttokens' (fastokens crate for high-performance BPE encoding). "
+                "Decoding always uses HuggingFace. Has no effect on TikToken models."
+            ),
+        )

components/src/dynamo/frontend/main.py

@@ -165,6 +165,8 @@ async def async_main():
     config, vllm_flags, sglang_flags = parse_args()
     dump_config(config.dump_config_to, config)
     os.environ["DYN_EVENT_PLANE"] = config.event_plane
+    if config.tokenizer_backend == "fasttokens":
+        os.environ["DYN_TOKENIZER_BACKEND"] = "fasttokens"
     logger.info(
         f"Request migration {'enabled' if config.migration_limit > 0 else 'disabled'} "
         f"(limit: {config.migration_limit})"