feat: integrate fastokens BPE tokenizer backend

Cargo.toml

@@ -46,6 +46,7 @@ dynamo-mocker = { path = "lib/mocker", version = "1.0.0" }
 dynamo-kv-router = { path = "lib/kv-router", version = "1.0.0", features = ["metrics"] }
 dynamo-async-openai = { path = "lib/async-openai", version = "1.0.0", features = ["byot"] }
 dynamo-parsers = { path = "lib/parsers", version = "1.0.0" }
+fastokens = { git = "https://github.com/biswapanda/fastokens", rev = "e79e7bd2a4b2e1bc3f372aa29a268954c95f43c7", version = "0.1.0", default-features = false }
 
 # kvbm
 kvbm-common = { path = "lib/kvbm-common", version = "0.1.0" }

components/src/dynamo/frontend/frontend_args.py

@@ -76,6 +76,9 @@ class FrontendConfig(KvRouterConfigBase):
     enable_streaming_tool_dispatch: bool
     enable_streaming_reasoning_dispatch: bool
     preprocess_workers: int
+    tokenizer_backend: str
+
+    _VALID_TOKENIZER_BACKENDS = {"default", "fastokens"}
 
     def validate(self) -> None:
         if bool(self.tls_cert_path) ^ bool(self.tls_key_path):  # ^ is XOR
@@ -88,6 +91,11 @@ def validate(self) -> None:
             )
         if self.router_enable_cache_control and self.router_mode != "kv":
             raise ValueError("--enable-cache-control requires --router-mode=kv")
+        if self.tokenizer_backend not in self._VALID_TOKENIZER_BACKENDS:
+            raise ValueError(
+                f"--tokenizer: invalid value '{self.tokenizer_backend}' "
+                f"(choose from {sorted(self._VALID_TOKENIZER_BACKENDS)})"
+            )
 
 
 @register_encoder(FrontendConfig)
@@ -424,3 +432,17 @@ def add_arguments(self, parser) -> None:
             ),
             arg_type=int,
         )
+
+        add_argument(
+            g,
+            flag_name="--tokenizer",
+            env_var="DYN_TOKENIZER",
+            default="default",
+            dest="tokenizer_backend",
+            help=(
+                "Tokenizer backend for BPE models: 'default' (HuggingFace tokenizers library) "
+                "or 'fastokens' (fastokens crate for high-performance BPE encoding). "
+                "Decoding always uses HuggingFace. Has no effect on TikToken models."
+            ),
+            choices=["default", "fastokens"],
+        )

components/src/dynamo/frontend/main.py

@@ -165,6 +165,10 @@ async def async_main():
     config, vllm_flags, sglang_flags = parse_args()
     dump_config(config.dump_config_to, config)
     os.environ["DYN_EVENT_PLANE"] = config.event_plane
+    if config.tokenizer_backend == "fastokens":
+        os.environ["DYN_TOKENIZER"] = "fastokens"
+    else:
+        os.environ.pop("DYN_TOKENIZER", None)
     logger.info(
         f"Request migration {'enabled' if config.migration_limit > 0 else 'disabled'} "
         f"(limit: {config.migration_limit})"