Add pytorch-tokenizers as a dependency of ExecuTorch (#15492)

larryliu0820 · web-flow · commit 67a94af6adf9 · 2025-10-31T10:36:22.000-07:00
So that we can `pip install executorch` and run python code like
`examples/models/llama/runner/generation.py` without installing
`pytorch-tokenizers` separately.
diff --git a/extension/llm/tokenizers b/extension/llm/tokenizers
@@ -1 +1 @@
-Subproject commit 0bcd9f5325c42d2e05765d584131900dbc8835c8
+Subproject commit 3aada3fe28c945d14d5ec62254eb56ccdf10eb11
diff --git a/pyproject.toml b/pyproject.toml
@@ -66,6 +66,7 @@ dependencies=[
   "pytest-xdist",
   "pytest-rerunfailures==15.1",
   "pytest-json-report",
+  "pytorch-tokenizers",
   "pyyaml",
   "ruamel.yaml",
   "sympy",
