Remove old tokenizer/ directory in ExecuTorch

larryliu0820 · facebook-github-bot · commit b7d15bd13d5c · 2025-03-27T17:18:40.000-07:00
Summary: X-link: pytorch/executorch#9728 See what happens Differential Revision: D72007597
diff --git a/pytorch_tokenizers/tools/llama2c/convert.py b/pytorch_tokenizers/tools/llama2c/convert.py
@@ -11,7 +11,7 @@
 
 import argparse
 
-from pytorch_tokenizers import Llama2cTokenizer
+from pytorch_tokenizers.llama2c import Llama2cTokenizer
 
 
 if __name__ == "__main__":
diff --git a/pytorch_tokenizers/tools/llama2c/targets.bzl b/pytorch_tokenizers/tools/llama2c/targets.bzl
@@ -24,6 +24,9 @@ def define_common_targets():
         external_deps = [
             "sentencepiece-py",
         ],
+        deps = [
+            "//pytorch/tokenizers/pytorch_tokenizers:tokenizers",
+        ],
     )
 
     runtime.python_binary(

Original file line number	Diff line number	Diff line change
`@@ -24,6 +24,9 @@ def define_common_targets():`
`24`	`24`	`external_deps = [`
`25`	`25`	`"sentencepiece-py",`
`26`	`26`	`],`
	`27`	`+ deps = [`
	`28`	`+ "//pytorch/tokenizers/pytorch_tokenizers:tokenizers",`
	`29`	`+ ],`
`27`	`30`	`)`
`28`	`31`
`29`	`32`	`runtime.python_binary(`