Remove old tokenizer/ directory in ExecuTorch

Differential Revision: D72007597

Pull Request resolved: meta-pytorch#39

Author: larryliu0820

pytorch_tokenizers/tools/__init__.py

@@ -11,7 +11,7 @@
 
 import argparse
 
-from pytorch_tokenizers import Llama2cTokenizer
+from pytorch_tokenizers.llama2c import Llama2cTokenizer
 
 
 if __name__ == "__main__":

pytorch_tokenizers/tools/llama2c/convert.py

@@ -24,6 +24,9 @@ def define_common_targets():
         external_deps = [
             "sentencepiece-py",
         ],
+        deps = [
+            "//pytorch/tokenizers/pytorch_tokenizers:tokenizers",
+        ],
     )
 
     runtime.python_binary(

pytorch_tokenizers/tools/llama2c/targets.bzl

@@ -14,6 +14,5 @@
     version="0.1.0",
     long_description=long_description,
     long_description_content_type="text/markdown",
-    packages=find_packages(where="pytorch_tokenizers"),
-    package_dir={"": "pytorch_tokenizers"},
+    packages=find_packages(),
 )