Remove old tokenizer/ directory in ExecuTorch (#39)

Summary:

X-link: pytorch/executorch#9728

See what happens

Differential Revision: D72007597

Author: larryliu0820

pytorch_tokenizers/tools/llama2c/convert.py

@@ -11,7 +11,7 @@
 
 import argparse
 
-from pytorch_tokenizers import Llama2cTokenizer
+from pytorch_tokenizers.llama2c import Llama2cTokenizer
 
 
 if __name__ == "__main__":

pytorch_tokenizers/tools/llama2c/targets.bzl

@@ -24,6 +24,9 @@ def define_common_targets():
         external_deps = [
             "sentencepiece-py",
         ],
+        deps = [
+            "//pytorch/tokenizers/pytorch_tokenizers:tokenizers",
+        ],
     )
 
     runtime.python_binary(

setup.py

@@ -5,7 +5,7 @@
 # LICENSE file in the root directory of this source tree.
 # @lint-ignore-every LICENSELINT
 # type: ignore[syntax]
-from setuptools import find_packages, setup
+from setuptools import setup
 
 with open("README.md", "r") as f:
     long_description = f.read()
@@ -14,6 +14,5 @@
     version="0.1.0",
     long_description=long_description,
     long_description_content_type="text/markdown",
-    packages=find_packages(where="pytorch_tokenizers"),
-    package_dir={"": "pytorch_tokenizers"},
+    packages=["pytorch_tokenizers"],
 )