Remove old tokenizer/ directory in ExecuTorch

Differential Revision: D72007597

Pull Request resolved: #9728

Author: larryliu0820

.ci/scripts/test_llama.sh

@@ -269,7 +269,7 @@ $PYTHON_EXECUTABLE -m examples.models.llama.export_llama ${EXPORT_ARGS}
 
 # Create tokenizer.bin.
 echo "Creating tokenizer.bin"
-$PYTHON_EXECUTABLE -m extension.llm.tokenizer.tokenizer -t tokenizer.model -o tokenizer.bin
+$PYTHON_EXECUTABLE -m pytorch_tokenizers.tools.llama2c.convert -t tokenizer.model -o tokenizer.bin
 
 
 RUNTIME_ARGS="--model_path=${EXPORTED_MODEL_NAME} --tokenizer_path=tokenizer.bin --prompt=Once --temperature=0 --seq_len=10 --warmup=1"

.ci/scripts/test_llama_torchao_lowbit.sh

@@ -55,7 +55,7 @@ cmake --build cmake-out/examples/models/llama -j16 --config Release
 download_stories_model_artifacts
 
 echo "Creating tokenizer.bin"
-$PYTHON_EXECUTABLE -m extension.llm.tokenizer.tokenizer -t tokenizer.model -o tokenizer.bin
+$PYTHON_EXECUTABLE -m pytorch_tokenizers.tools.llama2c.convert -t tokenizer.model -o tokenizer.bin
 
 # Export model
 LLAMA_CHECKPOINT=stories110M.pt

.ci/scripts/test_phi_3_mini.sh

@@ -56,7 +56,7 @@ cmake_build_phi_3_mini() {
 prepare_tokenizer() {
   echo "Downloading and converting tokenizer.model"
   wget -O tokenizer.model "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct/resolve/main/tokenizer.model?download=true"
-  $PYTHON_EXECUTABLE -m executorch.extension.llm.tokenizer.tokenizer -t tokenizer.model -o tokenizer.bin
+  $PYTHON_EXECUTABLE -m pytorch_tokenizers.tools.llama2c.convert -t tokenizer.model -o tokenizer.bin
 }
 
 # Export phi-3-mini model to pte

.ci/scripts/test_qnn_static_llama.sh

@@ -30,7 +30,7 @@ pip install graphviz
 # Download stories llama110m artifacts
 download_stories_model_artifacts
 echo "Creating tokenizer.bin"
-$PYTHON_EXECUTABLE -m extension.llm.tokenizer.tokenizer -t tokenizer.model -o tokenizer.bin
+$PYTHON_EXECUTABLE -m pytorch_tokenizers.tools.llama2c.convert -t tokenizer.model -o tokenizer.bin
 
 set +e
 # Compile only as weight sharing is not applicable on x86

CMakeLists.txt

@@ -757,7 +757,7 @@ if(EXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR)
 endif()
 
 if(EXECUTORCH_BUILD_EXTENSION_LLM)
-  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/llm/tokenizer)
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/llm/tokenizers)
 endif()
 
 if(EXECUTORCH_BUILD_EXTENSION_MODULE)

examples/models/llama/TARGETS

@@ -202,7 +202,7 @@ runtime.python_library(
         ":export_library",
         "//executorch/examples/models/llama/tokenizer:tiktoken_py",
         "//executorch/extension/llm/export:export_lib",
-        "//executorch/extension/llm/tokenizer:tokenizer_py_lib",
+        "//pytorch/tokenizers/pytorch_tokenizers:tokenizers",
         "//executorch/extension/pybindings:portable_lib",
     ],
 )

examples/models/llama/eval_llama_lib.py

@@ -15,14 +15,12 @@
 from executorch.examples.models.llama.export_llama_lib import (
     get_quantizer_and_quant_params,
 )
-from executorch.examples.models.llama.tokenizer.tiktoken import Tokenizer as Tiktoken
 
 from executorch.extension.llm.export.builder import LLMEdgeManager
-from executorch.extension.llm.tokenizer.tokenizer import (
-    Tokenizer as SentencePieceTokenizer,
-)
-from executorch.extension.llm.tokenizer.utils import get_tokenizer
 from lm_eval.evaluator import simple_evaluate
+from pytorch_tokenizers import get_tokenizer
+from pytorch_tokenizers.llama2c import Llama2cTokenizer as SentencePieceTokenizer
+from pytorch_tokenizers.tiktoken import TiktokenTokenizer as Tiktoken
 from torch.nn import CrossEntropyLoss
 from tqdm import tqdm
 

examples/models/llama/evaluate/eager_eval.py

@@ -8,12 +8,10 @@
 from typing import Optional, Union
 
 import torch
-from executorch.examples.models.llama.tokenizer.tiktoken import Tokenizer as Tiktoken
-from executorch.extension.llm.tokenizer.tokenizer import (
-    Tokenizer as SentencePieceTokenizer,
-)
 
 from lm_eval.models.huggingface import HFLM as eval_wrapper
+from pytorch_tokenizers.llama2c import Llama2cTokenizer as SentencePieceTokenizer
+from pytorch_tokenizers.tiktoken import TiktokenTokenizer as Tiktoken
 
 from torch import nn
 

examples/models/llama/runner/generation.py

@@ -10,7 +10,7 @@
 
 import torch
 
-from executorch.extension.llm.tokenizer.utils import get_tokenizer
+from pytorch_tokenizers import get_tokenizer
 
 
 def sample_top_p(probs, p):

examples/models/llama/tokenizer/targets.bzl

@@ -16,7 +16,6 @@ def define_common_targets():
         ],
         exported_deps = [
             "//pytorch/tokenizers:tiktoken",
-            "//executorch/extension/llm/tokenizer:tiktoken", # TODO: remove
         ],
         visibility = [
             "@EXECUTORCH_CLIENTS",