Fix encode, remove generated python tokenizer

Author: jackzhxng

examples/models/llama/runner/generation.py

@@ -165,8 +165,7 @@ def text_completion(
             This method generates text completion for the provided prompt, employing nucleus sampling to introduce controlled randomness.
         """
         return self.generate(
-            # prompt_tokens=self.tokenizer.encode(prompt, bos=True, eos=False),
-            prompt_tokens=self.tokenizer.encode(prompt).ids,
+            prompt_tokens=self.tokenizer.encode(prompt, bos=True, eos=False),
             max_seq_len=self.max_seq_len,
             temperature=temperature,
             top_p=top_p,
@@ -200,10 +199,9 @@ def chat_completion(
         prompt = input("Me: ")
         while prompt and prompt != exit_prompt:
             print("LLM: ", end="", flush=True)
-            # prompt_tokens = self.tokenizer.encode(
-            #     self._format_prompt(prompt), bos=True, eos=False
-            # )
-            prompt_tokens = self.tokenizer.encode(self._format_prompt(prompt)).ids
+            prompt_tokens = self.tokenizer.encode(
+                self._format_prompt(prompt), bos=True, eos=False
+            )
             generated_tokens = self.generate(
                 prompt_tokens=pre_stop_token + prompt_tokens,
                 max_seq_len=max_seq_len,

extension/llm/tokenizer/hf_tokenizer.py

@@ -8,7 +8,6 @@
 from typing import Optional
 
 from executorch.examples.models.llama.tokenizer.tiktoken import Tokenizer as Tiktoken
-from executorch.extension.llm.tokenizer.hf_tokenizer import HFTokenizer
 from executorch.extension.llm.tokenizer.tokenizer import (
     Tokenizer as SentencePieceTokenizer,
 )
@@ -25,6 +24,8 @@ def get_tokenizer(tokenizer_path: str, tokenizer_config_path: Optional[str] = No
         tokenizer.n_words = tokenizer.get_vocab_size()
         # Keep in line with internal tokenizer apis.
         tokenizer.decode_token = lambda token: tokenizer.decode([token])
+        original_encode = tokenizer.encode
+        tokenizer.encode = lambda prompt, **kwargs: original_encode(prompt).ids
 
         if tokenizer_config_path:
             with open(tokenizer_config_path) as f: