Lint

Author: jackzhxng

examples/models/llama/runner/eager.py

@@ -28,7 +28,7 @@ def __init__(self, args):
             params = json.loads(f.read())
         super().__init__(
             tokenizer_path=args.tokenizer_path,
-            tokenizer_config_path=args.tokenizer_config_path
+            tokenizer_config_path=args.tokenizer_config_path,
             max_seq_len=args.max_seq_length,
             max_batch_size=1,
             use_kv_cache=args.use_kv_cache,
@@ -73,12 +73,12 @@ def build_args_parser() -> argparse.ArgumentParser:
         action="store_true",
         default=False,
         help="Have multi-turn chat with the model",
-    )p
+    )
 
     parser.add_argument(
         "--tokenizer_config_path",
         type=str,
-        deafult=None,
+        default=None,
     )
 
     return parser

extension/llm/tokenizer/hf_tokenizer.py

@@ -15,6 +15,7 @@ class HuggingFaceTokenizer:
     """
     Tokenizing and encoding/decoding text using the Hugging face tokenizer.
     """
+
     def __init__(self, model_path: str, config_path: Optional[str] = None):
         """
         Initializes the Tokenizer with a tokenizer.json from HuggingFace.
@@ -30,7 +31,11 @@ def __init__(self, model_path: str, config_path: Optional[str] = None):
         if config_path:
             with open(config_path) as f:
                 tokenizer_config = json.load(f)
-                self.bos_id = self.model.token_to_id(tokenizer_config["bos_token"])if tokenizer_config["bos_token"] else None
+                self.bos_id = (
+                    self.model.token_to_id(tokenizer_config["bos_token"])
+                    if tokenizer_config["bos_token"]
+                    else None
+                )
                 self.eos_id = self.model.token_to_id(tokenizer_config["eos_token"])
         else:  # Fallback guess.
             self.bos_id = self.model.token_to_id("<|begin_of_text|>")
@@ -49,4 +54,3 @@ def decode(self, t: List[int]) -> str:
 
     def decode_token(self, t: int) -> str:
         return self.model.decode([t])
-

extension/llm/tokenizer/utils.py

@@ -4,7 +4,6 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-import json
 from typing import Optional
 
 from executorch.examples.models.llama.tokenizer.tiktoken import Tokenizer as Tiktoken