fix(hf_tokenizer): Rename to HFTokenizer and corresponding flags

#1251
Branch: TokenizersTokenizer-1251

Co-Authored-By: [email protected]
Signed-off-by: Gabe Goodhart <[email protected]>

Author: gabe-l-hart

tokenizer/tokenizers.py renamed to tokenizer/hf_tokenizer.py

@@ -16,9 +16,9 @@
 from .base import TokenizerBase
 
 
-class TokenizersTokenizer(TokenizerBase):
+class HFTokenizer(TokenizerBase):
     """
-    Wrapper around the `tokenizers` library for API compatibility
+    Wrapper around the Huggingface `tokenizers` library for API compatibility
     """
 
     def __init__(self, file_path: str):

torchchat/cli/builder.py

@@ -204,7 +204,7 @@ class TokenizerArgs:
     tokenizer_path: Optional[Union[Path, str]] = None
     is_sentencepiece: bool = False
     is_tiktoken: bool = False
-    is_tokenizers: bool = False
+    is_hf_tokenizer: bool = False
     t: Optional[Any] = None
 
     def __post_init__(self):
@@ -214,7 +214,7 @@ def __post_init__(self):
             self.t = TiktokenTokenizer(model_path=str(self.tokenizer_path))
             self.is_tiktoken = True
             self.is_sentencepiece = False
-            self.is_tokenizers = False
+            self.is_hf_tokenizer = False
             return
         except:
             pass
@@ -225,25 +225,25 @@ def __post_init__(self):
             self.t = SentencePieceProcessor(model_file=str(self.tokenizer_path))
             self.is_tiktoken = False
             self.is_sentencepiece = True
-            self.is_tokenizers = False
+            self.is_hf_tokenizer = False
             return
         except:
             pass
 
         try:
-            from tokenizer.tokenizers import TokenizersTokenizer
+            from tokenizer.hf_tokenizer import HFTokenizer
 
-            self.t = TokenizersTokenizer(str(self.tokenizer_path))
+            self.t = HFTokenizer(str(self.tokenizer_path))
             self.is_tiktoken = False
             self.is_sentencepiece = False
-            self.is_tokenizers = True
+            self.is_hf_tokenizer = True
             return
         except:
             pass
 
         self.is_tiktoken = False
         self.is_sentencepiece = False
-        self.is_tokenizers = False
+        self.is_hf_tokenizer = False
         self.t = None
         return
 
@@ -255,25 +255,25 @@ def validate_model(
         if model is None:
             return
 
-        if len(list(filter(lambda x: x, [self.is_tiktoken, self.is_tokenizers, self.is_sentencepiece]))) != 1:
+        if sum([self.is_tiktoken, self.is_hf_tokenizer, self.is_sentencepiece]) != 1:
             raise RuntimeError(f"no tokenizer was found at {self.tokenizer_path}")
 
         is_tiktoken = self.is_tiktoken
         is_sentencepiece = self.is_sentencepiece
-        is_tokenizers = self.is_tokenizers
+        is_hf_tokenizer = self.is_hf_tokenizer
         use_tiktoken = model.config.use_tiktoken
-        use_tokenizers = model.config.use_tokenizers
-        use_sentencepiece = not (use_tiktoken or use_tokenizers)
+        use_hf_tokenizer = model.config.use_hf_tokenizer
+        use_sentencepiece = not (use_tiktoken or use_hf_tokenizer)
 
         if (
             (is_tiktoken and not use_tiktoken) or
-            (is_tokenizers and not use_tokenizers) or
+            (is_hf_tokenizer and not use_hf_tokenizer) or
             (is_sentencepiece and not use_sentencepiece)
         ):
             raise RuntimeError(
                 "model-specified tokenizer ({}) does not match provided tokenizer ({}) for {}".format(
-                    tokenizer_setting_to_name(use_tiktoken, use_tokenizers),
-                    tokenizer_setting_to_name(is_tiktoken, is_tokenizers),
+                    tokenizer_setting_to_name(use_tiktoken, use_hf_tokenizer),
+                    tokenizer_setting_to_name(is_tiktoken, is_hf_tokenizer),
                     model_description,
                 )
             )

torchchat/model.py

@@ -272,7 +272,7 @@ class TransformerArgs:
     ffn_dim_multiplier: Optional[int] = None
     # Select the desired tokenizer. Defaults to sentencepiece
     use_tiktoken: bool = False
-    use_tokenizers: bool = False
+    use_hf_tokenizer: bool = False
     max_seq_length: int = 8192
     rope_scaling: Optional[Dict[str, Any]] = None
     # For pipeline parallel
@@ -329,14 +329,14 @@ class ModelArgs:
     model_type: ModelType
     transformer_args: Dict[str, Dict[str, Any]]
     use_tiktoken: bool
-    use_tokenizers: bool
+    use_hf_tokenizer: bool
 
     def __init__(
         self,
         transformer_args: Dict[str, Dict[str, Any]],
         model_type: ModelType = ModelType.TextOnly,
         use_tiktoken: bool = False,
-        use_tokenizers: bool = False,
+        use_hf_tokenizer: bool = False,
     ) -> None:
         self._sanity_check(transformer_args, model_type)
 
@@ -345,7 +345,7 @@ def __init__(
 
         # Model-level attributes
         self.use_tiktoken = use_tiktoken
-        self.use_tokenizers = use_tokenizers
+        self.use_hf_tokenizer = use_hf_tokenizer
 
     def _sanity_check(
         self,
@@ -372,8 +372,8 @@ def from_params(cls, params_path):
             }
 
         use_tiktoken = loaded_params.get("use_tiktoken", False)
-        use_tokenizers = loaded_params.get("use_tokenizers", False)
-        return cls(transformer_args, model_type, use_tiktoken, use_tokenizers)
+        use_hf_tokenizer = loaded_params.get("use_hf_tokenizer", False)
+        return cls(transformer_args, model_type, use_tiktoken, use_hf_tokenizer)
 
     @classmethod
     def from_table(cls, name: str):