[FastTokenizer] Fix fast_tokenizer not found when using auto tokenizer (#4060)

* Fix fast_tokenizer not found when using auto tokenizer

* faster -> fast

Co-authored-by: Linjie Chen <[email protected]>

Author: joey12300

paddlenlp/transformers/auto/tokenizer.py

@@ -21,7 +21,6 @@
 from huggingface_hub import hf_hub_download
 
 from paddlenlp import __version__
-from paddlenlp.transformers import *
 from paddlenlp.utils.downloader import COMMUNITY_MODEL_PREFIX, get_path_from_url
 from paddlenlp.utils.env import HF_CACHE_HOME, MODEL_HOME
 from paddlenlp.utils.import_utils import is_fast_tokenizer_available
@@ -119,7 +118,7 @@ def get_configurations():
         # So same config would map more than one tokenizer
         if MAPPING_NAMES.get(name, None) is None:
             MAPPING_NAMES[name] = []
-        # (tokenizer_name, is_faster)
+        # (tokenizer_name, is_fast)
         MAPPING_NAMES[name].append((tokenizer_name, fast_name != ""))
     return MAPPING_NAMES
 
@@ -135,7 +134,7 @@ class AutoTokenizer:
     MAPPING_NAMES = get_configurations()
     _tokenizer_mapping = MAPPING_NAMES
     _name_mapping = TOKENIZER_MAPPING_NAMES
-    _faster_name_mapping = FAST_TOKENIZER_MAPPING_NAMES
+    _fast_name_mapping = FAST_TOKENIZER_MAPPING_NAMES
     tokenizer_config_file = "tokenizer_config.json"
 
     def __init__(self, *args, **kwargs):
@@ -158,10 +157,10 @@ def _get_tokenizer_class_from_config(cls, pretrained_model_name_or_path, config_
             import_class = importlib.import_module(f"paddlenlp.transformers.{class_name}.tokenizer")
             tokenizer_class = getattr(import_class, init_class)
             if use_fast:
-                for faster_tokenizer_class, name in cls._faster_name_mapping.items():
+                for fast_tokenizer_class, name in cls._fast_name_mapping.items():
                     if name == class_name:
-                        import_class = importlib.import_module(f"paddlenlp.transformers.{class_name}.faster_tokenizer")
-                        tokenizer_class = getattr(import_class, faster_tokenizer_class)
+                        import_class = importlib.import_module(f"paddlenlp.transformers.{class_name}.fast_tokenizer")
+                        tokenizer_class = getattr(import_class, fast_tokenizer_class)
             return tokenizer_class
         # If no `init_class`, we use pattern recognition to recognize the tokenizer class.
         else:
@@ -219,7 +218,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, from_hf_hub=False, *mode
                 print(type(tokenizer))
                 # <class 'paddlenlp.transformers.bert.tokenizer.BertTokenizer'>
         """
-        # Default not to use faster tokenizer
+        # Default not to use fast tokenizer
         use_fast = kwargs.pop("use_fast", False)
         if "use_fast" in kwargs:
             use_fast = kwargs.pop("use_fast", False)
@@ -267,7 +266,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, from_hf_hub=False, *mode
                                         break
                                 if not is_support_fast_tokenizer:
                                     logger.warning(
-                                        f"The tokenizer {actual_tokenizer_class} doesn't have the faster version."
+                                        f"The tokenizer {actual_tokenizer_class} doesn't have the fast version."
                                         " Please check the map `paddlenlp.transformers.auto.tokenizer.FAST_TOKENIZER_MAPPING_NAMES`"
                                         " to see which fast tokenizers are currently supported."
                                     )

paddlenlp/utils/import_utils.py

@@ -13,16 +13,16 @@
 # limitations under the License.
 from __future__ import annotations
 
-import sys
+import importlib.util
 import os
-import site
 import shutil
+import site
+import sys
 from typing import Optional, Type
+
 import pip
-import importlib.util
+
 from paddlenlp.utils.log import logger
-import importlib.util
-import importlib_metadata
 
 
 def is_torch_available() -> bool:
@@ -44,12 +44,12 @@ def is_package_available(package_name: str) -> bool:
     return package_spec is not None and package_spec.has_location
 
 
-def is_faster_tokenizer_available() -> bool:
-    """check if `faster_tokenizer` ia avaliable
+def is_fast_tokenizer_available() -> bool:
+    """check if `fast_tokenizer` ia avaliable
     Returns:
-        bool: if `faster_tokenizer` is avaliable
+        bool: if `fast_tokenizer` is avaliable
     """
-    return is_package_available("faster_tokenizer")
+    return is_package_available("fast_tokenizer")
 
 
 def is_transformers_available() -> bool:
@@ -60,11 +60,6 @@ def is_transformers_available() -> bool:
     return is_package_available("transformers")
 
 
-def is_fast_tokenizer_available():
-    package_spec = importlib.util.find_spec("fast_tokenizer")
-    return package_spec is not None and package_spec.has_location
-
-
 def install_package(
     package_name: str,
     version: Optional[str] = None,