Tokenizer update (#2457)

Author: SdeeRK

paddleformers/transformers/auto/tokenizer.py

@@ -13,13 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import json
-import logging
 import os
 import warnings
 from typing import Dict, Optional, Union
 
 import transformers as hf
-from transformers import PretrainedConfig
+from transformers import AutoConfig, PretrainedConfig
 from transformers.dynamic_module_utils import (
     get_class_from_dynamic_module,
     resolve_trust_remote_code,
@@ -40,10 +39,8 @@
 from transformers.utils import cached_file
 
 from ...utils.download import DownloadSource, resolve_file_path
+from ...utils.log import logger
 from ..tokenizer_utils import PaddleTokenizerMixin
-from .configuration import AutoConfig
-
-logger = logging.getLogger(__name__)
 
 
 def get_paddleformers_tokenizer_config(
@@ -158,7 +155,6 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
         download_hub = kwargs.get("download_hub", None)
         if download_hub is None:
             download_hub = os.environ.get("DOWNLOAD_SOURCE", "huggingface")
-        logger.info(f"Using download source: {download_hub}")
         use_auth_token = kwargs.pop("use_auth_token", None)
         if use_auth_token is not None:
             warnings.warn(
@@ -216,7 +212,29 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
                 kwargs["_commit_hash"] = tokenizer_config["_commit_hash"]
             config_tokenizer_class = tokenizer_config.get("tokenizer_class")
         else:
-            tokenizer_config = get_paddleformers_tokenizer_config(pretrained_model_name_or_path, **kwargs)
+            try:
+                tokenizer_config = get_paddleformers_tokenizer_config(pretrained_model_name_or_path, **kwargs)
+            except Exception as e:
+                if any(
+                    keyword in str(e).lower()
+                    for keyword in ["not exist", "not found", "entrynotfound", "notexist", "does not appear"]
+                ):
+                    hf_link = f"https://huggingface.co/{pretrained_model_name_or_path}"
+                    modelscope_link = f"https://modelscope.cn/models/{pretrained_model_name_or_path}"
+                    encoded_model_name = pretrained_model_name_or_path.replace("/", "%2F")
+                    aistudio_link = f"https://aistudio.baidu.com/modelsoverview?sortBy=weight&q={encoded_model_name}"
+
+                    raise ValueError(
+                        f"Unable to find {TOKENIZER_CONFIG_FILE} in the model repository '{pretrained_model_name_or_path}'. Please check:\n"
+                        f"The model repository ID is correct for your chosen source:\n"
+                        f"   - Hugging Face Hub: {hf_link}\n"
+                        f"   - ModelScope: {modelscope_link}\n"
+                        f"   - AI Studio: {aistudio_link}\n"
+                        f"Note: The repository ID may differ between ModelScope, AI Studio, and Hugging Face Hub.\n"
+                        f"You are currently using the download source: {download_hub}. Please check the repository ID on the official website."
+                    ) from None
+                else:
+                    raise
             config_tokenizer_class = tokenizer_config.get("tokenizer_class")
 
         tokenizer_auto_map = None

paddleformers/transformers/tokenizer_utils.py

@@ -209,14 +209,38 @@ def from_pretrained(
                     download_hub=download_hub,
                     local_files_only=local_files_only,
                 )
-            except Exception:
+            except (FileNotFoundError, EnvironmentError):
                 pass
+            except Exception as e:
+                raise e
         # 获得cache_dir的目录
         for file_id, file_path in resolved_vocab_files.items():
             if resolved_vocab_files[file_id] is not None:
                 cache_dir = os.path.dirname(resolved_vocab_files[file_id])
                 break
 
+        if not any(key in resolved_vocab_files for key in cls.vocab_files_names.keys()):
+            hf_link = f"https://huggingface.co/{pretrained_model_name_or_path}"
+            modelscope_link = f"https://modelscope.cn/models/{pretrained_model_name_or_path}"
+            encoded_model_name = pretrained_model_name_or_path.replace("/", "%2F")
+            aistudio_link = f"https://aistudio.baidu.com/modelsoverview?sortBy=weight&q={encoded_model_name}"
+
+            raise ValueError(
+                f"No vocabulary files found for model '{pretrained_model_name_or_path}'. "
+                f"Please check:\n"
+                f"1. The model repository ID is correct for your chosen source:\n"
+                f"   - Hugging Face Hub: {hf_link}\n"
+                f"   - ModelScope: {modelscope_link}\n"
+                f"   - AI Studio: {aistudio_link}\n"
+                f"2. You have permission to access this model repository\n"
+                f"3. Network connection is working properly\n"
+                f"4. Try clearing cache and downloading again\n"
+                f"Expected vocabulary files: {list(cls.vocab_files_names.keys())}\n"
+                f"Valid files found: {list(resolved_vocab_files.keys())}\n"
+                f"Note: The repository ID may differ between ModelScope, AI Studio, and Hugging Face Hub.\n"
+                f"You are currently using the download source: {download_hub}. Please check the repository ID on the official website."
+            )
+
         return super()._from_pretrained(
             resolved_vocab_files,
             pretrained_model_name_or_path,

paddleformers/utils/download/download.py

@@ -180,7 +180,7 @@ def resolve_file_path(
         cache_file_name = hf_try_to_load_from_cache(repo_id, filename, cache_dir, subfolder, revision, repo_type)
         if download_hub == DownloadSource.HUGGINGFACE and cache_file_name is _CACHED_NO_EXIST:
             cache_file_name = None
-        if cache_file_name is not None:
+        if cache_file_name is not None and os.path.exists(str(cache_file_name)):
             return cache_file_name
 
     # download file from different origins

requirements-dev.txt

@@ -17,4 +17,5 @@ huggingface_hub>=0.19.2
 tiktoken
 tokenizers<=0.20.3; python_version<="3.8"
 tokenizers>=0.21,<0.22; python_version>"3.8"
-modelscope
+modelscope
+transformers

requirements.txt

@@ -19,4 +19,5 @@ ml_dtypes
 tokenizers<=0.20.3; python_version<="3.8"
 tokenizers>=0.21,<0.22; python_version>"3.8"
 omegaconf
-modelscope
+modelscope
+transformers

tests/transformers/qwen2/test_tokenizer.py

@@ -40,11 +40,11 @@ def tearDown(self):
                 shutil.rmtree(test_dir)
 
     def test_slow_tokenizer_from_pretrained(self):
-        tokenizer = Qwen2Tokenizer.from_pretrained(self.from_pretrained_id, from_aistudio=True)
+        tokenizer = Qwen2Tokenizer.from_pretrained(self.from_pretrained_id, download_hub="aistudio")
         self.assertTrue(tokenizer is not None)
 
     def test_slow_tokenizer_save_pretrained(self):
-        tokenizer = Qwen2Tokenizer.from_pretrained(self.from_pretrained_id, from_aistudio=True)
+        tokenizer = Qwen2Tokenizer.from_pretrained(self.from_pretrained_id)
         special_tokens_dict = {"additional_special_tokens": ["[ENT_START]", "[ENT_END]"]}
         tokenizer.add_special_tokens(special_tokens_dict)
         tokenizer.add_tokens(["new_word", "another_word"])
@@ -53,11 +53,11 @@ def test_slow_tokenizer_save_pretrained(self):
         self.assertTrue(os.path.exists("./slow_tokenizer/tokenizer_config.json"))
 
     def test_fast_tokenizer_from_pretrained(self):
-        tokenizer = Qwen2TokenizerFast.from_pretrained(self.from_pretrained_id, from_aistudio=True)
+        tokenizer = Qwen2TokenizerFast.from_pretrained(self.from_pretrained_id, download_hub="aistudio")
         self.assertTrue(tokenizer is not None)
 
     def test_fast_tokenizer_save_pretrained(self):
-        tokenizer = Qwen2TokenizerFast.from_pretrained(self.from_pretrained_id, from_aistudio=True)
+        tokenizer = Qwen2TokenizerFast.from_pretrained(self.from_pretrained_id, download_hub="aistudio")
         special_tokens_dict = {"additional_special_tokens": ["[ENT_START]", "[ENT_END]"]}
         tokenizer.add_special_tokens(special_tokens_dict)
         tokenizer.add_tokens(["new_word", "another_word"])
@@ -66,7 +66,7 @@ def test_fast_tokenizer_save_pretrained(self):
         self.assertTrue(os.path.exists("./fast_tokenizer/tokenizer_config.json"))
 
     def test_tokenize(self):
-        tokenizer = Qwen2TokenizerFast.from_pretrained(self.from_pretrained_id, from_aistudio=True)
+        tokenizer = Qwen2TokenizerFast.from_pretrained(self.from_pretrained_id, download_hub="aistudio")
         text = "hello world, this is a tokenizer test"
         output_dict = tokenizer(text)
         decode_text = tokenizer.decode(output_dict["input_ids"], skip_special_tokens=True)

tests/transformers/test_hf_tokenizer.py

@@ -19,7 +19,7 @@
 from paddleformers.transformers import AutoTokenizer, Qwen2Tokenizer
 
 
-@unittest.skip("multi source download CI not support")
+@unittest.skip("don't support multisource download")
 class TestHFMultiSourceTokenizer(unittest.TestCase):
     def encode(self, tokenizer):
         input_text = "hello world, 你好"
@@ -68,7 +68,7 @@ def test_auto_tokenizer(self):
 
 class TestHFTokenizer(unittest.TestCase):
     def setUp(self):
-        self.tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-7B", from_hf_hub=True)
+        self.tokenizer = AutoTokenizer.from_pretrained("PaddleNLP/Qwen2.5-7B")
 
     def test_encode(self):
         input_text = "hello world, this is paddle format checker"