fix sentencepiece.bpe.model download (#2454)

Author: fjjF77

.github/workflows/unittest-gpu.yml

@@ -138,7 +138,7 @@ jobs:
             rm -rf /root/.cache/aistudio/
             cd /workspace/PaddleFormers && git config --global --add safe.directory $PWD
             source $work_dir/../../../proxy
-            timeout 50m bash scripts/unit_test/ci_unit.sh ${paddle_whl}
+            timeout 100m bash scripts/unit_test/ci_unit.sh ${paddle_whl}
             '
           fi
           

paddleformers/transformers/llama/tokenizer.py

@@ -30,7 +30,7 @@
 class LlamaTokenizer(PretrainedTokenizer):
     model_input_names = ["input_ids", "attention_mask", "position_ids"]
     resource_files_names = {
-        "vocab_file": "sentencepiece.bpe.model",
+        "vocab_file": "tokenizer.model",
     }
     pretrained_resource_files_map = {
         "vocab_file": {

paddleformers/utils/download/download.py

@@ -230,13 +230,6 @@ def resolve_file_path(
                     endpoint=endpoint,
                 )
                 if is_available:
-                    import json
-
-                    with open(
-                        "/root/paddlejob/workspace/env_run/fujinji/erniekit/mine_test/download_source/download_kwargs.json",
-                        "w",
-                    ) as f:
-                        json.dump(download_kwargs, f)
                     cached_file = hf_hub_download(
                         **download_kwargs,
                     )

tests/utils/test_aistudio_download.py

@@ -24,20 +24,19 @@
 #         repo_id = "PaddleNLP/DeepSeek-R1-Distill-Qwen-1.5B"
 #         filename = "model.safetensors"
 #         revision = "master"
-#         local_dir = "./local/model"
+#         cache_dir = "./local/model"
 
 #         # 调用待测试的函数
 #         result = resolve_file_path(
 #             repo_id=repo_id,
 #             filenames=filename,
 #             revision=revision,
 #             download_hub="aistudio",
-#             local_dir=local_dir,
+#             cache_dir=cache_dir,
 #         )
 
 #         # 验证结果
-#         print(result)
-#         self.assertEqual(result, f"{local_dir}/{filename}")
+#         self.assertEqual(result, f"{cache_dir}/{repo_id}/{filename}")
 
 #     def test_aistudio_download_transformer(self):
 #         # 设置测试数据
@@ -55,7 +54,6 @@
 #         )
 
 #         # 验证结果
-#         print(result)
 #         self.assertEqual(result, f"{cache_dir}/{filename}")