support multi download source (#2427)

Co-authored-by: lugimzzz <[email protected]>
Co-authored-by: liujie44 <[email protected]>
Co-authored-by: PaddleCI <[email protected]>
Co-authored-by: Liujie0926 <[email protected]>

Author: 4 people

.github/workflows/unittest-cpu.yml

@@ -11,8 +11,6 @@ env:
   COMMIT_ID: ${{ github.event.pull_request.head.sha }}
   BRANCH: ${{ github.event.pull_request.base.ref }}
   TASK: PaddleFormers-CI-${{ github.event.pull_request.number }}-unittest-cpu
-  HF_ENDPOINT: https://hf-mirror.com
-  STUDIO_GIT_HOST: http://git.prod.idc-to-cloud.aistudio.baidu-int.com
 
 jobs:
   unittest-cpu-ci:
@@ -41,8 +39,6 @@ jobs:
             -e work_dir \
             -e no_proxy \
             -e python_version \
-            -e HF_ENDPOINT \
-            -e STUDIO_GIT_HOST \
             -w /workspace ${docker_image}
 
       - name: Download Code
@@ -92,10 +88,12 @@ jobs:
       - name: Test
         run: |
           docker exec -t $container_name /bin/bash -c '
-          source $work_dir/../../../proxy
+          source $work_dir/../../../proxy_aistudio
+          rm -rf /root/.cache/aistudio/
           cd /workspace/PaddleFormers
           set -e
           make test
+          echo "finished"
           '
 
       # - name: Upload Coverage To Codecov

.github/workflows/unittest-gpu.yml

@@ -28,11 +28,6 @@ env:
   AGILE_COMPILE_BRANCH: ${{ github.event.pull_request.base.ref }}
   CI_name: unittest-gpu-ci
   no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
-  HF_ENDPOINT: https://hf-mirror.com
-  STUDIO_GIT_HOST: http://git.prod.idc-to-cloud.aistudio.baidu-int.com
-  PPNLP_HOME: /ssd1/PaddleNLP
-  HF_DATASETS_CACHE: /ssd1/PaddleNLP/huggingface/datasets
-  TRANSFORMERS_CACHE: /ssd1/PaddleNLP/huggingface
   RUN_DOWNSTREAM: ${{ inputs.run_downstream }}
 
 defaults:
@@ -84,7 +79,6 @@ jobs:
               -v $work_dir/../../..:$work_dir/../../.. \
               -v $work_dir:/workspace \
               -v /home/.cache/pip:/home/.cache/pip \
-              -v /ssd1/PaddleNLP:/ssd1/PaddleNLP \
               -e BRANCH \
               -e AGILE_COMPILE_BRANCH \
               -e PR_ID \
@@ -96,7 +90,6 @@ jobs:
               -e paddle_whl \
               -e FLAGS_dynamic_static_unified_comm \
               -e python_version \
-              -e CUDA_VISIBLE_DEVICES=$cudaid \
               -w /workspace --runtime=nvidia $IMAGE_NAME
           fi
             
@@ -142,7 +135,9 @@ jobs:
             ln -sf $(which python${python_version}) /usr/bin/python3
             pip config set global.cache-dir "/home/.cache/pip"
             set -e
+            rm -rf /root/.cache/aistudio/
             cd /workspace/PaddleFormers && git config --global --add safe.directory $PWD
+            source $work_dir/../../../proxy
             timeout 50m bash scripts/unit_test/ci_unit.sh ${paddle_whl}
             '
           fi

Makefile

@@ -35,8 +35,8 @@ lint:
 test: unit-test
 
 unit-test:
+	DOWNLOAD_SOURCE=aistudio \
 	PYTHONPATH=$(shell pwd) pytest -v \
-		-n auto \
 		--retries 1 --retry-delay 1 \
 		--durations 20 \
 		--cov paddleformers \

paddleformers/generation/configuration_utils.py

@@ -26,7 +26,7 @@
 
 from ..transformers.configuration_utils import PretrainedConfig
 from ..utils import GENERATION_CONFIG_NAME
-from ..utils.download import resolve_file_path
+from ..utils.download import DownloadSource, resolve_file_path
 from ..utils.downloader import hf_file_exists
 from ..utils.log import logger
 
@@ -337,9 +337,7 @@ def save_pretrained(
     def from_pretrained(
         cls,
         pretrained_model_name_or_path: Union[str, os.PathLike],
-        from_hf_hub: bool = False,
-        from_aistudio: bool = False,
-        from_modelscope: bool = False,
+        download_hub: DownloadSource = None,
         config_file_name: Optional[Union[str, os.PathLike]] = None,
         cache_dir: Optional[Union[str, os.PathLike]] = None,
         force_download: bool = False,
@@ -358,8 +356,8 @@ def from_pretrained(
                 - a path to a *directory* containing a configuration file saved using the
                   [`~PretrainedConfig.save_pretrained`] method, e.g., `./my_model_directory/`.
                 - a path or url to a saved configuration JSON *file*, e.g., `./my_model_directory/configuration.json`.
-            from_hf_hub (bool, *optional*):
-                load config from huggingface hub: https://huggingface.co/models
+            download_hub (DownloadSource, *optional*):
+                The source for model downloading, options include `huggingface`, `aistudio`, `modelscope`, default `aistudio`.
             cache_dir (`str` or `os.PathLike`, *optional*):
                 Path to a directory in which a downloaded pretrained model configuration should be cached if the
                 standard cache should not be used.
@@ -418,9 +416,7 @@ def from_pretrained(
             subfolder,
             cache_dir=cache_dir,
             force_download=force_download,
-            from_aistudio=from_aistudio,
-            from_hf_hub=from_hf_hub,
-            from_modelscope=from_modelscope,
+            download_hub=download_hub,
         )
         assert (
             resolved_config_file is not None

paddleformers/transformers/auto/configuration.py

@@ -294,19 +294,15 @@ def from_pretrained(cls, pretrained_model_name_or_path: str, *model_args, **kwar
         subfolder = kwargs.get("subfolder", "")
         if subfolder is None:
             subfolder = ""
-        from_hf_hub = kwargs.get("from_hf_hub", False)
-        from_aistudio = kwargs.get("from_aistudio", False)
-        from_modelscope = kwargs.get("from_modelscope", False)
         cache_dir = kwargs.pop("cache_dir", None)
+        download_hub = kwargs.get("download_hub", None)
 
         config_file = resolve_file_path(
             pretrained_model_name_or_path,
             [cls.config_file, cls.legacy_config_file],
             subfolder,
             cache_dir=cache_dir,
-            from_hf_hub=from_hf_hub,
-            from_aistudio=from_aistudio,
-            from_modelscope=from_modelscope,
+            download_hub=download_hub,
         )
         config_dict, unused_kwargs = PretrainedConfig.get_config_dict(pretrained_model_name_or_path, **kwargs)
         if "model_type" in config_dict:

paddleformers/transformers/auto/image_processing.py

@@ -128,9 +128,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
         subfolder = kwargs.get("subfolder", "")
         if subfolder is None:
             subfolder = ""
-        from_hf_hub = kwargs.get("from_hf_hub", False)
-        from_aistudio = kwargs.get("from_aistudio", False)
-        from_modelscope = kwargs.get("from_modelscope", False)
+        download_hub = kwargs.get("download_hub", None)
         kwargs["subfolder"] = subfolder
         kwargs["cache_dir"] = cache_dir
 
@@ -157,9 +155,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
             [cls.image_processor_config_file],
             subfolder,
             cache_dir=cache_dir,
-            from_hf_hub=from_hf_hub,
-            from_aistudio=from_aistudio,
-            from_modelscope=from_modelscope,
+            download_hub=download_hub,
         )
         if config_file is not None and os.path.exists(config_file):
             processor_class = cls._get_image_processor_class_from_config(

paddleformers/transformers/auto/modeling.py

@@ -223,9 +223,7 @@ def _from_pretrained(cls, pretrained_model_name_or_path, task=None, *model_args,
             else:
                 print("We only support task choice for AutoModel.")
         cache_dir = kwargs.get("cache_dir", None)
-        from_hf_hub = kwargs.get("from_hf_hub", False)
-        from_aistudio = kwargs.get("from_aistudio", False)
-        from_modelscope = kwargs.get("from_modelscope", False)
+        download_hub = kwargs.get("download_hub", None)
         subfolder = kwargs.get("subfolder", "")
         if subfolder is None:
             subfolder = ""
@@ -273,9 +271,7 @@ def _from_pretrained(cls, pretrained_model_name_or_path, task=None, *model_args,
             [cls.model_config_file, cls.legacy_model_config_file],
             subfolder,
             cache_dir=cache_dir,
-            from_hf_hub=from_hf_hub,
-            from_aistudio=from_aistudio,
-            from_modelscope=from_modelscope,
+            download_hub=download_hub,
         )
         if config_file is not None and os.path.exists(config_file):
             model_class = cls._get_model_class_from_config(pretrained_model_name_or_path, config_file)

paddleformers/transformers/auto/processing.py

@@ -136,9 +136,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
         subfolder = kwargs.get("subfolder", "")
         if subfolder is None:
             subfolder = ""
-        from_hf_hub = kwargs.get("from_hf_hub", False)
-        from_aistudio = kwargs.get("from_aistudio", False)
-        from_modelscope = kwargs.get("from_modelscope", False)
+        download_hub = kwargs.get("download_hub", None)
         kwargs["subfolder"] = subfolder
         kwargs["cache_dir"] = cache_dir
 
@@ -165,9 +163,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
             [cls.processor_config_file],
             subfolder,
             cache_dir=cache_dir,
-            from_hf_hub=from_hf_hub,
-            from_aistudio=from_aistudio,
-            from_modelscope=from_modelscope,
+            download_hub=download_hub,
         )
         if config_file is not None and os.path.exists(config_file):
             processor_class = cls._get_processor_class_from_config(

paddleformers/transformers/auto/tokenizer.py

@@ -226,6 +226,7 @@ def get_tokenizer_config(
     tokenizer.save_pretrained("tokenizer-test")
     tokenizer_config = get_tokenizer_config("tokenizer-test")
     ```"""
+    download_hub = kwargs.get("download_hub", None)
 
     resolved_config_file = resolve_file_path(
         pretrained_model_name_or_path,
@@ -238,6 +239,7 @@ def get_tokenizer_config(
         revision=revision,
         local_files_only=local_files_only,
         subfolder=subfolder,
+        download_hub=download_hub,
     )
     if resolved_config_file is None:
         logger.info("Could not locate the tokenizer configuration file, will try to use the model config instead.")

paddleformers/transformers/configuration_utils.py

@@ -837,9 +837,7 @@ def _get_config_dict(
         cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs
     ) -> Tuple[Dict[str, Any], Dict[str, Any]]:
         cache_dir = kwargs.pop("cache_dir", None)
-        from_hf_hub = kwargs.pop("from_hf_hub", False)
-        from_aistudio = kwargs.pop("from_aistudio", False)
-        from_modelscope = kwargs.pop("from_modelscope", False)
+        download_hub = kwargs.pop("download_hub", None)
         subfolder = kwargs.pop("subfolder", "")
         if subfolder is None:
             subfolder = ""
@@ -879,9 +877,7 @@ def _get_config_dict(
             subfolder,
             cache_dir=cache_dir,
             force_download=force_download,
-            from_aistudio=from_aistudio,
-            from_hf_hub=from_hf_hub,
-            from_modelscope=from_modelscope,
+            download_hub=download_hub,
         )
         if resolved_config_file is None:
             return None, kwargs