Support HUGGINGFACE_HUB_CACHE in HF hub download through os env (#3911)

* support hf hub cache

* merge conflict

Co-authored-by: 骑马小猫 <[email protected]>

Author: sijunhe

paddlenlp/transformers/auto/tokenizer.py

@@ -24,7 +24,7 @@
 from paddlenlp.transformers import *
 from paddlenlp.utils.downloader import (COMMUNITY_MODEL_PREFIX,
                                         get_path_from_url)
-from paddlenlp.utils.env import MODEL_HOME
+from paddlenlp.utils.env import HF_CACHE_HOME
 from paddlenlp.utils.import_utils import is_fast_tokenizer_available
 from paddlenlp.utils.log import logger
 
@@ -243,7 +243,7 @@ def from_pretrained(cls,
         if from_hf_hub:
             config_file = hf_hub_download(repo_id=pretrained_model_name_or_path,
                                           filename=cls.tokenizer_config_file,
-                                          cache_dir=MODEL_HOME,
+                                          cache_dir=HF_CACHE_HOME,
                                           library_name="PaddleNLP",
                                           library_version=__version__)
             if os.path.exists(config_file):

paddlenlp/transformers/configuration_utils.py

@@ -31,7 +31,7 @@
 
 from paddlenlp import __version__
 from paddlenlp.transformers.utils import resolve_cache_dir
-from paddlenlp.utils.env import MODEL_HOME
+from paddlenlp.utils.env import HF_CACHE_HOME
 from paddlenlp.utils.log import logger
 
 from ..utils import CONFIG_NAME
@@ -748,7 +748,7 @@ def _get_config_dict(cls, pretrained_model_name_or_path: Union[str,
             resolved_config_file = hf_hub_download(
                 repo_id=pretrained_model_name_or_path,
                 filename=CONFIG_NAME,
-                cache_dir=MODEL_HOME,
+                cache_dir=HF_CACHE_HOME,
                 library_name="PaddleNLP",
                 library_version=__version__)
 

paddlenlp/transformers/model_utils.py

@@ -35,7 +35,7 @@
 from paddlenlp import __version__
 from paddlenlp.utils.downloader import (COMMUNITY_MODEL_PREFIX, download_check,
                                         get_path_from_url_with_filelock)
-from paddlenlp.utils.env import LOCK_FILE_HOME, MODEL_HOME
+from paddlenlp.utils.env import HF_CACHE_HOME, LOCK_FILE_HOME, MODEL_HOME
 from paddlenlp.utils.file_lock import FileLock
 from paddlenlp.utils.log import logger
 
@@ -440,7 +440,7 @@ def from_pretrained(cls,
                 resolved_resource_files[file_id] = hf_hub_download(
                     repo_id=pretrained_model_name_or_path,
                     filename=file_path,
-                    cache_dir=MODEL_HOME,
+                    cache_dir=HF_CACHE_HOME,
                     library_name="PaddleNLP",
                     library_version=__version__)
             else:
@@ -851,7 +851,7 @@ def _resolve_model_file_path(cls: Type[PretrainedModel],
             return hf_hub_download(
                 repo_id=pretrained_model_name_or_path,
                 filename=cls.resource_files_names['model_state'],
-                cache_dir=MODEL_HOME,
+                cache_dir=HF_CACHE_HOME,
                 library_name="PaddleNLP",
                 library_version=__version__)
 

paddlenlp/utils/env.py

@@ -50,6 +50,7 @@ def _get_sub_home(directory, parent_home=_get_ppnlp_home()):
 USER_HOME = _get_user_home()
 PPNLP_HOME = _get_ppnlp_home()
 MODEL_HOME = _get_sub_home('models')
+HF_CACHE_HOME = os.environ.get('HUGGINGFACE_HUB_CACHE', MODEL_HOME)
 DATA_HOME = _get_sub_home('datasets')
 LOCK_FILE_HOME = _get_sub_home(".lock")
 DOWNLOAD_SERVER = "http://paddlepaddle.org.cn/paddlehub"