Add library name and version to hf_hub_download for better analytics (#3881)

* add lib name and version for hf hub download

* add hf hub version

Author: sijunhe

paddlenlp/transformers/auto/modeling.py

@@ -20,6 +20,7 @@
 
 from huggingface_hub import hf_hub_download
 
+from paddlenlp import __version__
 from paddlenlp.transformers import *
 from paddlenlp.utils.downloader import (COMMUNITY_MODEL_PREFIX,
                                         get_path_from_url)
@@ -237,7 +238,9 @@ def _from_pretrained(cls,
         if from_hf_hub:
             config_file = hf_hub_download(repo_id=pretrained_model_name_or_path,
                                           filename=cls.model_config_file,
-                                          cache_dir=MODEL_HOME)
+                                          cache_dir=MODEL_HOME,
+                                          library_name="PaddleNLP",
+                                          library_version=__version__)
             if os.path.exists(config_file):
                 model_class = cls._get_model_class_from_config(
                     pretrained_model_name_or_path, config_file)

paddlenlp/transformers/auto/tokenizer.py

@@ -20,6 +20,7 @@
 
 from huggingface_hub import hf_hub_download
 
+from paddlenlp import __version__
 from paddlenlp.transformers import *
 from paddlenlp.utils.downloader import (COMMUNITY_MODEL_PREFIX,
                                         get_path_from_url)
@@ -242,7 +243,9 @@ def from_pretrained(cls,
         if from_hf_hub:
             config_file = hf_hub_download(repo_id=pretrained_model_name_or_path,
                                           filename=cls.tokenizer_config_file,
-                                          cache_dir=MODEL_HOME)
+                                          cache_dir=MODEL_HOME,
+                                          library_name="PaddleNLP",
+                                          library_version=__version__)
             if os.path.exists(config_file):
                 tokenizer_class = cls._get_tokenizer_class_from_config(
                     pretrained_model_name_or_path, config_file, use_fast)

paddlenlp/transformers/configuration_utils.py

@@ -748,7 +748,9 @@ def _get_config_dict(cls, pretrained_model_name_or_path: Union[str,
             resolved_config_file = hf_hub_download(
                 repo_id=pretrained_model_name_or_path,
                 filename=CONFIG_NAME,
-                cache_dir=MODEL_HOME)
+                cache_dir=MODEL_HOME,
+                library_name="PaddleNLP",
+                library_version=__version__)
 
         # 3. get the configuration file from url, eg: https://ip/path/to/model_config.jsons
         elif is_url(pretrained_model_name_or_path):

paddlenlp/transformers/model_utils.py

@@ -31,12 +31,13 @@
 from paddle.nn import Embedding, Layer
 # TODO(fangzeyang) Temporary fix and replace by paddle framework downloader later
 from paddle.utils.download import is_url
-from paddlenlp.utils.downloader import (download_check, COMMUNITY_MODEL_PREFIX)
-from paddlenlp.utils.downloader import get_path_from_url_with_filelock
-from paddlenlp.utils.env import MODEL_HOME, LOCK_FILE_HOME
 
-from paddlenlp.utils.log import logger
+from paddlenlp import __version__
+from paddlenlp.utils.downloader import (COMMUNITY_MODEL_PREFIX, download_check,
+                                        get_path_from_url_with_filelock)
+from paddlenlp.utils.env import LOCK_FILE_HOME, MODEL_HOME
 from paddlenlp.utils.file_lock import FileLock
+from paddlenlp.utils.log import logger
 
 from .configuration_utils import PretrainedConfig
 from .generation_utils import GenerationMixin
@@ -439,7 +440,9 @@ def from_pretrained(cls,
                 resolved_resource_files[file_id] = hf_hub_download(
                     repo_id=pretrained_model_name_or_path,
                     filename=file_path,
-                    cache_dir=MODEL_HOME)
+                    cache_dir=MODEL_HOME,
+                    library_name="PaddleNLP",
+                    library_version=__version__)
             else:
                 path = os.path.join(default_root, file_path.split('/')[-1])
                 if os.path.exists(path):
@@ -848,7 +851,9 @@ def _resolve_model_file_path(cls: Type[PretrainedModel],
             return hf_hub_download(
                 repo_id=pretrained_model_name_or_path,
                 filename=cls.resource_files_names['model_state'],
-                cache_dir=MODEL_HOME)
+                cache_dir=MODEL_HOME,
+                library_name="PaddleNLP",
+                library_version=__version__)
 
         # 2. when it is model-name
         if pretrained_model_name_or_path in cls.pretrained_init_configuration:

paddlenlp/transformers/tokenizer_utils_base.py

@@ -32,8 +32,9 @@
 import paddle
 from huggingface_hub import hf_hub_download
 
-from paddlenlp.utils.downloader import COMMUNITY_MODEL_PREFIX
-from paddlenlp.utils.downloader import get_path_from_url_with_filelock
+from paddlenlp import __version__
+from paddlenlp.utils.downloader import (COMMUNITY_MODEL_PREFIX,
+                                        get_path_from_url_with_filelock)
 from paddlenlp.utils.env import MODEL_HOME
 from paddlenlp.utils.log import logger
 
@@ -1567,7 +1568,9 @@ def from_pretrained(cls,
                 resolved_vocab_files[file_id] = hf_hub_download(
                     repo_id=pretrained_model_name_or_path,
                     filename=file_path,
-                    cache_dir=MODEL_HOME)
+                    cache_dir=MODEL_HOME,
+                    library_name="PaddleNLP",
+                    library_version=__version__)
             else:
                 path = os.path.join(default_root, file_path.split('/')[-1])
                 if os.path.exists(path):

requirements.txt

@@ -8,7 +8,7 @@ datasets >= 2.0.0
 tqdm
 paddlefsl
 sentencepiece
-huggingface_hub
+huggingface_hub>=0.10.1
 paddle2onnx
 protobuf>=3.1.0, <=3.20.0
 visualdl