Better logging (#262)

* Progress bars

* Remove print statement

* add log progress to git_pull

* Add logging utility

* Implement logging in modules

* add to __init__

* Align on `datasets` logging

Author: LysandreJik

src/huggingface_hub/__init__.py

@@ -35,3 +35,4 @@
 from .inference_api import InferenceApi
 from .repository import Repository
 from .snapshot_download import snapshot_download
+from .utils import logging

src/huggingface_hub/commands/lfs.py

@@ -17,7 +17,6 @@
 """
 
 import json
-import logging
 import os
 import subprocess
 import sys
@@ -29,8 +28,10 @@
 from huggingface_hub.commands import BaseHuggingfaceCLICommand
 from huggingface_hub.lfs import LFS_MULTIPART_UPLOAD_COMMAND
 
+from ..utils import logging
 
-logger = logging.getLogger(__name__)
+
+logger = logging.get_logger(__name__)
 
 
 class LfsCommands(BaseHuggingfaceCLICommand):

src/huggingface_hub/file_download.py

@@ -2,7 +2,6 @@
 import fnmatch
 import io
 import json
-import logging
 import os
 import sys
 import tempfile
@@ -28,9 +27,10 @@
     REPO_TYPES_URL_PREFIXES,
 )
 from .hf_api import HfFolder
+from .utils import logging
 
 
-logger = logging.getLogger(__name__)
+logger = logging.get_logger(__name__)
 
 _PY_VERSION: str = sys.version.split()[0].rstrip("+")
 

src/huggingface_hub/hub_mixin.py

@@ -1,5 +1,4 @@
 import json
-import logging
 import os
 from pathlib import Path
 from typing import Dict, Optional, Union
@@ -10,13 +9,14 @@
 from .file_download import hf_hub_download, is_torch_available
 from .hf_api import HfApi, HfFolder
 from .repository import Repository
+from .utils import logging
 
 
 if is_torch_available():
     import torch
 
 
-logger = logging.getLogger(__name__)
+logger = logging.get_logger(__name__)
 
 
 class ModelHubMixin:

src/huggingface_hub/inference_api.py

@@ -1,12 +1,12 @@
-import logging
 from typing import Dict, List, Optional, Union
 
 import requests
 
 from .hf_api import HfApi
+from .utils import logging
 
 
-logger = logging.getLogger(__name__)
+logger = logging.get_logger(__name__)
 
 
 ENDPOINT = "https://api-inference.huggingface.co"

src/huggingface_hub/repository.py

@@ -1,4 +1,3 @@
-import logging
 import os
 import re
 import subprocess
@@ -15,9 +14,10 @@
 
 from .hf_api import ENDPOINT, HfApi, HfFolder, repo_type_and_id_from_hf_id
 from .lfs import LFS_MULTIPART_UPLOAD_COMMAND
+from .utils import logging
 
 
-logger = logging.getLogger(__name__)
+logger = logging.get_logger(__name__)
 
 
 def is_git_repo(folder: Union[str, Path]) -> bool:
@@ -313,9 +313,6 @@ def __init__(
             if is_git_repo(self.local_dir):
                 logger.debug("[Repository] is a valid git repo")
             else:
-                logger.error(
-                    "If not specifying `clone_from`, you need to pass Repository a valid git clone."
-                )
                 raise ValueError(
                     "If not specifying `clone_from`, you need to pass Repository a valid git clone."
                 )
@@ -468,7 +465,7 @@ def clone_from(self, repo_url: str, use_auth_token: Union[bool, str, None] = Non
 
                 if in_repository:
                     if is_local_clone(self.local_dir, repo_url):
-                        logger.debug(
+                        logger.warning(
                             f"{self.local_dir} is already a clone of {clean_repo_url}. Make sure you pull the latest"
                             "changes with `repo.git_pull()`."
                         )

src/huggingface_hub/utils/__init__.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+# coding=utf-8
+# Copyright 2021 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License

src/huggingface_hub/utils/logging.py

@@ -0,0 +1,143 @@
+# coding=utf-8
+# Copyright 2020 Optuna, Hugging Face
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Logging utilities. """
+
+import logging
+import os
+from logging import CRITICAL  # NOQA
+from logging import DEBUG  # NOQA
+from logging import ERROR  # NOQA
+from logging import FATAL  # NOQA
+from logging import INFO  # NOQA
+from logging import NOTSET  # NOQA
+from logging import WARN  # NOQA
+from logging import WARNING  # NOQA
+from typing import Optional
+
+
+log_levels = {
+    "debug": logging.DEBUG,
+    "info": logging.INFO,
+    "warning": logging.WARNING,
+    "error": logging.ERROR,
+    "critical": logging.CRITICAL,
+}
+
+_default_log_level = logging.WARNING
+
+
+def _get_library_name() -> str:
+    return __name__.split(".")[0]
+
+
+def _get_library_root_logger() -> logging.Logger:
+
+    return logging.getLogger(_get_library_name())
+
+
+def _get_default_logging_level():
+    """
+    If HUGGINGFACE_HUB_VERBOSITY env var is set to one of the valid choices return that as the new default level.
+    If it is not - fall back to ``_default_log_level``
+    """
+    env_level_str = os.getenv("HUGGINGFACE_HUB_VERBOSITY", None)
+    if env_level_str:
+        if env_level_str in log_levels:
+            return log_levels[env_level_str]
+        else:
+            logging.getLogger().warning(
+                f"Unknown option HUGGINGFACE_HUB_VERBOSITY={env_level_str}, "
+                f"has to be one of: { ', '.join(log_levels.keys()) }"
+            )
+    return _default_log_level
+
+
+def _configure_library_root_logger() -> None:
+    library_root_logger = _get_library_root_logger()
+    library_root_logger.setLevel(logging.INFO)
+
+
+def _reset_library_root_logger() -> None:
+    library_root_logger = _get_library_root_logger()
+    library_root_logger.setLevel(logging.NOTSET)
+
+
+def get_logger(name: Optional[str] = None) -> logging.Logger:
+    """Return a logger with the specified name.
+    This function is not supposed to be directly accessed by library users.
+    """
+
+    if name is None:
+        name = _get_library_name()
+
+    return logging.getLogger(name)
+
+
+def get_verbosity() -> int:
+    """Return the current level for the HuggingFace Hub's root logger.
+    Returns:
+        Logging level, e.g., ``huggingface_hub.logging.DEBUG`` and ``huggingface_hub.logging.INFO``.
+    .. note::
+        HuggingFace Hub has following logging levels:
+        - ``huggingface_hub.logging.CRITICAL``, ``huggingface_hub.logging.FATAL``
+        - ``huggingface_hub.logging.ERROR``
+        - ``huggingface_hub.logging.WARNING``, ``huggingface_hub.logging.WARN``
+        - ``huggingface_hub.logging.INFO``
+        - ``huggingface_hub.logging.DEBUG``
+    """
+    return _get_library_root_logger().getEffectiveLevel()
+
+
+def set_verbosity(verbosity: int) -> None:
+    """Set the level for the HuggingFace Hub's root logger.
+    Args:
+        verbosity:
+            Logging level, e.g., ``huggingface_hub.logging.DEBUG`` and ``huggingface_hub.logging.INFO``.
+    """
+    _get_library_root_logger().setLevel(verbosity)
+
+
+def set_verbosity_info():
+    return set_verbosity(INFO)
+
+
+def set_verbosity_warning():
+    return set_verbosity(WARNING)
+
+
+def set_verbosity_debug():
+    return set_verbosity(DEBUG)
+
+
+def set_verbosity_error():
+    return set_verbosity(ERROR)
+
+
+def disable_propagation() -> None:
+    """Disable propagation of the library log outputs.
+    Note that log propagation is disabled by default.
+    """
+    _get_library_root_logger().propagate = False
+
+
+def enable_propagation() -> None:
+    """Enable propagation of the library log outputs.
+    Please disable the HuggingFace Hub's default handler to prevent double logging if the root logger has
+    been configured.
+    """
+    _get_library_root_logger().propagate = True
+
+
+_configure_library_root_logger()