improved logging (#55)

* updated logging in core

* updated infrastructure

* updated changelog for version 0.0.5

* fixed un-declared variable in core

* updated maintainer email

* incremented queue delay from 2 to 5

---------

Co-authored-by: Daniel Heydebreck <heydebreck@dkry.de>

Author: neumannd

.gitignore

@@ -33,4 +33,3 @@ report.xml
 
 # IDEs
 .idea
-

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## v0.0.5
+- a few new debugging logging messages
+- improved way to set logging level for ``slkspec``
+
 ## v0.0.4
 - incremented dependency on ``pyslk`` to ``2.3.0`` => improved workflows + new required functions
 - internal ``list`` of files needed to be retrieved was changed to type ``set`` => remove duplicate files

pyproject.toml

@@ -8,7 +8,7 @@ dynamic = ["version"]
 description = "fsspec implementation for StrongLink tape archive"
 readme = "README.md"
 license = {text = "MIT License"}
-authors = [{name = "Hauke Schulz", email = "haschulz@uw.edu"}]
+authors = [{name = "Hauke Schulz", email = "has@dmi.dk"}]
 classifiers = [
   "Programming Language :: Python :: 3",
   "License :: OSI Approved :: MIT License",

slkspec/core.py

@@ -30,7 +30,11 @@
 from fsspec.spec import AbstractFileSystem
 
 logger = logging.getLogger("slkspec")
-logger.setLevel(logging.INFO)
+if logger.level == 0:
+    if logging.root.level == 0:
+        logger.setLevel(logging.INFO)
+    else:
+        logger.setLevel(logging.root.level)
 
 MAX_RETRIES = 2
 MAX_PARALLEL_RECALLS = 3
@@ -114,7 +118,7 @@ def __init__(
         touch: bool = False,
         file_permissions: int = 0o644,
         dir_permissions: int = 0o3775,
-        delay: int = 2,
+        delay: int = 5,
         _lock: threading.Lock = _retrieval_lock,
         _file_queue: Queue[Tuple[str, str]] = FileQueue,
         **kwargs: Any,
@@ -915,7 +919,7 @@ def _recall_split_files_not_needed(self, file_id: int) -> None:
     def _check_tapes_of_split_file_available(
         self, file_id: int, tmp_tapes: list[str]
     ) -> list[bool]:
-        tmp_tapes_available: List[bool]
+        tmp_tapes_available: List[bool] = list()
         msg: str
         for tape in tmp_tapes:
             # go through tape list and start new recalls
@@ -1230,6 +1234,14 @@ def _write_file_lists(
         and file_path not in slk_retrieval.files_retrieval_failed.keys()
     ]
     tmp_str: str
+    logger.debug(
+        "Start writing file lists if one of these values is larger than 0: "
+        + "#'files retrieval reasonable' is "
+        + f"{len(slk_retrieval.files_retrieval_reasonable)}; #'files recall "
+        + f"failed' is {len(slk_recall.files_recall_failed.keys())}; #'files "
+        + "retrieval failed' is "
+        + f"{len(slk_retrieval.files_retrieval_failed.keys())}"
+    )
     if (
         len(slk_retrieval.files_retrieval_reasonable) > 0
         or len(slk_recall.files_recall_failed.keys()) > 0
@@ -1246,12 +1258,20 @@ def _write_file_lists(
             + f"in directory '{str(slk_cache)}'."
         )
         if len(slk_recall.files_recall_failed) > 0:
+            logger.debug(
+                "Write list of files which recall failed into "
+                + f"{os.path.join(slk_cache, file_failed_recall)}"
+            )
             tmp_str = "\n  ".join(slk_recall.files_recall_failed)
             logger.error(f"files, recall failed:\n  {tmp_str}")
             with open(os.path.join(slk_cache, file_failed_recall), "w") as f:
                 for file_path, reason in slk_recall.files_recall_failed.items():
                     f.write(f"{file_path}: {reason}\n")
         if len(slk_retrieval.files_retrieval_failed) > 0:
+            logger.debug(
+                "Write list of files which retrieval failed into "
+                + f"{os.path.join(slk_cache, file_failed_retrieve)}"
+            )
             tmp_str = "\n  ".join(slk_retrieval.files_retrieval_failed)
             logger.error(f"files, retrieval failed (recall successful):\n  {tmp_str}")
             with open(os.path.join(slk_cache, file_failed_retrieve), "w") as f:
@@ -1261,6 +1281,10 @@ def _write_file_lists(
                 ) in slk_retrieval.files_retrieval_failed.items():
                     f.write(f"{file_path}: {reason}\n")
         if len(missing_files) > 0:
+            logger.debug(
+                "Write list of files which were not retrieved for other "
+                + f"reasons to {os.path.join(slk_cache, file_failed_other)}"
+            )
             tmp_str = "\n  ".join(missing_files)
             logger.error(f"files, missing for other reasons:\n  {tmp_str}")
             with open(os.path.join(slk_cache, file_failed_other), "w") as f:

versioneer.py

@@ -1,4 +1,5 @@
 # mypy: ignore-errors
+# flake8: noqa
 # Version: 0.21
 
 """The Versioneer - like a rocketeer, but for versions.