Raising warnings for scandir instead of erroring

Author: cdgriffith

.github/workflows/black.yaml

@@ -9,6 +9,8 @@ on:
 jobs:
   test:
 
+    runs-on: ubuntu-latest
+
     strategy:
       matrix:
         python-version: [3.5, 3.6, 3.7, 3.8]

.github/workflows/tests.yaml

@@ -26,7 +26,7 @@
 
 
 def cmd(command, ignore_stderr=False, raise_on_return=False, timeout=None, encoding="utf-8"):
-    """ Run a shell command and have it automatically decoded and printed
+    """Run a shell command and have it automatically decoded and printed
 
     :param command: Command to run as str
     :param ignore_stderr: To not print stderr
@@ -99,7 +99,7 @@ def ls(params="", directory=".", printed=True):
 
 
 def find(name=None, ext=None, directory=".", match_case=False, disable_glob=False, depth=None):
-    """ Designed for the interactive interpreter by making default order
+    """Designed for the interactive interpreter by making default order
     of find_files faster.
 
     :param name: Part of the file name

reusables/cli.py

@@ -15,6 +15,7 @@
 import shutil
 from collections import defaultdict
 from pathlib import Path
+import warnings
 
 try:
     import ConfigParser as ConfigParser
@@ -53,6 +54,17 @@
 
 logger = logging.getLogger("reusables")
 
+scandir_warning_given = False
+
+
+def scandir_warning():
+    global scandir_warning_given
+    scandir_warning_given = True
+    warnings.warn(
+        '"enable_scandir" option will be removed in the next release of Reusables.'
+        ' Please remove all references to "enable_scandir" or version lock to Reusables < 1.0.0'
+    )
+
 
 def extract(archive_file, path=".", delete_on_success=False, enable_rar=False):
     """
@@ -124,7 +136,7 @@ def archive(
     allow_zip_64=True,
     **tarfile_kwargs,
 ):
-    """ Archive a list of files (or files inside a folder), can chose between
+    """Archive a list of files (or files inside a folder), can chose between
 
         - zip
         - tar
@@ -166,8 +178,7 @@ def archive(
             raise ValueError(err_msg)
         logger.debug("{0} file detected for {1}".format(archive_type, name))
     elif archive_type not in ("tar", "gz", "bz2", "zip", "lzma"):
-        err_msg = ("archive_type must be zip, gz, bz2,"
-                   " or gz, was {0}".format(archive_type))
+        err_msg = "archive_type must be zip, gz, bz2, lzma, or gz, was {0}".format(archive_type)
         logger.error(err_msg)
         raise ValueError(err_msg)
 
@@ -183,16 +194,14 @@ def archive(
         elif store:
             compression = zipfile.ZIP_STORED
 
-        arch = zipfile.ZipFile(name, 'w',
-                               compression,
-                               allowZip64=allow_zip_64)
+        arch = zipfile.ZipFile(name, "w", compression, allowZip64=allow_zip_64)
         write = arch.write
     elif archive_type in ("tar", "gz", "bz2"):
         mode = archive_type if archive_type != "tar" else ""
         arch = tarfile.open(name, "w:{0}".format(mode), **tarfile_kwargs)
         write = arch.add
     else:
-        raise ValueError("archive_type must be zip, gz, bz2, or gz")
+        raise ValueError("archive_type must be zip, gz, bz2, lzma, or gz")
 
     try:
         for file_path in files_to_archive:
@@ -396,7 +405,7 @@ def config_namespace(config_file=None, auto_find=False, verify=True, **cfg_optio
     return ConfigNamespace(**config_dict(config_file, auto_find, verify, **cfg_options))
 
 
-def os_tree(directory):
+def os_tree(directory, enable_scandir=False):
     """
     Return a directories contents as a dictionary hierarchy.
 
@@ -413,6 +422,9 @@ def os_tree(directory):
     :param directory: path to directory to created the tree of.
     :return: dictionary of the directory
     """
+    if enable_scandir and not scandir_warning_given:
+        scandir_warning()
+
     if not os.path.exists(directory):
         raise OSError("Directory does not exist")
     if not os.path.isdir(directory):
@@ -484,6 +496,7 @@ def find_files(
     disable_glob=False,
     depth=None,
     abspath=False,
+    enable_scandir=False,
     disable_pathlib=False,
 ):
     """
@@ -525,6 +538,8 @@ def find_files(
     :param disable_pathlib: only return string, not path objects
     :return: generator of all files in the specified directory
     """
+    if enable_scandir and not scandir_warning_given:
+        scandir_warning()
 
     def pathed(path):
         if disable_pathlib:
@@ -571,7 +586,7 @@ def pathed(path):
             yield pathed(os.path.join(root, file_name))
 
 
-def remove_empty_directories(root_directory, dry_run=False, ignore_errors=True):
+def remove_empty_directories(root_directory, dry_run=False, ignore_errors=True, enable_scandir=False):
     """
     Remove all empty folders from a path. Returns list of empty directories.
 
@@ -580,6 +595,8 @@ def remove_empty_directories(root_directory, dry_run=False, ignore_errors=True):
     :param ignore_errors: Permissions are a pain, just ignore if you blocked
     :return: list of removed directories
     """
+    if enable_scandir and not scandir_warning_given:
+        scandir_warning()
 
     directory_list = []
     for root, directories, files in os.walk(root_directory, topdown=False):
@@ -609,7 +626,7 @@ def remove_empty_directories(root_directory, dry_run=False, ignore_errors=True):
     return directory_list
 
 
-def remove_empty_files(root_directory, dry_run=False, ignore_errors=True):
+def remove_empty_files(root_directory, dry_run=False, ignore_errors=True, enable_scandir=False):
     """
     Remove all empty files from a path. Returns list of the empty files removed.
 
@@ -618,6 +635,9 @@ def remove_empty_files(root_directory, dry_run=False, ignore_errors=True):
     :param ignore_errors: Permissions are a pain, just ignore if you blocked
     :return: list of removed files
     """
+    if enable_scandir and not scandir_warning_given:
+        scandir_warning()
+
     file_list = []
     for root, directories, files in os.walk(root_directory):
         for file_name in files:
@@ -641,7 +661,7 @@ def remove_empty_files(root_directory, dry_run=False, ignore_errors=True):
     return file_list
 
 
-def dup_finder(file_path, directory="."):
+def dup_finder(file_path, directory=".", enable_scandir=False):
     """
     Check a directory for duplicates of the specified file. This is meant
     for a single file only, for checking a directory for dups, use
@@ -668,6 +688,9 @@ def dup_finder(file_path, directory="."):
     :param directory: Directory to dig recursively into to look for duplicates
     :return: generators
     """
+    if enable_scandir and not scandir_warning_given:
+        scandir_warning()
+
     size = os.path.getsize(file_path)
     if size == 0:
         for empty_file in remove_empty_files(directory, dry_run=True):

reusables/file_operations.py

@@ -177,7 +177,7 @@ def add_rotating_file_handler(
     backup_count=5,
     **handler_kwargs,
 ):
-    """ Adds a rotating file handler to the specified logger.
+    """Adds a rotating file handler to the specified logger.
 
     :param logger: logging name or object to modify, defaults to root logger
     :param file_path: path to file to log to
@@ -213,7 +213,7 @@ def add_timed_rotating_file_handler(
     backup_count=5,
     **handler_kwargs,
 ):
-    """ Adds a timed rotating file handler to the specified logger.
+    """Adds a timed rotating file handler to the specified logger.
     Defaults to weekly rotation, with 5 backups.
 
     :param logger: logging name or object to modify, defaults to root logger

reusables/log.py

@@ -176,7 +176,7 @@ def __getattr__(self, item):
             return super(ConfigNamespace, self).__getattr__(item.lower())
 
     def bool(self, item, default=None):
-        """ Return value of key as a boolean
+        """Return value of key as a boolean
 
         :param item: key of value to transform
         :param default: value to return if item does not exist
@@ -198,7 +198,7 @@ def bool(self, item, default=None):
         return True if item else False
 
     def int(self, item, default=None):
-        """ Return value of key as an int
+        """Return value of key as an int
 
         :param item: key of value to transform
         :param default: value to return if item does not exist
@@ -213,7 +213,7 @@ def int(self, item, default=None):
         return int(item)
 
     def float(self, item, default=None):
-        """ Return value of key as a float
+        """Return value of key as a float
 
         :param item: key of value to transform
         :param default: value to return if item does not exist
@@ -228,7 +228,7 @@ def float(self, item, default=None):
         return float(item)
 
     def list(self, item, default=None, spliter=",", strip=True, mod=None):
-        """ Return value of key as a list
+        """Return value of key as a list
 
         :param item: key of value to transform
         :param mod: function to map against list

reusables/namespace.py

@@ -94,7 +94,7 @@ def check_returncode(self):
 
 
 def run_in_pool(target, iterable, threaded=True, processes=4, asynchronous=False, target_kwargs=None):
-    """ Run a set of iterables to a function in a Threaded or MP Pool.
+    """Run a set of iterables to a function in a Threaded or MP Pool.
 
     .. code: python
 

reusables/process_helpers.py

@@ -26,40 +26,40 @@ def sanitized_input(
     message="", cast_as=None, number_of_retries=-1, error_message="", valid_input=(), raise_on_invalid=False
 ):
     """
-        Clean up and cast user input.
-
-        :param message: string to show the user (default: "")
-        :param cast_as: an object to cast the string into. Object must have a __new__
-                       method that can take a string as the first positional argument
-                       and be a subclass of type.
-                       The object should raise a ValueError exception if a
-                       string can't be cast into that object.
-                       cast_as can also be a tuple or a list, which will
-                       chain casts until the end of the list. Casts are chained in
-                       reverse order of the list (to mimic the syntax int(float(x))) (default: str)
-        :param number_of_retries: number of retries. No limit if n_retries == -1 (default: -1)
-        :param error_message: message to show the user before asking the input again in
-                            case an error occurs (default: repr of the exception).
-                            Can include {error}.
-        :param valid_input: an iterable to check if the result is allowed.
-        :param raise_on_invalid: boolean, whether this function will raise a
-                               reusables.InvalidInputError if the input doesn't match
-                               the valid_input argument.
-        :return: string literal casted into the cast_as as per that object's rules.
-
-        :raises: RetryCountExceededError if the retry count has exceeded the n_retries limit.
-
-
-        Examples:
-            integer = sanitized_input("How many apples?", int,
-                      error_msg="Please enter a valid number")
-            # returns an int, will prompt until the user enters an integer.
-
-            validated = sanitized_input(">>>", valid_input=["string"], raise_on_invalid=True)
-            # returns the value "string", and will raise InvalidInputError otherwise.
-
-            chain_cast = sanitized_input(">>>", cast_as=[int, float])
-            # returns an int, prompts like '2.3' won't raise a ValueError Exception.
+    Clean up and cast user input.
+
+    :param message: string to show the user (default: "")
+    :param cast_as: an object to cast the string into. Object must have a __new__
+                   method that can take a string as the first positional argument
+                   and be a subclass of type.
+                   The object should raise a ValueError exception if a
+                   string can't be cast into that object.
+                   cast_as can also be a tuple or a list, which will
+                   chain casts until the end of the list. Casts are chained in
+                   reverse order of the list (to mimic the syntax int(float(x))) (default: str)
+    :param number_of_retries: number of retries. No limit if n_retries == -1 (default: -1)
+    :param error_message: message to show the user before asking the input again in
+                        case an error occurs (default: repr of the exception).
+                        Can include {error}.
+    :param valid_input: an iterable to check if the result is allowed.
+    :param raise_on_invalid: boolean, whether this function will raise a
+                           reusables.InvalidInputError if the input doesn't match
+                           the valid_input argument.
+    :return: string literal casted into the cast_as as per that object's rules.
+
+    :raises: RetryCountExceededError if the retry count has exceeded the n_retries limit.
+
+
+    Examples:
+        integer = sanitized_input("How many apples?", int,
+                  error_msg="Please enter a valid number")
+        # returns an int, will prompt until the user enters an integer.
+
+        validated = sanitized_input(">>>", valid_input=["string"], raise_on_invalid=True)
+        # returns the value "string", and will raise InvalidInputError otherwise.
+
+        chain_cast = sanitized_input(">>>", cast_as=[int, float])
+        # returns an int, prompts like '2.3' won't raise a ValueError Exception.
     """
     retry_count = 0
 

reusables/sanitizers.py

@@ -239,7 +239,7 @@ def hook_post_task(self):
     def main_loop(self, stop_at_empty=False):
         """Blocking function that can be run directly, if so would probably
         want to specify 'stop_at_empty' to true, or have a separate process
-        adding items to the queue. """
+        adding items to the queue."""
         try:
             while True:
                 self.hook_pre_command()

reusables/tasker.py

@@ -35,7 +35,7 @@
     include_package_data=True,
     platforms="any",
     setup_requires=["pytest-runner"],
-    python_requires='>3.6',
+    python_requires=">3.6",
     classifiers=[
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
@@ -60,5 +60,7 @@
         "Topic :: System :: Filesystems",
         "Topic :: System :: Logging",
     ],
-    extras_require={"testing": ["pytest", "coverage >= 3.6", "argparse", "rarfile", "tox", "scandir", "pytest-cov"],},
+    extras_require={
+        "testing": ["pytest", "coverage >= 3.6", "argparse", "rarfile", "tox", "scandir", "pytest-cov"],
+    },
 )