Fix issues with logging and blacklist files

Author: jyapayne

command_line.py

@@ -899,10 +899,13 @@ def make_output_dirs(self, write_json=True):
 
         output_blacklist = os.path.basename(self.output_dir())
 
+        blacklist_vals = re.split(r'\\n?,?\n?', blacklist_setting.value)
+        whitelist_vals = re.split(r'\\n?,?\n?', whitelist_setting.value)
+
         self.file_tree.init(self.project_dir(),
-                            blacklist=(blacklist_setting.value.split('\n') +
+                            blacklist=(blacklist_vals +
                                        ['*'+output_blacklist+'*']),
-                            whitelist=whitelist_setting.value.split('\n'))
+                            whitelist=whitelist_vals)
 
         self.copy_files_to_project_folder()
 
@@ -1537,7 +1540,7 @@ def get_arguments(command_base, args=None):
                               'to export to.'))
 
     if args:
-        return parse.parse_args(args)
+        return parser.parse_args(args)
 
     return parser.parse_args()
 
@@ -1601,19 +1604,12 @@ def setup_logging(args, command_base):
     if args.verbose:
         logging.basicConfig(
             stream=sys.stdout,
-            format=("%(levelname) -10s %(module)s.py: "
-                    "%(lineno)s %(funcName)s - %(message)s"),
-            level=logging.DEBUG
-        )
-    else:
-        # Log to the logfile in config.py
-        logging.basicConfig(
-            filename=config.LOG_FILENAME,
-            format=("%(levelname) -10s %(asctime)s %(module)s.py: "
-                    "%(lineno)s %(funcName)s - %(message)s"),
+            format=("%(levelname) -6s %(module)s.py %(lineno)s: "
+                    "%(message)s"),
             level=logging.DEBUG
         )
 
+
     config.logger = logging.getLogger('CMD Logger')
     config.handler = lh.RotatingFileHandler(config.LOG_FILENAME,
                                             maxBytes=100000,

config.py

@@ -92,14 +92,6 @@ def is_installed():
 
 LOG_FILENAME = utils.get_data_file_path(ERROR_LOG_FILE)
 
-if __name__ != '__main__':
-    logging.basicConfig(
-        filename=LOG_FILENAME,
-        format=("%(levelname) -10s %(asctime)s %(module)s.py: "
-                "%(lineno)s %(funcName)s - %(message)s"),
-        level=logging.DEBUG
-    )
-
 logger = logging.getLogger(__name__)
 
 ## Custom except hook to log all errors ----------------------

main.py

@@ -24,6 +24,7 @@
 import os
 import glob
 import sys
+import re
 import logging
 import platform
 
@@ -988,11 +989,13 @@ def load_project(self, directory, readonly=False):
         self.open_export_button.setEnabled(True)
 
         blacklist_setting = self.get_setting('blacklist')
+        whitelist_setting = self.get_setting('whitelist')
 
         output_blacklist = os.path.basename(self.output_dir())
 
         self.tree_browser.init(directory,
-                               blacklist=(blacklist_setting.value.split('\n') +
+                               whitelist=re.split('\n?,?', whitelist_setting.value),
+                               blacklist=(re.split('\n?,?', blacklist_setting.value) +
                                           ['*'+output_blacklist+'*']))
 
         self.update_json = True
@@ -1260,12 +1263,12 @@ def create_blacklist_layout(self, blacklist_layout):
     def blacklist_changed(self, text, blacklist_setting):
         new_val = text.toPlainText()
         output_blacklist = os.path.basename(self.output_dir())
-        self.tree_browser.refresh(blacklist=(new_val.split('\n') +
+        self.tree_browser.refresh(blacklist=(re.split('\n?,?', new_val) +
                                             ['*'+output_blacklist+'*']))
 
     def whitelist_changed(self, text, whitelist_setting):
         new_val = text.toPlainText()
-        self.tree_browser.refresh(whitelist=new_val.split('\n'))
+        self.tree_browser.refresh(whitelist=re.split('\n?,?', new_val))
 
     @property
     def used_project_files(self):

util_classes.py

@@ -6,6 +6,8 @@
 import zipfile
 import tarfile
 import time
+import logging
+from pprint import pformat
 
 import config
 import utils
@@ -40,6 +42,8 @@ def __init__(self, directory=None,
     def init(self, directory=None,
              whitelist=None, blacklist=None):
 
+        self.logger = logging.getLogger(__name__)
+
         if directory:
             self.directory = directory + os.sep
         else:
@@ -104,12 +108,32 @@ def add_to_cache(self, *args):
         if self.cache:
             self.walkcache[self.directory].append(args)
 
+    def is_in_skipped(self, skipped, path):
+        temp = path
+
+        while temp:
+            if temp in skipped:
+                return True
+            if temp == os.path.dirname(temp):
+                return False
+            temp = os.path.dirname(temp)
+        return False
+
     def generate_files(self):
         if self.directory is None:
             return
 
+        self.logger.debug('Blacklist pattern:')
+        self.logger.debug(pformat(self.blacklist))
+        self.logger.debug('')
+        self.logger.debug('Whitelist pattern:')
+        self.logger.debug(pformat(self.whitelist))
+        self.logger.debug('')
+
         self.init_cache()
 
+        skipped_files = set()
+
         for root, dirs, files in self.walk(self.directory):
             self.add_to_cache(root, dirs, files)
 
@@ -119,15 +143,27 @@ def generate_files(self):
                 path = os.path.join(proj_path, directory)
 
                 if self.determine_skip(path):
+                    if not self.is_in_skipped(skipped_files, path):
+                        self.logger.debug('Skipping dir: %s', path)
+                    skipped_files.add(path)
                     continue
+                else:
+                    if self.is_in_skipped(skipped_files, path):
+                        self.logger.debug('Keeping dir: %s', path)
 
                 self.dirs.append(path)
 
             for file in files:
                 path = os.path.join(proj_path, file)
 
                 if self.determine_skip(path):
+                    if not self.is_in_skipped(skipped_files, path):
+                        self.logger.debug('Skipping file: %s', path)
+                    skipped_files.add(path)
                     continue
+                else:
+                    if self.is_in_skipped(skipped_files, path):
+                        self.logger.debug('Keeping file: %s', path)
 
                 self.files.append(path)