Add failed to load error logging (#27)

Author: alfinkel

CHANGES.md

@@ -1,3 +1,7 @@
+# 1.2.4
+
+- [IMPROVED] Fetchers and checks that failed to load appear as errors in STDERR now.
+
 # 1.2.3
 
 - [IMPROVED] Github service `get_commit_details` now take `path` as an optional argument.

compliance/__init__.py

@@ -14,4 +14,4 @@
 # limitations under the License.
 """Compliance automation package."""
 
-__version__ = '1.2.3'
+__version__ = '1.2.4'

compliance/controls.py

@@ -18,6 +18,7 @@
 import itertools
 import json
 import os
+from collections import defaultdict
 
 
 class ControlDescriptor(object):
@@ -49,6 +50,18 @@ def as_dict(self):
         """Provide control descriptor content as a modifiable dictionary."""
         return copy.deepcopy(self._controls)
 
+    @property
+    def accred_checks(self):
+        """Provide all checks by accreditation (key) as a dictionary."""
+        if not hasattr(self, '_accred_checks'):
+            self._accred_checks = defaultdict(list)
+            for check, evidence in self._controls.items():
+                for control in evidence.values():
+                    for accreds in control.values():
+                        for accred in accreds:
+                            self._accred_checks[accred].append(check)
+        return self._accred_checks
+
     def get_accreditations(self, test_path):
         """
         Provide the accreditation list for a given test_path.

compliance/runners.py

@@ -16,6 +16,7 @@
 
 import inspect
 import os
+import re
 import sys
 import time
 import unittest
@@ -40,6 +41,7 @@ class _BaseRunner(object):
     def __init__(self, opts, extra_opts):
         self.opts = opts
         self.extra_opts = extra_opts
+        self.load_errors = set()
 
     def __enter__(self):
         self.init_config()
@@ -168,6 +170,15 @@ def get_fetchers(self):
             for candidate in candidates:
                 if issubclass(candidate.__class__, ComplianceFetcher):
                     fetchers.add(candidate.__class__)
+            for load_err in tl.errors:
+                try:
+                    locate = re.search(
+                        '^Failed to import test module: (.+?)\n.*?', load_err
+                    )
+                    if locate.group(1).split('.')[-1].startswith(FETCH_PREFIX):
+                        self.load_errors.add(load_err)
+                except AttributeError:
+                    pass
         return fetchers
 
     def run_fetchers(self, reruns=None):
@@ -243,6 +254,7 @@ def init_config(self):
     def get_checks(self):
         """Provide the appropriate compliance framework check classes."""
         checks = set()
+        tests_found = set()
         for loc in self.dirs:
             tl = unittest.TestLoader()
             tl.testMethodPrefix = CHECK_PREFIX
@@ -251,6 +263,7 @@ def get_checks(self):
             )
             for test in [c.__class__ for c in candidates]:
                 path = f'{test.__module__}.{test.__name__}'
+                tests_found.add(path)
                 in_accred_grouping = self.controls.is_test_included(
                     path, self.accreds
                 )
@@ -264,6 +277,32 @@ def get_checks(self):
                         )
                     ]
                     checks.add(test)
+            for load_err in tl.errors:
+                try:
+                    locate = re.search(
+                        '^Failed to import test module: (.+?)\n.*?', load_err
+                    )
+                    for accred in self.accreds:
+                        for check in self.controls.accred_checks[accred]:
+                            if check.startswith(locate.group(1)):
+                                self.load_errors.add(
+                                    f'Unable to load {check}\n\n{load_err}'
+                                )
+                                tests_found.add(check)
+                except AttributeError:
+                    pass
+        expected_checks = set()
+        for accred, checks_in_accred in self.controls.accred_checks.items():
+            if accred in self.accreds:
+                expected_checks.update(checks_in_accred)
+        for check_not_found in expected_checks - tests_found:
+            self.load_errors.add(
+                (
+                    f'Unable to load {check_not_found}\n\n'
+                    f'The check {check_not_found} was not found.  '
+                    'Please validate that the path provided is correct.'
+                )
+            )
         return checks
 
     def run_checks(self):

compliance/scripts/compliance_cli.py

@@ -157,11 +157,15 @@ def _run(self, args):
                         '\nUnable to resolve dependency issues with %s.\n',
                         ', '.join(reruns)
                     )
+                for fetch_load_error in fetch.load_errors:
+                    self.err(f'\nERROR: {fetch_load_error}\n')
         if args.check:
             with CheckMode(args, self.extra_args) as check:
                 accreds = ', '.join(check.accreds)
-                self.out(f'\nCheck Run - Accreditations: {accreds} \n')
+                self.out(f'\nCheck Run - Accreditations: {accreds}\n')
                 success = check.run_checks() and success
+                for check_load_error in check.load_errors:
+                    self.err(f'\nERROR: {check_load_error}\n')
         return 0 if success else 1