Add option to ignore failing test step

Users can now use --ignore-test-failure instead of --skip-test-step
which will run the tests but does not abort on error and just prints it instead.

Using both options doesn't make sense and is hence disallowed and a
warning is show to make users move to ignoring failures instead of
skipping the test completely.

Author: Flamefire

easybuild/base/testing.py

@@ -35,6 +35,7 @@
 import pprint
 import re
 import sys
+from contextlib import contextmanager
 
 try:
     from cStringIO import StringIO  # Python 2
@@ -185,6 +186,26 @@ def get_stderr(self):
         """Return output captured from stderr until now."""
         return sys.stderr.getvalue()
 
+    @contextmanager
+    def mocked_stdout_stderr(self, mock_stdout=True, mock_stderr=True):
+        """Context manager to mock stdout and stderr"""
+        if mock_stdout:
+            self.mock_stdout(True)
+        if mock_stderr:
+            self.mock_stderr(True)
+        try:
+            if mock_stdout and mock_stderr:
+                yield sys.stdout, sys.stderr
+            elif mock_stdout:
+                yield sys.stdout
+            else:
+                yield sys.stderr
+        finally:
+            if mock_stdout:
+                self.mock_stdout(False)
+            if mock_stderr:
+                self.mock_stderr(False)
+
     def tearDown(self):
         """Cleanup after running a test."""
         self.mock_stdout(False)

easybuild/framework/easyblock.py

@@ -258,6 +258,8 @@ def __init__(self, ec):
         if group_name is not None:
             self.group = use_group(group_name)
 
+        self.ignore_test_failure = build_option('ignore_test_failure')
+
         # generate build/install directories
         self.gen_builddir()
         self.gen_installdir()
@@ -1814,6 +1816,16 @@ def remove_module_file(self):
             self.log.info("Removing existing module file %s", self.mod_filepath)
             remove_file(self.mod_filepath)
 
+    def report_test_failure(self, msgOrError):
+        """Report a failing test either via an exception or warning depending on ignore-test-failure
+
+        msgOrError - Failure description. Either a string or an EasyBuildError"""
+        if self.ignore_test_failure:
+            print_warning("Test failure ignored: " + str(msgOrError), log=self.log)
+        else:
+            exception = msgOrError if isinstance(msgOrError, EasyBuildError) else EasyBuildError(msgOrError)
+            raise exception
+
     #
     # STEP FUNCTIONS
     #
@@ -2229,6 +2241,13 @@ def test_step(self):
 
             return out
 
+    def _test_step(self):
+        """Run the test_step and handles failures"""
+        try:
+            self.test_step()
+        except EasyBuildError as e:
+            self.report_test_failure(e)
+
     def stage_install_step(self):
         """
         Install in a stage directory before actual installation.
@@ -3439,7 +3458,7 @@ def install_step_spec(initial):
         prepare_step_spec = (PREPARE_STEP, 'preparing', [lambda x: x.prepare_step], False)
         configure_step_spec = (CONFIGURE_STEP, 'configuring', [lambda x: x.configure_step], True)
         build_step_spec = (BUILD_STEP, 'building', [lambda x: x.build_step], True)
-        test_step_spec = (TEST_STEP, 'testing', [lambda x: x.test_step], True)
+        test_step_spec = (TEST_STEP, 'testing', [lambda x: x._test_step], True)
         extensions_step_spec = (EXTENSIONS_STEP, 'taking care of extensions', [lambda x: x.extensions_step], False)
 
         # part 1: pre-iteration + first iteration

easybuild/main.py

@@ -55,6 +55,7 @@
 from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for
 from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available
 from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak
+from easybuild.tools.build_log import print_warning
 from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option
 from easybuild.tools.containers.common import containerize
 from easybuild.tools.docs import list_software
@@ -304,6 +305,14 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     init_session_state.update({'module_list': modlist})
     _log.debug("Initial session state: %s" % init_session_state)
 
+    if options.skip_test_step:
+        if options.ignore_test_failure:
+            raise EasyBuildError("Found both ignore-test-failure and skip-test-step being set. "
+                                 "Please use only one of them.")
+        else:
+            print_warning("Will not run the test step as requested via skip-test-step. "
+                          "Consider using ignore-test-failure instead and verify the results afterwards")
+
     # determine easybuild-easyconfigs package install path
     easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
     if not easyconfigs_pkg_paths:

easybuild/tools/config.py

@@ -248,6 +248,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'ignore_checksums',
         'ignore_index',
         'ignore_locks',
+        'ignore_test_failure',
         'install_latest_eb_release',
         'logtostdout',
         'minimal_toolchains',

easybuild/tools/options.py

@@ -418,6 +418,7 @@ def override_options(self):
                                           "\"client[A-z0-9]*.example.com': ['Authorization: Basic token']\".",
                                           None, 'append', None, {'metavar': '[URLPAT::][HEADER:]FILE|FIELD'}),
             'ignore-checksums': ("Ignore failing checksum verification", None, 'store_true', False),
+            'ignore-test-failure': ("Ignore a failing test step", None, 'store_true', False),
             'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False),
             'install-latest-eb-release': ("Install latest known version of easybuild", None, 'store_true', False),
             'lib-lib64-symlink': ("Automatically create symlinks for lib/ pointing to lib64/ if the former is missing",

test/framework/options.py

@@ -394,6 +394,28 @@ def test_skip_test_step(self):
         found = re.search(test_run_msg, outtxt)
         self.assertFalse(found, "Test execution command is NOT present, outtxt: %s" % outtxt)
 
+    def test_ignore_test_failure(self):
+        """Test ignore failing tests (--ignore-test-failure)."""
+
+        topdir = os.path.abspath(os.path.dirname(__file__))
+        toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-test.eb')
+
+        args = [toy_ec, '--ignore-test-failure', '--force']
+
+        with self.mocked_stdout_stderr() as (_, stderr):
+            outtxt = self.eb_main(args, do_build=True)
+
+        msg = 'Test failure ignored'
+        self.assertTrue(re.search(msg, outtxt),
+                        "Ignored test failure message in log not found, outtxt: %s" % outtxt)
+        self.assertTrue(re.search(msg, stderr.getvalue()),
+                        "Ignored test failure message in stderr not found, stderr: %s" % stderr.getvalue())
+
+        # Passing skip and ignore options is disallowed
+        args.append('--skip-test-step')
+        error_pattern = 'Found both ignore-test-failure and skip-test-step being set'
+        self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
+
     def test_job(self):
         """Test submitting build as a job."""