Merge pull request #3732 from Flamefire/ignore_test_failure

Add option to ignore failing test step

Author: boegel

easybuild/base/testing.py

@@ -35,6 +35,7 @@
 import pprint
 import re
 import sys
+from contextlib import contextmanager
 
 try:
     from cStringIO import StringIO  # Python 2
@@ -185,6 +186,26 @@ def get_stderr(self):
         """Return output captured from stderr until now."""
         return sys.stderr.getvalue()
 
+    @contextmanager
+    def mocked_stdout_stderr(self, mock_stdout=True, mock_stderr=True):
+        """Context manager to mock stdout and stderr"""
+        if mock_stdout:
+            self.mock_stdout(True)
+        if mock_stderr:
+            self.mock_stderr(True)
+        try:
+            if mock_stdout and mock_stderr:
+                yield sys.stdout, sys.stderr
+            elif mock_stdout:
+                yield sys.stdout
+            else:
+                yield sys.stderr
+        finally:
+            if mock_stdout:
+                self.mock_stdout(False)
+            if mock_stderr:
+                self.mock_stderr(False)
+
     def tearDown(self):
         """Cleanup after running a test."""
         self.mock_stdout(False)

easybuild/framework/easyblock.py

@@ -258,6 +258,8 @@ def __init__(self, ec):
         if group_name is not None:
             self.group = use_group(group_name)
 
+        self.ignore_test_failure = build_option('ignore_test_failure')
+
         # generate build/install directories
         self.gen_builddir()
         self.gen_installdir()
@@ -1814,6 +1816,18 @@ def remove_module_file(self):
             self.log.info("Removing existing module file %s", self.mod_filepath)
             remove_file(self.mod_filepath)
 
+    def report_test_failure(self, msg_or_error):
+        """
+        Report a failing test either via an exception or warning depending on ignore-test-failure
+
+        :param msg_or_error: failure description (string value or an EasyBuildError instance)
+        """
+        if self.ignore_test_failure:
+            print_warning("Test failure ignored: " + str(msg_or_error), log=self.log)
+        else:
+            exception = msg_or_error if isinstance(msg_or_error, EasyBuildError) else EasyBuildError(msg_or_error)
+            raise exception
+
     #
     # STEP FUNCTIONS
     #
@@ -2229,6 +2243,13 @@ def test_step(self):
 
             return out
 
+    def _test_step(self):
+        """Run the test_step and handles failures"""
+        try:
+            self.test_step()
+        except EasyBuildError as err:
+            self.report_test_failure(err)
+
     def stage_install_step(self):
         """
         Install in a stage directory before actual installation.
@@ -3367,10 +3388,12 @@ def run_step(self, step, step_methods):
         run_hook(step, self.hooks, pre_step_hook=True, args=[self])
 
         for step_method in step_methods:
-            self.log.info("Running method %s part of step %s" % (extract_method_name(step_method), step))
+            # Remove leading underscore from e.g. "_test_step"
+            method_name = extract_method_name(step_method).lstrip('_')
+            self.log.info("Running method %s part of step %s", method_name, step)
 
             if self.dry_run:
-                self.dry_run_msg("[%s method]", step_method(self).__name__)
+                self.dry_run_msg("[%s method]", method_name)
 
                 # if an known possible error occurs, just report it and continue
                 try:
@@ -3443,7 +3466,7 @@ def install_step_spec(initial):
         prepare_step_spec = (PREPARE_STEP, 'preparing', [lambda x: x.prepare_step], False)
         configure_step_spec = (CONFIGURE_STEP, 'configuring', [lambda x: x.configure_step], True)
         build_step_spec = (BUILD_STEP, 'building', [lambda x: x.build_step], True)
-        test_step_spec = (TEST_STEP, 'testing', [lambda x: x.test_step], True)
+        test_step_spec = (TEST_STEP, 'testing', [lambda x: x._test_step], True)
         extensions_step_spec = (EXTENSIONS_STEP, 'taking care of extensions', [lambda x: x.extensions_step], False)
 
         # part 1: pre-iteration + first iteration

easybuild/main.py

@@ -55,6 +55,7 @@
 from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for
 from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available
 from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak
+from easybuild.tools.build_log import print_warning
 from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option
 from easybuild.tools.containers.common import containerize
 from easybuild.tools.docs import list_software
@@ -304,6 +305,14 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     init_session_state.update({'module_list': modlist})
     _log.debug("Initial session state: %s" % init_session_state)
 
+    if options.skip_test_step:
+        if options.ignore_test_failure:
+            raise EasyBuildError("Found both ignore-test-failure and skip-test-step enabled. "
+                                 "Please use only one of them.")
+        else:
+            print_warning("Will not run the test step as requested via skip-test-step. "
+                          "Consider using ignore-test-failure instead and verify the results afterwards")
+
     # determine easybuild-easyconfigs package install path
     easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
     if not easyconfigs_pkg_paths:

easybuild/tools/config.py

@@ -248,6 +248,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'ignore_checksums',
         'ignore_index',
         'ignore_locks',
+        'ignore_test_failure',
         'install_latest_eb_release',
         'logtostdout',
         'minimal_toolchains',

easybuild/tools/options.py

@@ -418,6 +418,7 @@ def override_options(self):
                                           "\"client[A-z0-9]*.example.com': ['Authorization: Basic token']\".",
                                           None, 'append', None, {'metavar': '[URLPAT::][HEADER:]FILE|FIELD'}),
             'ignore-checksums': ("Ignore failing checksum verification", None, 'store_true', False),
+            'ignore-test-failure': ("Ignore a failing test step", None, 'store_true', False),
             'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False),
             'install-latest-eb-release': ("Install latest known version of easybuild", None, 'store_true', False),
             'lib-lib64-symlink': ("Automatically create symlinks for lib/ pointing to lib64/ if the former is missing",

test/framework/options.py

@@ -394,6 +394,29 @@ def test_skip_test_step(self):
         found = re.search(test_run_msg, outtxt)
         self.assertFalse(found, "Test execution command is NOT present, outtxt: %s" % outtxt)
 
+    def test_ignore_test_failure(self):
+        """Test ignore failing tests (--ignore-test-failure)."""
+
+        topdir = os.path.abspath(os.path.dirname(__file__))
+        # This EC uses a `runtest` command which does not exist and hence will make the test step fail
+        toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-test.eb')
+
+        args = [toy_ec, '--ignore-test-failure', '--force']
+
+        with self.mocked_stdout_stderr() as (_, stderr):
+            outtxt = self.eb_main(args, do_build=True)
+
+        msg = 'Test failure ignored'
+        self.assertTrue(re.search(msg, outtxt),
+                        "Ignored test failure message in log should be found, outtxt: %s" % outtxt)
+        self.assertTrue(re.search(msg, stderr.getvalue()),
+                        "Ignored test failure message in stderr should be found, stderr: %s" % stderr.getvalue())
+
+        # Passing skip and ignore options is disallowed
+        args.append('--skip-test-step')
+        error_pattern = 'Found both ignore-test-failure and skip-test-step enabled'
+        self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
+
     def test_job(self):
         """Test submitting build as a job."""