Merge pull request #4881 from boegel/fix_ignore_test_fail

also ignore errors raised during test step when `--ignore-test-failure` is used

Author: lexming

easybuild/framework/easyblock.py

@@ -3000,6 +3000,8 @@ def _test_step(self):
         """Run the test_step and handles failures"""
         try:
             self.test_step()
+        except EasyBuildError as err:
+            self.report_test_failure(f"An error was raised during test step: {err}")
         except RunShellCmdError as err:
             err.print()
             ec_path = os.path.basename(self.cfg.path)

test/framework/sandbox/easybuild/easyblocks/t/toy.py

@@ -132,6 +132,13 @@ def build_step(self, name=None, cfg=None):
         if res.exit_code:
             print_warning("Command '%s' failed, but we'll ignore it..." % cmd)
 
+    def test_step(self, *args, **kwargs):
+        """Test toy."""
+        if self.cfg['runtest'] == 'RAISE_ERROR':
+            raise EasyBuildError("TOY_TEST_FAIL")
+        else:
+            super().test_step(*args, **kwargs)
+
     def install_step(self, name=None):
         """Install toy."""
         if name is None:

test/framework/toy_build.py

@@ -4361,6 +4361,9 @@ def test_toy_failing_test_step(self):
         """
         test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs')
         toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
+        toy_mod_path = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
+        if get_module_syntax() == 'Lua':
+            toy_mod_path += '.lua'
 
         test_ec_txt = read_file(toy_ec)
         test_ec_txt += '\nruntest = "false"'
@@ -4370,6 +4373,27 @@ def test_toy_failing_test_step(self):
         error_pattern = r"shell command 'false \.\.\.' failed in test step"
         self.assertErrorRegex(EasyBuildError, error_pattern, self.run_test_toy_build_with_output,
                               ec_file=test_ec, raise_error=True)
+        self.assertNotExists(toy_mod_path)
+
+        # make sure that option to ignore test failures works
+        self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=['--ignore-test-failure'],
+                                            raise_error=True, verbose=True)
+        self.assertExists(toy_mod_path)
+        remove_file(toy_mod_path)
+
+        # ignoring test failure should also work if an EasyBuildError is raises from test step
+        test_ec_txt = read_file(toy_ec)
+        test_ec_txt += '\nruntest = "RAISE_ERROR"'
+        write_file(test_ec, test_ec_txt)
+
+        error_pattern = r"An error was raised during test step: 'TOY_TEST_FAIL'"
+        self.assertErrorRegex(EasyBuildError, error_pattern, self.run_test_toy_build_with_output,
+                              ec_file=test_ec, raise_error=True)
+
+        # make sure that option to ignore test failures works
+        self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=['--ignore-test-failure'],
+                                            raise_error=True, verbose=True)
+        self.assertExists(toy_mod_path)
 
     def test_eb_crash(self):
         """