Merge pull request #3524 from Flamefire/skip_test_step

Add option to skip the unit tests (test_step)

Author: ocaisa

easybuild/framework/easyblock.py

@@ -2139,10 +2139,11 @@ def build_step(self):
 
     def test_step(self):
         """Run unit tests provided by software (if any)."""
-        if self.cfg['runtest']:
+        unit_test_cmd = self.cfg['runtest']
+        if unit_test_cmd:
 
-            self.log.debug("Trying to execute %s as a command for running unit tests...")
-            (out, _) = run_cmd(self.cfg['runtest'], log_all=True, simple=False)
+            self.log.debug("Trying to execute %s as a command for running unit tests...", unit_test_cmd)
+            (out, _) = run_cmd(unit_test_cmd, log_all=True, simple=False)
 
             return out
 
@@ -3310,6 +3311,10 @@ def build_and_install_one(ecdict, init_env):
         _log.debug("Skip set to %s" % skip)
         app.cfg['skip'] = skip
 
+    if build_option('skip_test_step'):
+        _log.debug('Adding test_step to skipped steps')
+        app.cfg.update('skipsteps', TEST_STEP, allow_duplicate=False)
+
     # build easyconfig
     errormsg = '(no error)'
     # timing info

easybuild/tools/config.py

@@ -253,6 +253,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'sequential',
         'set_gid_bit',
         'skip_test_cases',
+        'skip_test_step',
         'generate_devel_module',
         'sticky_bit',
         'trace',

easybuild/tools/options.py

@@ -439,6 +439,7 @@ def override_options(self):
             'silence-deprecation-warnings': ("Silence specified deprecation warnings", 'strlist', 'extend', None),
             'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False),
             'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'),
+            'skip-test-step': ("Skip running the test step (e.g. unit tests)", None, 'store_true', False),
             'generate-devel-module': ("Generate a develop module file, implies --force if disabled",
                                       None, 'store_true', True),
             'sysroot': ("Location root directory of system, prefix for standard paths like /usr/lib and /usr/include",

test/framework/docs.py

@@ -256,7 +256,7 @@ def test_list_software(self):
             "  * toy v0.0 (versionsuffix: '-deps'): system",
             "  * toy v0.0 (versionsuffix: '-iter'): system",
             "  * toy v0.0 (versionsuffix: '-multiple'): system",
-            "  * toy v0.0 (versionsuffix: '-test'): gompi/2018a",
+            "  * toy v0.0 (versionsuffix: '-test'): gompi/2018a, system",
         ]
         txt = list_software(output_format='txt', detailed=True)
         lines = txt.split('\n')
@@ -278,7 +278,7 @@ def test_list_software(self):
             '``0.0``    ``-deps``        ``system``                 ',
             '``0.0``    ``-iter``        ``system``                 ',
             '``0.0``    ``-multiple``    ``system``                 ',
-            '``0.0``    ``-test``        ``gompi/2018a``            ',
+            '``0.0``    ``-test``        ``gompi/2018a``, ``system``',
             '=======    =============    ===========================',
         ]
         txt = list_software(output_format='rst', detailed=True)

test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb

@@ -0,0 +1,35 @@
+name = 'toy'
+version = '0.0'
+versionsuffix = '-test'
+
+homepage = 'https://easybuilders.github.io/easybuild'
+description = "Toy C program, 100% toy."
+
+toolchain = SYSTEM
+
+sources = [SOURCE_TAR_GZ]
+checksums = [[
+    'be662daa971a640e40be5c804d9d7d10',  # default (MD5)
+    '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc',  # default (SHA256)
+    ('adler32', '0x998410035'),
+    ('crc32', '0x1553842328'),
+    ('md5', 'be662daa971a640e40be5c804d9d7d10'),
+    ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'),
+    ('size', 273),
+]]
+patches = [
+    'toy-0.0_fix-silly-typo-in-printf-statement.patch',
+    ('toy-extra.txt', 'toy-0.0'),
+]
+
+sanity_check_paths = {
+    'files': [('bin/yot', 'bin/toy')],
+    'dirs': ['bin'],
+}
+
+runtest = "make_test dummy_cmd" # Provide some value which is unique enough to be checked for
+
+postinstallcmds = ["echo TOY > %(installdir)s/README"]
+
+moduleclass = 'tools'
+# trailing comment, leave this here, it may trigger bugs with extract_comments()

test/framework/filetools.py

@@ -2013,7 +2013,7 @@ def test_index_functions(self):
         # test with specified path with and without trailing '/'s
         for path in [test_ecs, test_ecs + '/', test_ecs + '//']:
             index = ft.create_index(path)
-            self.assertEqual(len(index), 82)
+            self.assertEqual(len(index), 83)
 
             expected = [
                 os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'),

test/framework/options.py

@@ -340,6 +340,40 @@ def test_skip(self):
 
         self.assertEqual(len(glob.glob(toy_mod_glob)), 1)
 
+    def test_skip_test_step(self):
+        """Test skipping testing the build (--skip-test-step)."""
+
+        topdir = os.path.abspath(os.path.dirname(__file__))
+        toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-test.eb')
+
+        # check log message without --skip-test-step
+        args = [
+            toy_ec,
+            '--extended-dry-run',
+            '--force',
+            '--debug',
+        ]
+        self.mock_stdout(True)
+        outtxt = self.eb_main(args, do_build=True)
+        self.mock_stdout(False)
+        found_msg = "Running method test_step part of step test"
+        found = re.search(found_msg, outtxt)
+        test_run_msg = "execute make_test dummy_cmd as a command for running unit tests"
+        self.assertTrue(found, "Message about test step being run is present, outtxt: %s" % outtxt)
+        found = re.search(test_run_msg, outtxt)
+        self.assertTrue(found, "Test execution command is present, outtxt: %s" % outtxt)
+
+        # And now with the argument
+        args.append('--skip-test-step')
+        self.mock_stdout(True)
+        outtxt = self.eb_main(args, do_build=True)
+        self.mock_stdout(False)
+        found_msg = "Skipping test step"
+        found = re.search(found_msg, outtxt)
+        self.assertTrue(found, "Message about test step being skipped is present, outtxt: %s" % outtxt)
+        found = re.search(test_run_msg, outtxt)
+        self.assertFalse(found, "Test execution command is NOT present, outtxt: %s" % outtxt)
+
     def test_job(self):
         """Test submitting build as a job."""