Merge pull request #2539 from vkarak/feat/repeat-tests

[feat] Add new command line option to repeat selected tests

Author: Vasileios Karakasis

docs/manpage.rst

@@ -454,6 +454,13 @@ Options controlling ReFrame execution
    An execution mode is simply a predefined invocation of ReFrame that is set with the :js:attr:`modes` configuration parameter.
    If an option is specified both in an execution mode and in the command-line, then command-line takes precedence.
 
+.. option:: --repeat=N
+
+   Repeat the selected tests ``N`` times.
+   This option can be used in conjunction with the :option:`--distribute` option in which case the selected tests will be repeated multiple times and distributed on individual nodes of the system's partitions.
+
+   .. versionadded:: 3.12.0
+
 .. option:: --restore-session [REPORT1[,REPORT2,...]]
 
    Restore a testing session that has run previously.

reframe/core/exceptions.py

@@ -154,6 +154,10 @@ class ContainerError(ReframeError):
     '''Raised when a container platform is not configured properly.'''
 
 
+class CommandLineError(ReframeError):
+    '''Raised when an error in command-line arguments occurs.'''
+
+
 class BuildError(ReframeError):
     '''Raised when a build fails.'''
 

reframe/frontend/cli.py

@@ -30,7 +30,8 @@
 import reframe.utility.typecheck as typ
 
 
-from reframe.frontend.distribute import distribute_tests, getallnodes
+from reframe.frontend.testgenerators import (distribute_tests,
+                                             getallnodes, repeat_tests)
 from reframe.frontend.executors.policies import (SerialExecutionPolicy,
                                                  AsynchronousExecutionPolicy)
 from reframe.frontend.executors import Runner, generate_testcases
@@ -408,6 +409,10 @@ def main():
     run_options.add_argument(
         '--mode', action='store', help='Execution mode to use'
     )
+    run_options.add_argument(
+        '--repeat', action='store', metavar='N',
+        help='Repeat selected tests N times'
+    )
     run_options.add_argument(
         '--restore-session', action='store', nargs='?', const='',
         metavar='REPORT',
@@ -1035,6 +1040,20 @@ def _case_failed(t):
                 f'{len(testcases)} remaining'
             )
 
+        if options.repeat is not None:
+            try:
+                num_repeats = int(options.repeat)
+                if num_repeats <= 0:
+                    raise ValueError
+            except ValueError:
+                raise errors.CommandLineError(
+                    "argument to '--repeat' option must be "
+                    "a non-negative integer"
+                ) from None
+
+            testcases = repeat_tests(testcases, num_repeats)
+            testcases_all = testcases
+
         if options.distribute:
             node_map = getallnodes(options.distribute, parsed_job_options)
 
@@ -1227,7 +1246,7 @@ def module_unuse(*paths):
             errmsg = "invalid option for --flex-alloc-nodes: '{0}'"
             sched_flex_alloc_nodes = int(options.flex_alloc_nodes)
             if sched_flex_alloc_nodes <= 0:
-                raise errors.ConfigError(
+                raise errors.CommandLineError(
                     errmsg.format(options.flex_alloc_nodes)
                 )
         except ValueError:
@@ -1236,7 +1255,7 @@ def module_unuse(*paths):
         exec_policy.sched_flex_alloc_nodes = sched_flex_alloc_nodes
         exec_policy.sched_options = parsed_job_options
         if options.maxfail < 0:
-            raise errors.ConfigError(
+            raise errors.CommandLineError(
                 f'--maxfail should be a non-negative integer: '
                 f'{options.maxfail!r}'
             )

reframe/frontend/distribute.py renamed to reframe/frontend/testgenerators.py

@@ -77,7 +77,7 @@ def distribute_tests(testcases, node_map):
     their partitions based on the nodemap
     '''
     tmp_registry = TestRegistry()
-    new_checks = []
+
     # We don't want to register the same check for every environment
     # per partition
     check_part_combs = set()
@@ -112,9 +112,9 @@ def distribute_tests(testcases, node_map):
                 ),
             ]
         )
+
         # We have to set the prefix manually
         nc._rfm_custom_prefix = check.prefix
-
         for i in range(nc.num_variants):
             # Check if this variant should be instantiated
             vinfo = nc.get_variant_info(i, recurse=True)
@@ -124,3 +124,36 @@ def distribute_tests(testcases, node_map):
 
     new_checks = tmp_registry.instantiate_all()
     return generate_testcases(new_checks)
+
+
+def repeat_tests(testcases, num_repeats):
+    '''Returns new test cases parameterized over their repetition number'''
+
+    tmp_registry = TestRegistry()
+    unique_checks = set()
+    for tc in testcases:
+        check = tc.check
+        if check.is_fixture() or check in unique_checks:
+            continue
+
+        unique_checks.add(check)
+        cls = type(check)
+        variant_info = cls.get_variant_info(
+            check.variant_num, recurse=True
+        )
+        nc = make_test(
+            f'{cls.__name__}', (cls,),
+            {
+                '$repeat_no': builtins.parameter(range(num_repeats))
+            }
+        )
+        nc._rfm_custom_prefix = check.prefix
+        for i in range(nc.num_variants):
+            # Check if this variant should be instantiated
+            vinfo = nc.get_variant_info(i, recurse=True)
+            vinfo['params'].pop('$repeat_no')
+            if vinfo == variant_info:
+                tmp_registry.add(nc, variant_num=i)
+
+    new_checks = tmp_registry.instantiate_all()
+    return generate_testcases(new_checks)

unittests/test_cli.py

@@ -808,6 +808,42 @@ def test_maxfail_negative(run_reframe):
     assert returncode == 1
 
 
+def test_repeat_option(run_reframe):
+    returncode, stdout, stderr = run_reframe(
+        more_options=['--repeat', '2', '-n', 'HelloTest'],
+        checkpath=['unittests/resources/checks/hellocheck.py']
+    )
+    assert 'Traceback' not in stdout
+    assert 'Traceback' not in stderr
+    assert ('Ran 2/2 test case(s) from 2 check(s) '
+            '(0 failure(s), 0 skipped)') in stdout
+    assert returncode == 0
+
+
+def test_repeat_invalid_option(run_reframe):
+    returncode, stdout, stderr = run_reframe(
+        more_options=['--repeat', 'foo'],
+        checkpath=['unittests/resources/checks/hellocheck.py']
+    )
+    errmsg = "argument to '--repeat' option must be a non-negative integer"
+    assert 'Traceback' not in stdout
+    assert 'Traceback' not in stderr
+    assert errmsg in stdout
+    assert returncode == 1
+
+
+def test_repeat_negative(run_reframe):
+    returncode, stdout, stderr = run_reframe(
+        more_options=['--repeat', 'foo'],
+        checkpath=['unittests/resources/checks/hellocheck.py']
+    )
+    errmsg = "argument to '--repeat' option must be a non-negative integer"
+    assert 'Traceback' not in stdout
+    assert 'Traceback' not in stderr
+    assert errmsg in stdout
+    assert returncode == 1
+
+
 def test_detect_host_topology(run_reframe):
     from reframe.utility.cpuinfo import cpuinfo
 

unittests/test_distribute_tests.py renamed to unittests/test_testgenerators.py

@@ -7,23 +7,19 @@
 
 import reframe.frontend.executors as executors
 import reframe.frontend.filters as filters
-from reframe.frontend.distribute import distribute_tests
+from reframe.frontend.testgenerators import (distribute_tests, repeat_tests)
 from reframe.frontend.loader import RegressionCheckLoader
 
 
 @pytest.fixture
-def default_exec_ctx(make_exec_ctx_g):
+def sys0_exec_ctx(make_exec_ctx_g):
     yield from make_exec_ctx_g(system='sys0')
 
 
-@pytest.fixture
-def loader():
-    return RegressionCheckLoader([
+def test_distribute_testcases(sys0_exec_ctx):
+    loader = RegressionCheckLoader([
         'unittests/resources/checks_unlisted/distribute.py'
     ])
-
-
-def test_distribute_testcases(loader, default_exec_ctx):
     testcases = executors.generate_testcases(loader.load_all())
     testcases = filter(
         filters.have_any_name('Simple'), testcases
@@ -62,3 +58,14 @@ def sys0p0_nodes():
     # Make sure we have consumed all the elements from nodelist_iter
     with pytest.raises(StopIteration):
         next(nodelist_iter)
+
+
+def test_repeat_testcases():
+    loader = RegressionCheckLoader([
+        'unittests/resources/checks/hellocheck.py'
+    ])
+    testcases = executors.generate_testcases(loader.load_all())
+    assert len(testcases) == 2
+
+    testcases = repeat_tests(testcases, 10)
+    assert len(testcases) == 20