Merge pull request #2839 from vkarak/feat/stress-testing

[feat] Add support for repeated testing through two new options: `--reruns` and `--duration`

Author: vkarak

docs/manpage.rst

@@ -459,6 +459,20 @@ Options controlling ReFrame execution
    .. versionadded:: 3.11.0
 
 
+.. option:: --duration=TIMEOUT
+
+   Run the test session repeatedly until the specified timeout expires.
+
+   ``TIMEOUT`` can be specified in one of the following forms:
+
+   - ``<int>`` or ``<float>``: number of seconds
+   - ``<days>d<hours>h<minutes>m<seconds>s``: a string denoting days, hours, minutes and/or seconds.
+
+   At the end, failures from every run will be reported and, similarly, the failure statistics printed by the :option:`--failure-stats` option will include all runs.
+
+   .. versionadded:: 4.2
+
+
 .. option:: --exec-order=ORDER
 
    Impose an execution order for the independent tests.
@@ -531,6 +545,24 @@ Options controlling ReFrame execution
 
    .. versionadded:: 3.12.0
 
+
+.. option:: --reruns=N
+
+   Rerun the whole test session ``N`` times.
+
+   In total, the selected tests will run ``N+1`` times as the first time does not count as a rerun.
+
+   At the end, failures from every run will be reported and, similarly, the failure statistics printed by the :option:`--failure-stats` option will include all runs.
+
+   Although similar to :option:`--repeat`, this option behaves differently.
+   This option repeats the *whole* test session multiple times.
+   All the tests of the session will finish before a new run is started.
+   The :option:`--repeat` option on the other hand generates clones of the selected tests and schedules them for running in a single session.
+   As a result, all the test clones will run (by default) concurrently.
+
+   .. versionadded:: 4.2
+
+
 .. option:: --restore-session [REPORT1[,REPORT2,...]]
 
    Restore a testing session that has run previously.

reframe/core/exceptions.py

@@ -101,6 +101,10 @@ class FailureLimitError(ReframeError):
     '''Raised when the limit of test failures has been reached.'''
 
 
+class RunSessionTimeout(ReframeError):
+    '''Raised when the maximum duration for a test session expires.'''
+
+
 class AbortTaskError(ReframeError):
     '''Raised by the runtime inside a regression task to denote that it has
     been aborted due to an external reason (e.g., keyboard interrupt, fatal
@@ -355,6 +359,12 @@ def is_severe(exc_type, exc_value, tb):
     return not is_user_error(exc_type, exc_value, tb)
 
 
+def is_warning(exc_type, exc_value, tb):
+    '''Check whether this exception can be treated as warning'''
+
+    return isinstance(exc_value, (RunSessionTimeout, KeyboardInterrupt))
+
+
 def what(exc_type, exc_value, tb):
     '''A short description of the error.'''
 

reframe/core/schedulers/local.py

@@ -98,7 +98,7 @@ def _kill_all(self, job):
         except (ProcessLookupError, PermissionError):
             # The process group may already be dead or assigned to a different
             # group, so ignore this error
-            self.log(f'pid {job.jobid} already dead or assigned elsewhere')
+            self.log(f'pid {job.jobid} already dead')
         finally:
             # Close file handles
             job.f_stdout.close()
@@ -107,8 +107,15 @@ def _kill_all(self, job):
 
     def _term_all(self, job):
         '''Send SIGTERM to all the processes of the spawned job.'''
-        os.killpg(job.jobid, signal.SIGTERM)
-        job._signal = signal.SIGTERM
+        try:
+            os.killpg(job.jobid, signal.SIGTERM)
+            job._signal = signal.SIGTERM
+        except (ProcessLookupError, PermissionError):
+            # Job has finished already, close file handles
+            self.log(f'pid {job.jobid} already dead')
+            job.f_stdout.close()
+            job.f_stderr.close()
+            job._state = 'FAILURE'
 
     def cancel(self, job):
         '''Cancel job.

reframe/frontend/cli.py

@@ -31,7 +31,6 @@
 import reframe.utility.osext as osext
 import reframe.utility.typecheck as typ
 
-
 from reframe.frontend.testgenerators import (distribute_tests,
                                              getallnodes, repeat_tests)
 from reframe.frontend.executors.policies import (SerialExecutionPolicy,
@@ -403,6 +402,10 @@ def main():
         help=('Distribute the selected single-node jobs on every node that'
               'is in STATE (default: "idle"')
     )
+    run_options.add_argument(
+        '--duration', action='store', metavar='TIMEOUT',
+        help='Run the test session repeatedly for the specified duration'
+    )
     run_options.add_argument(
         '--exec-order', metavar='ORDER', action='store',
         choices=['name', 'random', 'rname', 'ruid', 'uid'],
@@ -439,6 +442,10 @@ def main():
         '--repeat', action='store', metavar='N',
         help='Repeat selected tests N times'
     )
+    run_options.add_argument(
+        '--reruns', action='store', metavar='N', default=0,
+        help='Rerun the whole test session N times', type=int
+    )
     run_options.add_argument(
         '--restore-session', action='store', nargs='?', const='',
         metavar='REPORT',
@@ -1300,11 +1307,21 @@ def module_unuse(*paths):
         if options.maxfail < 0:
             raise errors.CommandLineError(
                 f'--maxfail should be a non-negative integer: '
-                f'{options.maxfail!r}'
+                f'{options.maxfail}'
+            )
+
+        if options.reruns and options.duration:
+            raise errors.CommandLineError(
+                f"'--reruns' option cannot be combined with '--duration'"
+            )
+
+        if options.reruns < 0:
+            raise errors.CommandLineError(
+                f"'--reruns' should be a non-negative integer: {options.reruns}"
             )
 
         runner = Runner(exec_policy, printer, options.max_retries,
-                        options.maxfail)
+                        options.maxfail, options.reruns, options.duration)
         try:
             time_start = time.time()
             session_info['time_start'] = time.strftime(
@@ -1319,18 +1336,21 @@ def module_unuse(*paths):
             session_info['time_elapsed'] = time_end - time_start
 
             # Print a retry report if we did any retries
-            if runner.stats.failed(run=0):
+            if options.max_retries and runner.stats.failed(run=0):
                 printer.info(runner.stats.retry_report())
 
             # Print a failure report if we had failures in the last run
             success = True
             if runner.stats.failed():
                 success = False
                 runner.stats.print_failure_report(
-                    printer, not options.distribute
+                    printer, not options.distribute,
+                    options.duration or options.reruns
                 )
                 if options.failure_stats:
-                    runner.stats.print_failure_stats(printer)
+                    runner.stats.print_failure_stats(
+                        printer, options.duration or options.reruns
+                    )
 
             if options.performance_report:
                 printer.info(runner.stats.performance_report())
@@ -1412,7 +1432,12 @@ def module_unuse(*paths):
     except (Exception, KeyboardInterrupt, errors.ReframeFatalError):
         exc_info = sys.exc_info()
         tb = ''.join(traceback.format_exception(*exc_info))
-        printer.error(f'run session stopped: {errors.what(*exc_info)}')
+        message = f'run session stopped: {errors.what(*exc_info)}'
+        if errors.is_warning(*exc_info):
+            printer.warning(message)
+        else:
+            printer.error(message)
+
         if errors.is_exit_request(*exc_info):
             # Print stack traces for exit requests only when TOO verbose
             printer.debug2(tb)