Merge pull request #1176 from ZQyou/feature/summary-rerun-failures

[feat] Print failure statistics table as well as the run options for failures

Author: Vasileios Karakasis

reframe/frontend/cli.py

@@ -231,6 +231,9 @@ def main():
     misc_options.add_argument(
         '--nocolor', action='store_false', dest='colorize', default=True,
         help='Disable coloring of output')
+    misc_options.add_argument(
+        '--failure-stats', action='store_true',
+        help='Print failure statistics')
     misc_options.add_argument('--performance-report', action='store_true',
                               help='Print the performance report')
     misc_options.add_argument(
@@ -630,6 +633,8 @@ def main():
                 if runner.stats.failures():
                     printer.info(runner.stats.failure_report())
                     success = False
+                    if options.failure_stats:
+                        printer.info(runner.stats.failure_stats())
 
                 if options.performance_report:
                     printer.info(runner.stats.performance_report())

reframe/frontend/statistics.py

@@ -97,6 +97,8 @@ def failure_report(self):
             report.append('  * Job type: %s (id=%s)' % (job_type, jobid))
             report.append('  * Maintainers: %s' % check.maintainers)
             report.append('  * Failing phase: %s' % tf.failed_stage)
+            report.append("  * Rerun with '-n %s -p %s --system %s'" %
+                          (check.name, environ_name, partname))
             reason = '  * Reason: '
             if tf.exc_info is not None:
                 from reframe.core.exceptions import format_exception
@@ -115,6 +117,50 @@ def failure_report(self):
         report.append(line_width * '-')
         return '\n'.join(report)
 
+    def failure_stats(self):
+        failures = {}
+        current_run = rt.runtime().current_run
+        for tf in (t for t in self.tasks(current_run) if t.failed):
+            check = tf.check
+            partition = check.current_partition
+            partname = partition.fullname if partition else 'None'
+            environ_name = (check.current_environ.name
+                            if check.current_environ else 'None')
+            f = f'[{check.name}, {environ_name}, {partname}]'
+            if tf.failed_stage not in failures:
+                failures[tf.failed_stage] = []
+
+            failures[tf.failed_stage].append(f)
+
+        line_width = 78
+        stats_start = line_width * '='
+        stats_title = 'FAILURE STATISTICS'
+        stats_end = line_width * '-'
+        stats_body = []
+        row_format = "{:<11} {:<5} {}"
+        stats_hline = row_format.format(11*'-', 5*'-', 60*'-')
+        stats_header = row_format.format('Phase', '#', 'Failing test cases')
+        num_tests = len(self.tasks(current_run))
+        num_failures = 0
+        for l in failures.values():
+            num_failures += len(l)
+
+        stats_body = ['']
+        stats_body.append('Total number of test cases: %s' % num_tests)
+        stats_body.append('Total number of failures: %s' % num_failures)
+        stats_body.append('')
+        stats_body.append(stats_header)
+        stats_body.append(stats_hline)
+        for p, l in failures.items():
+            stats_body.append(row_format.format(p, len(l), l[0]))
+            for f in l[1:]:
+                stats_body.append(row_format.format('', '', str(f)))
+
+        if stats_body:
+            return '\n'.join([stats_start, stats_title, *stats_body,
+                              stats_end])
+        return ''
+
     def performance_report(self):
         line_width = 78
         report_start = line_width * '='

unittests/test_cli.py

@@ -228,6 +228,17 @@ def test_performance_check_failure(self):
                                   self.environs)
         assert self._perflog_exists('PerformanceFailureCheck')
 
+    def test_failure_stats(self):
+        self.checkpath = ['unittests/resources/checks/frontend_checks.py']
+        self.more_options = ['-t', 'SanityFailureCheck', '--failure-stats']
+        returncode, stdout, stderr = self._run_reframe()
+
+        assert r'FAILURE STATISTICS' in stdout
+        assert r'sanity      1     [SanityFailureCheck' in stdout
+        assert 'Traceback' not in stdout
+        assert 'Traceback' not in stderr
+        assert returncode != 0
+
     def test_performance_report(self):
         self.checkpath = ['unittests/resources/checks/frontend_checks.py']
         self.more_options = ['-t', 'PerformanceFailureCheck',