Fix and expand JSON report

Author: Vasileios Karakasis

reframe/core/pipeline.py

@@ -925,6 +925,10 @@ def stderr(self):
         '''
         return self._job.stderr
 
+    @property
+    def build_job(self):
+        return self._build_job
+
     @property
     @sn.sanity_function
     def build_stdout(self):

reframe/frontend/cli.py

@@ -3,15 +3,14 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-import datetime
 import inspect
 import json
 import os
 import re
 import socket
 import sys
+import time
 import traceback
-from pathlib import Path
 
 import reframe
 import reframe.core.config as config
@@ -32,7 +31,6 @@
                                                  AsynchronousExecutionPolicy)
 from reframe.frontend.loader import RegressionCheckLoader
 from reframe.frontend.printer import PrettyPrinter
-from reframe.utility import get_next_runreport_index
 
 
 def format_check(check, detailed):
@@ -72,6 +70,22 @@ def list_checks(checks, printer, detailed=False):
     printer.info('\nFound %d check(s).' % len(checks))
 
 
+def generate_report_filename(filepatt):
+    if not '{sessionid}' in filepatt:
+        return filepatt
+
+    search_patt = os.path.basename(filepatt).replace('{sessionid}', r'(\d+)')
+    new_id = -1
+    for filename in os.listdir(os.path.dirname(filepatt)):
+        match = re.match(search_patt, filename)
+        if match:
+            found_id = int(match.group(1))
+            new_id = max(found_id, new_id)
+
+    new_id += 1
+    return filepatt.format(sessionid=new_id)
+
+
 def main():
     # Setup command line options
     argparser = argparse.ArgumentParser()
@@ -140,8 +154,10 @@ def main():
         envvar='RFM_SAVE_LOG_FILES', configvar='general/save_log_files'
     )
     output_options.add_argument(
-        '--runreport-name', action='store', metavar='NAME',
-        help="Do not overwrite runreport.json",
+        '--report-file', action='store', metavar='FILE',
+        help="Store JSON run report in FILE",
+        envvar='RFM_REPORT_FILE',
+        configvar='general/report_file'
     )
 
     # Check discovery options
@@ -516,32 +532,33 @@ def print_infoline(param, value):
         param = param + ':'
         printer.info(f"  {param.ljust(18)} {value}")
 
-    reframe_info = {
+    session_info = {
+        'cmdline': ' '.join(sys.argv),
+        'config_file': rt.site_config.filename,
+        'data_version': '1.0',
+        'hostname': socket.gethostname(),
+        'prefix_output': rt.output_prefix,
+        'prefix_stage': rt.stage_prefix,
+        'user': os_ext.osuser(),
         'version': os_ext.reframe_version(),
-        'command': repr(' '.join(sys.argv)),
-        'user': f"{os_ext.osuser() or '<unknown>'}",
-        'host': socket.gethostname(),
-        'working_directory': repr(os.getcwd()),
-        'check_search_path': f"{':'.join(loader.load_path)!r}",
-        'recursive_search_path': loader.recurse,
-        'settings_file': site_config.filename,
-        'stage_prefix': repr(rt.stage_prefix),
-        'output_prefix': repr(rt.output_prefix),
+        'workdir': os.getcwd(),
     }
 
     # Print command line
     printer.info(f"[ReFrame Setup]")
-    print_infoline('version', reframe_info['version'])
-    print_infoline('command', reframe_info['command'])
-    print_infoline('launched by',
-                   f"{reframe_info['user']}@{reframe_info['host']}")
-    print_infoline('working directory', reframe_info['working_directory'])
-    print_infoline('settings file', f"{reframe_info['settings_file']!r}")
+    print_infoline('version', session_info['version'])
+    print_infoline('command', session_info['cmdline'])
+    print_infoline(
+        f"launched by",
+        f"{session_info['user'] or '<unknown>'}@{session_info['hostname']}"
+    )
+    print_infoline('working directory', repr(session_info['workdir']))
+    print_infoline('settings file', f"{session_info['config_file']!r}")
     print_infoline('check search path',
-                   f"{'(R) ' if reframe_info['recursive_search_path'] else ''}"
-                   f"{reframe_info['check_search_path']!r}")
-    print_infoline('stage directory', reframe_info['stage_prefix'])
-    print_infoline('output directory', reframe_info['output_prefix'])
+                   f"{'(R) ' if loader.recurse else ''}"
+                   f"{':'.join(loader.load_path)!r}")
+    print_infoline('stage directory', repr(session_info['prefix_stage']))
+    print_infoline('output directory', repr(session_info['prefix_output']))
     printer.info('')
     try:
         # Locate and load checks
@@ -716,13 +733,19 @@ def print_infoline(param, value):
                                   max_retries) from None
             runner = Runner(exec_policy, printer, max_retries)
             try:
-                reframe_info['start_time'] = (
-                    datetime.datetime.today().strftime('%c %Z'))
+                time_start = time.time()
+                session_info['time_start'] = time.strftime(
+                    '%FT%T%z', time.localtime(time_start),
+                )
                 runner.runall(testcases)
             finally:
+                time_end = time.time()
+                session_info['time_end'] = time.strftime(
+                    '%FT%T%z', time.localtime(time_end)
+                )
+                session_info['time_elapsed'] = time_end - time_start
+
                 # Print a retry report if we did any retries
-                reframe_info['end_time'] = (
-                    datetime.datetime.today().strftime('%c %Z'))
                 if runner.stats.failures(run=0):
                     printer.info(runner.stats.retry_report())
 
@@ -736,23 +759,30 @@ def print_infoline(param, value):
                 if options.performance_report:
                     printer.info(runner.stats.performance_report())
 
-                if options.runreport_name:
-                    runreport_file = options.runreport_name
-                else:
-                    runreport_dir = os.path.join(Path.home(), '.local/reframe')
-                    runreport_id = get_next_runreport_index(runreport_dir)
-                    runreport_name = f'runreport-{runreport_id}.json'
-                    Path(runreport_dir).mkdir(parents=True, exist_ok=True)
-                    runreport_file = os.path.join(runreport_dir,
-                                                  runreport_name)
-
+                # Generate the report for this session
+                report_file = os.path.normpath(
+                    os_ext.expandvars(rt.get_option('general/0/report_file'))
+                )
+                os.makedirs(os.path.dirname(report_file), exist_ok=True)
+
+                # Build final report JSON
+                run_stats = runner.stats.json()
+                session_info.update({
+                    'num_cases': run_stats[0]['num_cases'],
+                    'num_failures': run_stats[-1]['num_failures']
+                })
+                json_report = {
+                    'session_info': session_info,
+                    'runs': run_stats
+                }
+                report_file = generate_report_filename(report_file)
                 try:
-                    with open(runreport_file, 'w') as fp:
-                        json.dump(runner.stats.json(reframe_info, force=True),
-                                  fp, indent=4)
-                except OSError:
-                    printer.error(f'invalid path: {runreport_file}')
-                    sys.exit(1)
+                    with open(report_file, 'w') as fp:
+                        json.dump(json_report, fp, indent=2)
+                except OSError as e:
+                    printer.warning(
+                        f'failed to generate report in {report_file!r}: {e}'
+                    )
 
         else:
             printer.error("No action specified. Please specify `-l'/`-L' for "

reframe/frontend/statistics.py

@@ -13,7 +13,9 @@ class TestStats:
     def __init__(self):
         # Tasks per run stored as follows: [[run0_tasks], [run1_tasks], ...]
         self._alltasks = [[]]
-        self._records = []
+
+        # Data collected for all the runs of this session in JSON format
+        self._run_data = []
 
     def add_task(self, task):
         current_run = rt.runtime().current_run
@@ -44,7 +46,6 @@ def retry_report(self):
         report.append('SUMMARY OF RETRIES')
         report.append(line_width * '-')
         messages = {}
-
         for run in range(1, len(self._alltasks)):
             for t in self.tasks(run):
                 partition_name = ''
@@ -67,42 +68,41 @@ def retry_report(self):
 
         return '\n'.join(report)
 
-    def json(self, reframe_info=None, force=False):
-        if not force and self._records:
-            return self._records
+    def json(self, force=False):
+        if not force and self._run_data:
+            return self._run_data
 
-        self._records = {'run_info': []}
-        current_run = rt.runtime().current_run
-        for run_no, run in enumerate(self._alltasks):
-            tests = []
+        for runid, run in enumerate(self._alltasks):
+            testcases = []
+            num_failures = 0
             for t in run:
                 check = t.check
                 partition = check.current_partition
                 entry = {
-                    'testname': check.name,
+                    'build_stderr': None,
+                    'build_stdout': None,
                     'description': check.descr,
-                    'system': check.current_system.name,
                     'environment': None,
-                    'tags': list(check.tags),
-                    'maintainers': check.maintainers,
-                    'scheduler': None,
+                    'fail_reason': None,
+                    'fail_phase': None,
                     'jobid': None,
-                    'nodelist': [],
-                    'job_stdout': None,
                     'job_stderr': None,
-                    'build_stdout': None,
-                    'build_stderr': None,
-                    'failing_reason': None,
-                    'failing_phase': None,
+                    'job_stdout': None,
+                    'name': check.name,
+                    'maintainers': check.maintainers,
+                    'nodelist': [],
                     'outputdir': None,
+                    'perfvars': None,
+                    'result': None,
                     'stagedir': None,
-                    'job_stdout': None,
-                    'job_stderr': None,
-                    'time_setup': t.duration('setup'),
-                    'time_compile': t.duration('complete_complete'),
+                    'scheduler': None,
+                    'system': check.current_system.name,
+                    'tags': list(check.tags),
+                    'time_compile': t.duration('compile_complete'),
+                    'time_performance': t.duration('performance'),
                     'time_run': t.duration('run_complete'),
                     'time_sanity': t.duration('sanity'),
-                    'time_performance': t.duration('performance'),
+                    'time_setup': t.duration('setup'),
                     'time_total': t.duration('total')
                 }
                 partition = check.current_partition
@@ -116,47 +116,66 @@ def json(self, reframe_info=None, force=False):
 
                 if check.job:
                     entry['jobid'] = check.job.jobid
-                    entry['nodelist'] = check.job.nodelist or []
-                    entry['job_stdout'] = check.stdout.evaluate()
                     entry['job_stderr'] = check.stderr.evaluate()
+                    entry['job_stdout'] = check.stdout.evaluate()
+                    entry['nodelist'] = check.job.nodelist or []
 
-                if check._build_job:
-                    entry['build_stdout'] = check.build_stdout.evaluate()
+                if check.build_job:
                     entry['build_stderr'] = check.build_stderr.evaluate()
+                    entry['build_stdout'] = check.build_stdout.evaluate()
 
                 if t.failed:
-                    entry['result'] = 'fail'
+                    num_failures += 1
+                    entry['result'] = 'failure'
+                    entry['stagedir'] = check.stagedir
+                    entry['fail_phase'] = t.failed_stage
                     if t.exc_info is not None:
-                        entry['failing_reason'] = format_exception(
-                            *t.exc_info)
-                        entry['failing_phase'] = t.failed_stage
-                        entry['stagedir'] = check.stagedir
+                        entry['fail_reason'] = format_exception(*t.exc_info)
                 else:
                     entry['result'] = 'success'
                     entry['outputdir'] = check.outputdir
 
-                tests.append(entry)
-
-            self._records['run_info'].append({'runid': run_no, 'tests': tests})
-
-        if reframe_info:
-            self._records['reframe_info'] = reframe_info
-
-        return self._records
+                if check.perf_patterns:
+                    # Record performance variables
+                    entry['perfvars'] = []
+                    for key, ref in check.perfvalues.items():
+                        var = key.split(':')[-1]
+                        val, ref, lower, upper, unit = ref
+                        entry['perfvars'].append({
+                            'name': var,
+                            'reference': ref,
+                            'thres_lower': lower,
+                            'thres_upper': upper,
+                            'unit': unit,
+                            'value': val
+                        })
+
+                testcases.append(entry)
+
+            self._run_data.append({
+                'num_cases': len(run),
+                'num_failures': num_failures,
+                'runid': runid,
+                'testcases': testcases
+            })
+
+        return self._run_data
 
     def failure_report(self):
         line_width = 78
         report = [line_width * '=']
         report.append('SUMMARY OF FAILURES')
-        last_run = rt.runtime().current_run
-        for r in self.json()['run_info'][last_run]['tests']:
+        run_report = self.json()[-1]
+        last_run = run_report['runid']
+        for r in run_report['testcases']:
             if r['result'] == 'success':
                 continue
 
-            retry_info = (f'(for the last of {last_run} retries)'
-                          if last_run > 0 else '')
+            retry_info = (
+                f'(for the last of {last_run} retries)' if last_run > 0 else ''
+            )
             report.append(line_width * '-')
-            report.append(f"FAILURE INFO for {r['testname']} {retry_info}")
+            report.append(f"FAILURE INFO for {r['name']} {retry_info}")
             report.append(f"  * Test Description: {r['description']}")
             report.append(f"  * System partition: {r['system']}")
             report.append(f"  * Environment: {r['environment']}")
@@ -167,13 +186,13 @@ def failure_report(self):
             jobid = r['jobid']
             report.append(f"  * Job type: {job_type} (id={r['jobid']})")
             report.append(f"  * Maintainers: {r['maintainers']}")
-            report.append(f"  * Failing phase: {r['failing_phase']}")
-            report.append(f"  * Rerun with '-n {r['testname']}"
+            report.append(f"  * Failing phase: {r['fail_phase']}")
+            report.append(f"  * Rerun with '-n {r['name']}"
                           f" -p {r['environment']} --system {r['system']}'")
-            report.append(f"  * Reason: {r['failing_reason']}")
-            if r['failing_phase'] == 'sanity':
+            report.append(f"  * Reason: {r['fail_reason']}")
+            if r['fail_phase'] == 'sanity':
                 report.append('Sanity check failure')
-            elif r['failing_phase'] == 'performance':
+            elif r['fail_phase'] == 'performance':
                 report.append('Performance check failure')
             else:
                 # This shouldn't happen...
@@ -227,6 +246,8 @@ def failure_stats(self):
         return ''
 
     def performance_report(self):
+        # FIXME: Adapt this function to use the JSON report
+
         line_width = 78
         report_start = line_width * '='
         report_title = 'PERFORMANCE REPORT'