new runreport format

Author: rsarm

reframe/frontend/cli.py

@@ -3,13 +3,15 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
+import datetime
 import inspect
 import json
 import os
 import re
 import socket
 import sys
 import traceback
+from pathlib import Path
 
 import reframe
 import reframe.core.config as config
@@ -138,7 +140,7 @@ def main():
         envvar='RFM_SAVE_LOG_FILES', configvar='general/save_log_files'
     )
     output_options.add_argument(
-        '--keep-runreport', action='store_true', default=False,
+        '--runreport-name', action='store', metavar='NAME',
         help="Do not overwrite runreport.json",
     )
 
@@ -502,19 +504,32 @@ def print_infoline(param, value):
         param = param + ':'
         printer.info(f"  {param.ljust(18)} {value}")
 
+    reframe_info = {
+        'version': os_ext.reframe_version(),
+        'command': repr(' '.join(sys.argv)),
+        'user': f"{os_ext.osuser() or '<unknown>'}",
+        'host': socket.gethostname(),
+        'working_directory': repr(os.getcwd()),
+        'check_search_path': f"{':'.join(loader.load_path)!r}",
+        'recursive_search_path': loader.recurse,
+        'settings_file': site_config.filename,
+        'stage_prefix': repr(rt.stage_prefix),
+        'output_prefix': repr(rt.output_prefix),
+    }
+
     # Print command line
     printer.info(f"[ReFrame Setup]")
-    print_infoline('version', os_ext.reframe_version())
-    print_infoline('command', repr(' '.join(sys.argv)))
+    print_infoline('version', reframe_info['version'])
+    print_infoline('command', reframe_info['command'])
     print_infoline('launched by',
-                   f"{os_ext.osuser() or '<unknown>'}@{socket.gethostname()}")
-    print_infoline('working directory', repr(os.getcwd()))
-    print_infoline('settings file', f'{site_config.filename!r}')
+                   f"{reframe_info['user']}@{reframe_info['host']}")
+    print_infoline('working directory', reframe_info['working_directory'])
+    print_infoline('settings file', f"{reframe_info['settings_file']!r}")
     print_infoline('check search path',
-                   f"{'(R) ' if loader.recurse else ''}"
-                   f"{':'.join(loader.load_path)!r}")
-    print_infoline('stage directory', repr(rt.stage_prefix))
-    print_infoline('output directory', repr(rt.output_prefix))
+                   f"{'(R) ' if reframe_info['recursive_search_path'] else ''}"
+                   f"{reframe_info['check_search_path']!r}")
+    print_infoline('stage directory', reframe_info['stage_prefix'])
+    print_infoline('output directory', reframe_info['output_prefix'])
     printer.info('')
     try:
         # Locate and load checks
@@ -689,9 +704,13 @@ def print_infoline(param, value):
                                   max_retries) from None
             runner = Runner(exec_policy, printer, max_retries)
             try:
+                reframe_info['start_time'] = (
+                    datetime.datetime.today().strftime('%c %Z'))
                 runner.runall(testcases)
             finally:
                 # Print a retry report if we did any retries
+                reframe_info['end_time'] = (
+                    datetime.datetime.today().strftime('%c %Z'))
                 if runner.stats.failures(run=0):
                     printer.info(runner.stats.retry_report())
 
@@ -705,14 +724,23 @@ def print_infoline(param, value):
                 if options.performance_report:
                     printer.info(runner.stats.performance_report())
 
-                if options.keep_runreport:
-                    runreport_id = get_next_runreport_index()
-                    runreport_name = f'runreport-{runreport_id}.json'
+                if options.runreport_name:
+                    runreport_file = options.runreport_name
                 else:
-                    runreport_name = 'runreport.json'
-
-                with open(runreport_name, 'w') as fp:
-                    json.dump(runner.stats.json(), fp, indent=4)
+                    runreport_dir = os.path.join(Path.home(), '.local/reframe')
+                    runreport_id = get_next_runreport_index(runreport_dir)
+                    runreport_name = f'runreport-{runreport_id}.json'
+                    Path(runreport_dir).mkdir(parents=True, exist_ok=True)
+                    runreport_file = os.path.join(runreport_dir,
+                                                  runreport_name)
+
+                try:
+                    with open(runreport_file, 'w') as fp:
+                        json.dump(runner.stats.json(reframe_info, force=True),
+                                  fp, indent=4)
+                except OSError:
+                    printer.error(f'invalid path: {runreport_file}')
+                    sys.exit(1)
 
         else:
             printer.error("No action specified. Please specify `-l'/`-L' for "

reframe/frontend/statistics.py

@@ -71,13 +71,14 @@ def retry_report(self):
 
         return '\n'.join(report)
 
-    def json(self, force=False):
+    def json(self, reframe_info=None, force=False):
         if not force and self._records:
             return self._records
 
-        self._records = []
+        self._records = {'run_info': []}
         current_run = rt.runtime().current_run
         for run_no, run in enumerate(self._alltasks):
+            tests = []
             for t in run:
                 check = t.check
                 partition = check.current_partition
@@ -138,9 +139,12 @@ def json(self, force=False):
                     entry['result'] = 'success'
                     entry['outputdir'] = check.outputdir
 
-                entry['runid'] = run_no
+                tests.append(entry)
+
+            self._records['run_info'].append({'runid': run_no, 'tests': tests})
 
-                self._records.append(entry)
+        if reframe_info:
+            self._records['reframe_info'] = reframe_info
 
         return self._records
 
@@ -149,8 +153,8 @@ def failure_report(self):
         report = [line_width * '=']
         report.append('SUMMARY OF FAILURES')
         last_run = rt.runtime().current_run
-        for r in self.json():
-            if r['result'] == 'success' or r['runid'] != last_run:
+        for r in self.json()['run_info'][last_run]['tests']:
+            if r['result'] == 'success':
                 continue
 
             retry_info = (f'(for the last of {last_run} retries)'

reframe/schemas/runreport.json

@@ -2,55 +2,95 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "$id": "https://raw.githubusercontent.com/eth-cscs/reframe/master/schemas/runreport.json",
     "title": "Validation schema for ReFrame's run report",
-    "type": "array",
-    "items": {
-        "type": "object",
-        "properties": {
-            "testname": {"type": "string"},
-            "description": {"type": ["string"]},
-            "system": {"type": "string"},
-            "environment": {"type": ["string", "null"]},
-            "stagedir": {"type": ["string", "null"]},
-            "outputdir": {"type": ["string", "null"]},
-            "nodelist": {
-                "type": "array",
-                "items": {"type": "string"}
+    "type": "object",
+    "properties": {
+        "reframe_info": {
+            "type": "object",
+            "properties": {
+                "version": {"type": "string"},
+                "command": {"type": "string"},
+                "user": {"type": "string"},
+                "host": {"type": "string"},
+                "check_search_path": {"type": "string"},
+                "working_directory": {"type": "string"},
+                "recursive_search_path": {"type": "boolean"},
+                "settings_file": {"type": "string"},
+                "stage_prefix": {"type": "string"},
+                "output_prefix": {"type": "string"},
+                "start_time": {"type": "string"},
+                "end_time": {"type": "string"}
             },
-            "scheduler": {"type": ["string", "null"]},
-            "jobid": {"type": ["number", "null"]},
-            "result": {
-                "type": "string",
-                "enum": ["success", "fail"]
-            },
-            "failing_phase": {"type": ["string", "null"]},
-            "failing_reason": {"type": ["string", "null"]},
-            "build_stdout": {"type": ["string", "null"]},
-            "build_stderr": {"type": ["string", "null"]},
-            "job_stdout": {"type": ["string", "null"]},
-            "job_stderr": {"type": ["string", "null"]},
-            "maintainers": {
-                "type": "array",
-                "items": {"type": ["string"]}
-            },
-            "tags": {
-                "type": "array",
-                "items": {"type": ["string"]}
-            },
-            "runid": {"type": "number"},
-            "time_setup": {"type": ["number", "null"]},
-            "time_compile": {"type": ["number", "null"]},
-            "time_run": {"type": ["number", "null"]},
-            "time_sanity": {"type": ["number", "null"]},
-            "time_performance": {"type": ["number", "null"]},
-            "time_total": {"type": ["number", "null"]}
+            "additionalProperties": false,
+            "required": ["check_search_path", "command", "end_time", "host",
+                         "output_prefix", "recursive_search_path",
+                         "stage_prefix", "start_time", "user", "version",
+                         "working_directory"]
         },
-        "additionalProperties": false,
-        "required": ["build_stderr", "build_stdout", "description",
-                     "environment", "failing_phase", "failing_reason",
-                     "job_stderr", "job_stdout", "jobid", "maintainers",
-                     "nodelist", "outputdir", "result", "runid",
-                     "scheduler", "stagedir", "system", "tags", "testname",
-                     "time_compile", "time_performance", "time_run",
-                     "time_sanity", "time_setup", "time_total"]
-    }
+        "run_info": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "runid": {"type": "number"},
+                    "tests": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "testname": {"type": "string"},
+                                "description": {"type": "string"},
+                                "system": {"type": "string"},
+                                "environment": {"type": ["string", "null"]},
+                                "stagedir": {"type": ["string", "null"]},
+                                "outputdir": {"type": ["string", "null"]},
+                                "nodelist": {
+                                    "type": "array",
+                                    "items": {"type": "string"}
+                                },
+                                "scheduler": {"type": ["string", "null"]},
+                                "jobid": {"type": ["number", "null"]},
+                                "result": {
+                                    "type": "string",
+                                    "enum": ["success", "fail"]
+                                },
+                                "failing_phase": {"type": ["string", "null"]},
+                                "failing_reason": {"type": ["string", "null"]},
+                                "build_stdout": {"type": ["string", "null"]},
+                                "build_stderr": {"type": ["string", "null"]},
+                                "job_stdout": {"type": ["string", "null"]},
+                                "job_stderr": {"type": ["string", "null"]},
+                                "maintainers": {
+                                    "type": "array",
+                                    "items": {"type": ["string"]}
+                                },
+                                "tags": {
+                                    "type": "array",
+                                    "items": {"type": "string"}
+                                },
+                                "runid": {"type": "number"},
+                                "time_setup": {"type": ["number", "null"]},
+                                "time_compile": {"type": ["number", "null"]},
+                                "time_run": {"type": ["number", "null"]},
+                                "time_sanity": {"type": ["number", "null"]},
+                                "time_performance": {"type": ["number", "null"]},
+                                "time_total": {"type": ["number", "null"]}
+                            },
+                            "additionalProperties": false,
+                            "required": ["build_stderr", "build_stdout", "description",
+                                         "environment", "failing_phase", "failing_reason",
+                                         "job_stderr", "job_stdout", "jobid", "maintainers",
+                                         "nodelist", "outputdir", "result", "scheduler",
+                                         "stagedir", "system", "tags", "testname",
+                                         "time_compile", "time_performance", "time_run",
+                                         "time_sanity", "time_setup", "time_total"]
+                        },
+                        "additionalProperties": false,
+                        "required": ["runid", "tests"]
+                    }
+                }
+            }
+        }
+    },
+    "additionalProperties": false,
+    "required": ["reframe_info", "run_info"]
 }

reframe/utility/__init__.py

@@ -16,18 +16,17 @@
 from collections import UserDict
 
 
-def get_next_runreport_index():
-    runreports = os.listdir('.')
+def get_next_runreport_index(runreport_dir):
+    runreports = os.listdir(runreport_dir)
     pattern = r'runreport-\d+.json'
     filenames = filter(re.compile(pattern).match, runreports)
-    filenames = sorted(filenames)
 
     if not filenames:
         return 1
 
-    for i, f in enumerate(filenames, 1):
-        idx = f.split('.')[0].split('-')[1]
-        idx = int(idx)
+    idx_list = sorted([int(f.split('.')[0].split('-')[1])
+                       for f in filenames])
+    for i, idx in enumerate(idx_list):
         if idx != i:
             return i
 

unittests/test_policies.py

@@ -3,10 +3,13 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
+import datetime
 import json
 import jsonschema
 import os
 import pytest
+import socket
+import sys
 
 import reframe
 import reframe.core.runtime as rt
@@ -116,10 +119,25 @@ def validate_report(runreport):
 
 
 def test_runall(make_runner, make_cases, common_exec_ctx):
+    reframe_info = {
+        'version': os_ext.reframe_version(),
+        'command': repr(' '.join(sys.argv)),
+        'user': f"{os_ext.osuser() or '<unknown>'}",
+        'host': socket.gethostname(),
+        'working_directory': repr(os.getcwd()),
+        'check_search_path': 'unittests/resources/checks',
+        'recursive_search_path': True,
+        'settings_file': fixtures.TEST_CONFIG_FILE,
+        'stage_prefix': repr(rt.runtime().stage_prefix),
+        'output_prefix': repr(rt.runtime().output_prefix),
+    }
     runner = make_runner()
+    reframe_info['start_time'] = datetime.datetime.today().strftime('%c %Z')
     runner.runall(make_cases())
+    reframe_info['end_time'] = datetime.datetime.today().strftime('%c %Z')
     stats = runner.stats
-    validate_report(runner.stats.json())
+    runreport = runner.stats.json(reframe_info, force=True)
+    validate_report(runreport)
     assert 8 == stats.num_cases()
     assert_runall(runner)
     assert 5 == len(stats.failures())