Merge pull request #3313 from vkarak/feat/default-disable-results-db

[enhancement] Disable results storage by default

Author: vkarak

examples/tutorial/scripts/runall.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -xe
+
+export RFM_ENABLE_RESULTS_STORAGE=1
+
+pushd reframe-examples/tutorial
+reframe -c stream/stream_runonly.py -r
+reframe -c stream/stream_runonly.py -r
+reframe -C config/baseline.py -c stream/stream_runonly.py -r
+reframe -C config/baseline_environs.py -c stream/stream_build_run.py --exec-policy=serial -r
+reframe -C config/baseline_environs.py -c stream/stream_fixtures.py -l
+reframe -C config/baseline_environs.py -c stream/stream_fixtures.py -r
+reframe -C config/baseline_environs.py -c stream/stream_variables.py -S num_threads=2 -r
+reframe -C config/baseline_environs.py -c stream/stream_variables_fixtures.py --exec-policy=serial -S stream_test.stream_binary.array_size=50000000 -r
+reframe -C config/baseline_environs.py -c stream/stream_parameters.py --exec-policy=serial -r
+reframe -C config/baseline_environs.py -c stream/stream_variables_fixtures.py -P num_threads=1,2,4,8 --exec-policy=serial -r
+reframe -c deps/deps_complex.py -r
+reframe --restore-session --failed -r
+reframe -c deps/deps_complex.py --keep-stage-files -r
+reframe --restore-session --keep-stage-files -n T6 -r
+reframe -c deps/deps_complex.py -n T6 -r
+popd

reframe/frontend/cli.py

@@ -628,7 +628,7 @@ def main():
         configvar='general/perf_report_spec',
         envvar='RFM_PERF_REPORT_SPEC',
         help=('Print a report for performance tests '
-              '(default: "now:now/last:+job_nodelist/+result")')
+              '(default: "now-1d:now/last:+job_nodelist/+result")')
     )
     reporting_options.add_argument(
         '--session-extras', action='append', metavar='KV_DATA',
@@ -979,8 +979,7 @@ def restrict_logging():
                                      '--describe-stored-testcases',
                                      '--list-stored-sessions',
                                      '--list-stored-testcases',
-                                     '--performance-compare',
-                                     '--performance-report']):
+                                     '--performance-compare']):
         sys.exit(1)
 
     rt = runtime.runtime()
@@ -1655,9 +1654,12 @@ def module_unuse(*paths):
             if (options.performance_report and
                 not options.dry_run and not report.is_empty()):
                 try:
-                    data = reporting.performance_compare(
-                        rt.get_option('general/0/perf_report_spec'), report
-                    )
+                    if rt.get_option('storage/0/enable'):
+                        data = reporting.performance_compare(
+                            rt.get_option('general/0/perf_report_spec'), report
+                        )
+                    else:
+                        data = report.report_data()
                 except Exception as err:
                     printer.warning(
                         f'failed to generate performance report: {err}'
@@ -1699,7 +1701,8 @@ def module_unuse(*paths):
                     )
 
             # Store the generated report for analytics
-            if not report.is_empty() and not options.dry_run:
+            if (rt.get_option('storage/0/enable') and
+                not report.is_empty() and not options.dry_run):
                 try:
                     sess_uuid = report.store()
                 except Exception as e:

reframe/frontend/reporting/__init__.py

@@ -17,7 +17,6 @@
 import uuid
 from collections import UserDict
 from collections.abc import Hashable
-from filelock import FileLock
 
 import reframe as rfm
 import reframe.utility.jsonext as jsonext
@@ -419,7 +418,11 @@ def update_run_stats(self, stats):
             'num_skipped': self.__report['runs'][-1]['num_skipped']
         })
 
-    def _save(self, filename, compress, link_to_last):
+    def is_empty(self):
+        '''Return :obj:`True` is no test cases where run'''
+        return self.__report['session_info']['num_cases'] == 0
+
+    def save(self, filename, compress=False, link_to_last=True):
         filename = _expand_report_filename(filename, newfile=True)
         with open(filename, 'w') as fp:
             if compress:
@@ -446,20 +449,47 @@ def _save(self, filename, compress, link_to_last):
                 else:
                     raise ReframeError('path exists and is not a symlink')
 
-    def is_empty(self):
-        '''Return :obj:`True` is no test cases where run'''
-        return self.__report['session_info']['num_cases'] == 0
-
-    def save(self, filename, compress=False, link_to_last=True):
-        prefix = os.path.dirname(filename) or '.'
-        with FileLock(os.path.join(prefix, '.report.lock')):
-            self._save(filename, compress, link_to_last)
-
     def store(self):
         '''Store the report in the results storage.'''
 
         return StorageBackend.default().store(self, self.filename)
 
+    def report_data(self):
+        '''Get tabular data from this report'''
+
+        columns = ['name', 'sysenv', 'job_nodelist',
+                   'pvar', 'punit', 'pval', 'result']
+        data = [columns]
+        num_runs = len(self.__report['runs'])
+        for runid, runinfo in enumerate(self.__report['runs']):
+            for tc in map(_TCProxy, runinfo['testcases']):
+                if tc['result'] != 'success' and runid != num_runs - 1:
+                    # Skip this testcase until its last retry
+                    continue
+
+                for pvar, reftuple in tc['perfvalues'].items():
+                    pvar = pvar.split(':')[-1]
+                    pval, _, _, _, punit = reftuple
+                    if pval is None:
+                        # Ignore `None` performance values
+                        # (performance tests that failed sanity)
+                        continue
+
+                    line = []
+                    for c in columns:
+                        if c == 'pvar':
+                            line.append(pvar)
+                        elif c == 'pval':
+                            line.append(pval)
+                        elif c == 'punit':
+                            line.append(punit)
+                        else:
+                            line.append(tc[c])
+
+                    data.append(line)
+
+        return data
+
     def generate_xml_report(self):
         '''Generate a JUnit report from a standard ReFrame JSON report.'''
 

reframe/schemas/config.json

@@ -589,7 +589,7 @@
         "general/module_mappings": [],
         "general/non_default_craype": false,
         "general/perf_info_level": "info",
-        "general/perf_report_spec": "now:now/last:/+job_nodelist+result",
+        "general/perf_report_spec": "now-1d:now/last:/+job_nodelist+result",
         "general/pipeline_timeout": 3,
         "general/purge_environment": false,
         "general/remote_detect": false,
@@ -634,7 +634,7 @@
         "logging/handlers_perflog/httpjson_debug": false,
         "modes/options": [],
         "modes/target_systems": ["*"],
-        "storage/enable": true,
+        "storage/enable": false,
         "storage/backend": "sqlite",
         "storage/sqlite_conn_timeout": 60,
         "storage/sqlite_db_file": "${HOME}/.reframe/reports/results.db",

unittests/test_cli.py

@@ -420,7 +420,14 @@ def test_perflogdir_from_env(run_reframe, tmp_path, monkeypatch):
                           'default' / 'PerformanceFailureCheck.log')
 
 
-def test_performance_report(run_reframe, run_action):
+@pytest.fixture(params=['storage=yes', 'storage=no'])
+def storage_enabled(request, monkeypatch):
+    value = request.param.split('=')[1]
+    monkeypatch.setenv('RFM_ENABLE_RESULTS_STORAGE', value)
+    return value == 'yes'
+
+
+def test_performance_report(run_reframe, run_action, storage_enabled):
     returncode, stdout, stderr = run_reframe(
         checkpath=['unittests/resources/checks/frontend_checks.py'],
         more_options=['-n', '^PerformanceFailureCheck',
@@ -433,6 +440,9 @@ def test_performance_report(run_reframe, run_action):
     else:
         assert returncode == 0
 
+    if run_action != 'dry_run':
+        assert 'PERFORMANCE REPORT' in stdout
+
     assert 'Traceback' not in stdout
     assert 'Traceback' not in stderr
 
@@ -1269,7 +1279,8 @@ def assert_no_crash(returncode, stdout, stderr, exitcode=0):
     return returncode, stdout, stderr
 
 
-def test_storage_options(run_reframe, tmp_path, table_format):
+def test_storage_options(run_reframe, tmp_path, table_format, monkeypatch):
+    monkeypatch.setenv('RFM_ENABLE_RESULTS_STORAGE', 'yes')
     run_reframe2 = functools.partial(
         run_reframe,
         checkpath=['unittests/resources/checks/frontend_checks.py'],
@@ -1335,8 +1346,7 @@ def test_storage_options(run_reframe, tmp_path, table_format):
     '--describe-stored-testcases=now-1d:now',
     '--list-stored-sessions',
     '--list-stored-testcases=now-1d:now/mean:/',
-    '--performance-compare=now-1d:now/now-1d/mean:/',
-    '--performance-report=now-1d:now/mean:/'
+    '--performance-compare=now-1d:now/now-1d/mean:/'
 ])
 def storage_option(request):
     return request.param
@@ -1359,7 +1369,8 @@ def test_disabled_results_storage(run_reframe, storage_option, monkeypatch):
     assert 'requires results storage' in stdout
 
 
-def test_session_annotations(run_reframe):
+def test_session_annotations(run_reframe, monkeypatch):
+    monkeypatch.setenv('RFM_ENABLE_RESULTS_STORAGE', 'yes')
     assert_no_crash(*run_reframe(
         checkpath=['unittests/resources/checks/frontend_checks.py'],
         action='-r',
@@ -1373,13 +1384,14 @@ def test_session_annotations(run_reframe):
         assert text in stdout
 
 
-def test_performance_compare(run_reframe, table_format):
+def test_performance_compare(run_reframe, table_format, monkeypatch):
     def assert_no_crash(returncode, stdout, stderr, exitcode=0):
         assert returncode == exitcode
         assert 'Traceback' not in stdout
         assert 'Traceback' not in stderr
         return returncode, stdout, stderr
 
+    monkeypatch.setenv('RFM_ENABLE_RESULTS_STORAGE', 'yes')
     run_reframe2 = functools.partial(
         run_reframe,
         checkpath=['unittests/resources/checks/frontend_checks.py'],