Merge branch 'master' into feat/progress-report

Author: Vasileios Karakasis

ci-scripts/ci-runner.bash

@@ -137,13 +137,15 @@ fi
 if [[ $(hostname) =~ tsa ]]; then
     # FIXME: Temporary workaround until we have a reframe module on Tsa
     module load python
-    python3 -m venv venv.unittests
-    source venv.unittests/bin/activate
-    pip install -r requirements.txt
 else
     module load reframe
 fi
 
+# Always install our requirements
+python3 -m venv venv.unittests
+source venv.unittests/bin/activate
+pip install -r requirements.txt
+
 echo "=============="
 echo "Loaded Modules"
 echo "=============="

cscs-checks/microbenchmarks/osu/osu_tests.py

@@ -56,8 +56,8 @@ def __init__(self, variant):
 class FlexAlltoallTest(rfm.RegressionTest):
     def __init__(self):
         self.valid_systems = ['daint:gpu', 'daint:mc',
-                              'dom:gpu', 'dom:mc', 'tiger:gpu',
-                              'kesch:cn', 'kesch:pn',
+                              'dom:gpu', 'dom:mc',
+                              'tiger:gpu', 'kesch:cn',
                               'arolla:cn', 'arolla:pn',
                               'tsa:cn', 'tsa:pn']
         self.valid_prog_environs = ['PrgEnv-cray']

reframe/core/config.py

@@ -4,14 +4,20 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import collections.abc
+import json
+import jsonschema
+import os
 import re
+import tempfile
 
+import reframe
 import reframe.core.debug as debug
 import reframe.core.fields as fields
 import reframe.utility as util
 import reframe.utility.os_ext as os_ext
 import reframe.utility.typecheck as types
-from reframe.core.exceptions import (ConfigError, ReframeFatalError)
+from reframe.core.exceptions import (ConfigError, ReframeError,
+                                     ReframeFatalError)
 
 
 _settings = None
@@ -222,3 +228,147 @@ def create_env(system, partition, name):
                 system.add_partition(part)
 
             self._systems[sys_name] = system
+
+
+def convert_old_config(filename):
+    old_config = load_settings_from_file(filename)
+    converted = {
+        'systems': [],
+        'environments': [],
+        'logging': [],
+        'perf_logging': [],
+    }
+    old_systems = old_config.site_configuration['systems'].items()
+    for sys_name, sys_specs in old_systems:
+        sys_dict = {'name': sys_name}
+        sys_dict.update(sys_specs)
+
+        # Make variables dictionary into a list of lists
+        if 'variables' in sys_specs:
+            sys_dict['variables'] = [
+                [vname, v] for vname, v in sys_dict['variables'].items()
+            ]
+
+        # Make partitions dictionary into a list
+        if 'partitions' in sys_specs:
+            sys_dict['partitions'] = []
+            for pname, p in sys_specs['partitions'].items():
+                new_p = {'name': pname}
+                new_p.update(p)
+                if p['scheduler'] == 'nativeslurm':
+                    new_p['scheduler'] = 'slurm'
+                    new_p['launcher'] = 'srun'
+                elif p['scheduler'] == 'local':
+                    new_p['scheduler'] = 'local'
+                    new_p['launcher'] = 'local'
+                else:
+                    sched, launch, *_ = p['scheduler'].split('+')
+                    new_p['scheduler'] = sched
+                    new_p['launcher'] = launch
+
+                # Make resources dictionary into a list
+                if 'resources' in p:
+                    new_p['resources'] = [
+                        {'name': rname, 'options': r}
+                        for rname, r in p['resources'].items()
+                    ]
+
+                # Make variables dictionary into a list of lists
+                if 'variables' in p:
+                    new_p['variables'] = [
+                        [vname, v] for vname, v in p['variables'].items()
+                    ]
+
+                if 'container_platforms' in p:
+                    new_p['container_platforms'] = []
+                    for cname, c in p['container_platforms'].items():
+                        new_c = {'name': cname}
+                        new_c.update(c)
+                        if 'variables' in c:
+                            new_c['variables'] = [
+                                [vn, v] for vn, v in c['variables'].items()
+                            ]
+
+                        new_p['container_platforms'].append(new_c)
+
+                sys_dict['partitions'].append(new_p)
+
+        converted['systems'].append(sys_dict)
+
+    old_environs = old_config.site_configuration['environments'].items()
+    for env_target, env_entries in old_environs:
+        for ename, e in env_entries.items():
+            new_env = {'name': ename}
+            if env_target != '*':
+                new_env['target_systems'] = [env_target]
+
+            new_env.update(e)
+
+            # Convert variables dictionary to a list of lists
+            if 'variables' in e:
+                new_env['variables'] = [
+                    [vname, v] for vname, v in e['variables'].items()
+                ]
+
+            # Type attribute is not used anymore
+            if 'type' in new_env:
+                del new_env['type']
+
+            converted['environments'].append(new_env)
+
+    if 'modes' in old_config.site_configuration:
+        converted['modes'] = []
+        old_modes = old_config.site_configuration['modes'].items()
+        for target_mode, mode_entries in old_modes:
+            for mname, m in mode_entries.items():
+                new_mode = {'name': mname, 'options': m}
+                if target_mode != '*':
+                    new_mode['target_systems'] = [target_mode]
+
+                converted['modes'].append(new_mode)
+
+    def update_logging_config(log_name, original_log):
+        new_handlers = []
+        for h in original_log['handlers']:
+            new_h = h
+            new_h['level'] = h['level'].lower()
+            new_handlers.append(new_h)
+
+        converted[log_name].append(
+            {
+                'level': original_log['level'].lower(),
+                'handlers': new_handlers
+            }
+        )
+
+    update_logging_config('logging', old_config.logging_config)
+    update_logging_config('perf_logging', old_config.perf_logging_config)
+    converted['general'] = [{}]
+    if hasattr(old_config, 'checks_path'):
+        converted['general'][0][
+            'check_search_path'
+        ] = old_config.checks_path
+
+    if hasattr(old_config, 'checks_path_recurse'):
+        converted['general'][0][
+            'check_search_recursive'
+        ] = old_config.checks_path_recurse
+
+    if converted['general'] == [{}]:
+        del converted['general']
+
+    # Validate the converted file
+    schema_filename = os.path.join(reframe.INSTALL_PREFIX,
+                                   'schemas', 'config.json')
+
+    # We let the following statements raise, because if they do, that's a BUG
+    with open(schema_filename) as fp:
+        schema = json.loads(fp.read())
+
+    jsonschema.validate(converted, schema)
+    with tempfile.NamedTemporaryFile(mode='w', delete=False) as fp:
+        fp.write(f"#\n# This file was automatically generated "
+                 f"by ReFrame based on '{filename}'.\n#\n\n")
+        fp.write(f'site_configuration = {util.ppretty(converted)}\n')
+
+    return fp.name

reframe/frontend/cli.py

@@ -39,13 +39,15 @@ def format_check(check, detailed):
 
     if detailed:
         lines += [
-            '      - description: %s' % check.descr,
-            '      - systems: %s' % ', '.join(check.valid_systems),
-            '      - environments: %s' % ', '.join(check.valid_prog_environs),
-            '      - modules: %s' % ', '.join(check.modules),
-            '      - task allocation: %s' % flex,
-            '      - tags: %s' % ', '.join(check.tags),
-            '      - maintainers: %s' % ', '.join(check.maintainers)
+            f"      - description: {check.descr}",
+            f"      - systems: {', '.join(check.valid_systems)}",
+            f"      - environments: {', '.join(check.valid_prog_environs)}",
+            f"      - modules: {', '.join(check.modules)}",
+            f"      - task allocation: {flex}",
+            f"      - dependencies: "
+            f"{', '.join([d[0] for d in check.user_deps()])}",
+            f"      - tags: {', '.join(check.tags)}",
+            f"      - maintainers: {', '.join(check.maintainers)}"
         ]
 
     return '\n'.join(lines)
@@ -231,6 +233,9 @@ def main():
     misc_options.add_argument(
         '--nocolor', action='store_false', dest='colorize', default=True,
         help='Disable coloring of output')
+    misc_options.add_argument(
+        '--failure-stats', action='store_true',
+        help='Print failure statistics')
     misc_options.add_argument('--performance-report', action='store_true',
                               help='Print the performance report')
     misc_options.add_argument(
@@ -630,6 +635,8 @@ def main():
                 if runner.stats.failures():
                     printer.info(runner.stats.failure_report())
                     success = False
+                    if options.failure_stats:
+                        printer.info(runner.stats.failure_stats())
 
                 if options.performance_report:
                     printer.info(runner.stats.performance_report())

reframe/frontend/statistics.py

@@ -97,6 +97,8 @@ def failure_report(self):
             report.append('  * Job type: %s (id=%s)' % (job_type, jobid))
             report.append('  * Maintainers: %s' % check.maintainers)
             report.append('  * Failing phase: %s' % tf.failed_stage)
+            report.append("  * Rerun with '-n %s -p %s --system %s'" %
+                          (check.name, environ_name, partname))
             reason = '  * Reason: '
             if tf.exc_info is not None:
                 from reframe.core.exceptions import format_exception
@@ -115,6 +117,50 @@ def failure_report(self):
         report.append(line_width * '-')
         return '\n'.join(report)
 
+    def failure_stats(self):
+        failures = {}
+        current_run = rt.runtime().current_run
+        for tf in (t for t in self.tasks(current_run) if t.failed):
+            check = tf.check
+            partition = check.current_partition
+            partname = partition.fullname if partition else 'None'
+            environ_name = (check.current_environ.name
+                            if check.current_environ else 'None')
+            f = f'[{check.name}, {environ_name}, {partname}]'
+            if tf.failed_stage not in failures:
+                failures[tf.failed_stage] = []
+
+            failures[tf.failed_stage].append(f)
+
+        line_width = 78
+        stats_start = line_width * '='
+        stats_title = 'FAILURE STATISTICS'
+        stats_end = line_width * '-'
+        stats_body = []
+        row_format = "{:<11} {:<5} {}"
+        stats_hline = row_format.format(11*'-', 5*'-', 60*'-')
+        stats_header = row_format.format('Phase', '#', 'Failing test cases')
+        num_tests = len(self.tasks(current_run))
+        num_failures = 0
+        for l in failures.values():
+            num_failures += len(l)
+
+        stats_body = ['']
+        stats_body.append('Total number of test cases: %s' % num_tests)
+        stats_body.append('Total number of failures: %s' % num_failures)
+        stats_body.append('')
+        stats_body.append(stats_header)
+        stats_body.append(stats_hline)
+        for p, l in failures.items():
+            stats_body.append(row_format.format(p, len(l), l[0]))
+            for f in l[1:]:
+                stats_body.append(row_format.format('', '', str(f)))
+
+        if stats_body:
+            return '\n'.join([stats_start, stats_title, *stats_body,
+                              stats_end])
+        return ''
+
     def performance_report(self):
         line_width = 78
         report_start = line_width * '='

reframe/utility/__init__.py

@@ -126,6 +126,64 @@ def toalphanum(s):
     return re.sub(r'\W', '_', s)
 
 
+def ppretty(value, htchar=' ', lfchar='\n', indent=4, basic_offset=0):
+    '''Format string of dictionaries, lists and tuples
+
+    :arg value: The value to be formatted.
+    :arg htchar: Horizontal-tab character.
+    :arg lfchar: Linefeed character.
+    :arg indent: Number of htchar characters for every indentation level.
+    :arg basic_offset: Basic offset for the representation, any additional
+        indentation space is added to the ``basic_offset``.
+    :returns: a formatted string of the ``value``.
+    '''
+
+    nlch = lfchar + htchar * indent * (basic_offset + 1)
+    if isinstance(value, tuple):
+        if value == ():
+            return '()'
+
+        items = [
+            nlch + ppretty(item, htchar, lfchar, indent, basic_offset + 1)
+            for item in value
+        ]
+        return '(%s)' % (','.join(items) + lfchar +
+                         htchar * indent * basic_offset)
+    elif isinstance(value, list):
+        if value == []:
+            return '[]'
+
+        items = [
+            nlch + ppretty(item, htchar, lfchar, indent, basic_offset + 1)
+            for item in value
+        ]
+        return '[%s]' % (','.join(items) + lfchar +
+                         htchar * indent * basic_offset)
+    elif isinstance(value, dict):
+        if value == {}:
+            return '{}'
+
+        items = [
+            nlch + repr(key) + ': ' +
+            ppretty(value[key], htchar, lfchar, indent, basic_offset + 1)
+            for key in value
+        ]
+        return '{%s}' % (','.join(items) + lfchar +
+                         htchar * indent * basic_offset)
+    elif isinstance(value, set):
+        if value == set():
+            return 'set()'
+
+        items = [
+            nlch + ppretty(item, htchar, lfchar, indent, basic_offset + 1)
+            for item in value
+        ]
+        return '{%s}' % (','.join(items) + lfchar +
+                         htchar * indent * basic_offset)
+    else:
+        return repr(value)
+
+
 class ScopedDict(UserDict):
     '''This is a special dict that imposes scopes on its keys.