Merge pull request #1186 from ekouts/feat/json-config-tool

[feat] Add a tool to convert from the old to the new configuration syntax

Author: Vasileios Karakasis

ci-scripts/ci-runner.bash

@@ -137,13 +137,15 @@ fi
 if [[ $(hostname) =~ tsa ]]; then
     # FIXME: Temporary workaround until we have a reframe module on Tsa
     module load python
-    python3 -m venv venv.unittests
-    source venv.unittests/bin/activate
-    pip install -r requirements.txt
 else
     module load reframe
 fi
 
+# Always install our requirements
+python3 -m venv venv.unittests
+source venv.unittests/bin/activate
+pip install -r requirements.txt
+
 echo "=============="
 echo "Loaded Modules"
 echo "=============="

reframe/core/config.py

@@ -4,14 +4,20 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import collections.abc
+import json
+import jsonschema
+import os
 import re
+import tempfile
 
+import reframe
 import reframe.core.debug as debug
 import reframe.core.fields as fields
 import reframe.utility as util
 import reframe.utility.os_ext as os_ext
 import reframe.utility.typecheck as types
-from reframe.core.exceptions import (ConfigError, ReframeFatalError)
+from reframe.core.exceptions import (ConfigError, ReframeError,
+                                     ReframeFatalError)
 
 
 _settings = None
@@ -222,3 +228,147 @@ def create_env(system, partition, name):
                 system.add_partition(part)
 
             self._systems[sys_name] = system
+
+
+def convert_old_config(filename):
+    old_config = load_settings_from_file(filename)
+    converted = {
+        'systems': [],
+        'environments': [],
+        'logging': [],
+        'perf_logging': [],
+    }
+    old_systems = old_config.site_configuration['systems'].items()
+    for sys_name, sys_specs in old_systems:
+        sys_dict = {'name': sys_name}
+        sys_dict.update(sys_specs)
+
+        # Make variables dictionary into a list of lists
+        if 'variables' in sys_specs:
+            sys_dict['variables'] = [
+                [vname, v] for vname, v in sys_dict['variables'].items()
+            ]
+
+        # Make partitions dictionary into a list
+        if 'partitions' in sys_specs:
+            sys_dict['partitions'] = []
+            for pname, p in sys_specs['partitions'].items():
+                new_p = {'name': pname}
+                new_p.update(p)
+                if p['scheduler'] == 'nativeslurm':
+                    new_p['scheduler'] = 'slurm'
+                    new_p['launcher'] = 'srun'
+                elif p['scheduler'] == 'local':
+                    new_p['scheduler'] = 'local'
+                    new_p['launcher'] = 'local'
+                else:
+                    sched, launch, *_ = p['scheduler'].split('+')
+                    new_p['scheduler'] = sched
+                    new_p['launcher'] = launch
+
+                # Make resources dictionary into a list
+                if 'resources' in p:
+                    new_p['resources'] = [
+                        {'name': rname, 'options': r}
+                        for rname, r in p['resources'].items()
+                    ]
+
+                # Make variables dictionary into a list of lists
+                if 'variables' in p:
+                    new_p['variables'] = [
+                        [vname, v] for vname, v in p['variables'].items()
+                    ]
+
+                if 'container_platforms' in p:
+                    new_p['container_platforms'] = []
+                    for cname, c in p['container_platforms'].items():
+                        new_c = {'name': cname}
+                        new_c.update(c)
+                        if 'variables' in c:
+                            new_c['variables'] = [
+                                [vn, v] for vn, v in c['variables'].items()
+                            ]
+
+                        new_p['container_platforms'].append(new_c)
+
+                sys_dict['partitions'].append(new_p)
+
+        converted['systems'].append(sys_dict)
+
+    old_environs = old_config.site_configuration['environments'].items()
+    for env_target, env_entries in old_environs:
+        for ename, e in env_entries.items():
+            new_env = {'name': ename}
+            if env_target != '*':
+                new_env['target_systems'] = [env_target]
+
+            new_env.update(e)
+
+            # Convert variables dictionary to a list of lists
+            if 'variables' in e:
+                new_env['variables'] = [
+                    [vname, v] for vname, v in e['variables'].items()
+                ]
+
+            # Type attribute is not used anymore
+            if 'type' in new_env:
+                del new_env['type']
+
+            converted['environments'].append(new_env)
+
+    if 'modes' in old_config.site_configuration:
+        converted['modes'] = []
+        old_modes = old_config.site_configuration['modes'].items()
+        for target_mode, mode_entries in old_modes:
+            for mname, m in mode_entries.items():
+                new_mode = {'name': mname, 'options': m}
+                if target_mode != '*':
+                    new_mode['target_systems'] = [target_mode]
+
+                converted['modes'].append(new_mode)
+
+    def update_logging_config(log_name, original_log):
+        new_handlers = []
+        for h in original_log['handlers']:
+            new_h = h
+            new_h['level'] = h['level'].lower()
+            new_handlers.append(new_h)
+
+        converted[log_name].append(
+            {
+                'level': original_log['level'].lower(),
+                'handlers': new_handlers
+            }
+        )
+
+    update_logging_config('logging', old_config.logging_config)
+    update_logging_config('perf_logging', old_config.perf_logging_config)
+    converted['general'] = [{}]
+    if hasattr(old_config, 'checks_path'):
+        converted['general'][0][
+            'check_search_path'
+        ] = old_config.checks_path
+
+    if hasattr(old_config, 'checks_path_recurse'):
+        converted['general'][0][
+            'check_search_recursive'
+        ] = old_config.checks_path_recurse
+
+    if converted['general'] == [{}]:
+        del converted['general']
+
+    # Validate the converted file
+    schema_filename = os.path.join(reframe.INSTALL_PREFIX,
+                                   'schemas', 'config.json')
+
+    # We let the following statements raise, because if they do, that's a BUG
+    with open(schema_filename) as fp:
+        schema = json.loads(fp.read())
+
+    jsonschema.validate(converted, schema)
+    with tempfile.NamedTemporaryFile(mode='w', delete=False) as fp:
+        fp.write(f"#\n# This file was automatically generated "
+                 f"by ReFrame based on '{filename}'.\n#\n\n")
+        fp.write(f'site_configuration = {util.ppretty(converted)}\n')
+
+    return fp.name

reframe/utility/__init__.py

@@ -126,6 +126,64 @@ def toalphanum(s):
     return re.sub(r'\W', '_', s)
 
 
+def ppretty(value, htchar=' ', lfchar='\n', indent=4, basic_offset=0):
+    '''Format string of dictionaries, lists and tuples
+
+    :arg value: The value to be formatted.
+    :arg htchar: Horizontal-tab character.
+    :arg lfchar: Linefeed character.
+    :arg indent: Number of htchar characters for every indentation level.
+    :arg basic_offset: Basic offset for the representation, any additional
+        indentation space is added to the ``basic_offset``.
+    :returns: a formatted string of the ``value``.
+    '''
+
+    nlch = lfchar + htchar * indent * (basic_offset + 1)
+    if isinstance(value, tuple):
+        if value == ():
+            return '()'
+
+        items = [
+            nlch + ppretty(item, htchar, lfchar, indent, basic_offset + 1)
+            for item in value
+        ]
+        return '(%s)' % (','.join(items) + lfchar +
+                         htchar * indent * basic_offset)
+    elif isinstance(value, list):
+        if value == []:
+            return '[]'
+
+        items = [
+            nlch + ppretty(item, htchar, lfchar, indent, basic_offset + 1)
+            for item in value
+        ]
+        return '[%s]' % (','.join(items) + lfchar +
+                         htchar * indent * basic_offset)
+    elif isinstance(value, dict):
+        if value == {}:
+            return '{}'
+
+        items = [
+            nlch + repr(key) + ': ' +
+            ppretty(value[key], htchar, lfchar, indent, basic_offset + 1)
+            for key in value
+        ]
+        return '{%s}' % (','.join(items) + lfchar +
+                         htchar * indent * basic_offset)
+    elif isinstance(value, set):
+        if value == set():
+            return 'set()'
+
+        items = [
+            nlch + ppretty(item, htchar, lfchar, indent, basic_offset + 1)
+            for item in value
+        ]
+        return '{%s}' % (','.join(items) + lfchar +
+                         htchar * indent * basic_offset)
+    else:
+        return repr(value)
+
+
 class ScopedDict(UserDict):
     '''This is a special dict that imposes scopes on its keys.
 

schemas/config.json

@@ -147,7 +147,7 @@
                                 "descr": {"type": "string"},
                                 "scheduler": {
                                     "type": "string",
-                                    "enum": ["local", "pbs", "slurm", "squeue"]
+                                    "enum": ["local", "pbs", "slurm", "squeue", "torque"]
                                 },
                                 "launcher": {
                                     "type": "string",
@@ -310,7 +310,7 @@
         "modes": {
             "type": "array",
             "items": {
-            "type": "object",
+                "type": "object",
                 "properties": {
                     "name": {"type": "string"},
                     "options": {
@@ -350,13 +350,13 @@
         "general/check_search_recursive": "true",
         "general/target_systems": ["*"],
         "perf_logging/target_systems": ["*"],
-        "logging/handlers/level": "DEBUG",
+        "logging/handlers/level": "debug",
         "logging/handlers/file/append": false,
         "logging/handlers/file/timestamp": false,
         "logging/handlers/stream/name": "stdout",
         "logging/handlers/syslog/socktype": "udp",
         "logging/handlers/syslog/facility": "user",
-        "logging/level": "INFO",
+        "logging/level": "info",
         "logging/target_systems": ["*"],
         "modes/target_systems": ["*"],
         "schedulers/job_submit_timeout": 60,

schemas/settings.py

@@ -68,7 +68,7 @@
         },
         {
             'name': 'sys0',
-            'descr': 'System for testing check dependencies',
+            'descr': 'System for checking test dependencies',
             'hostnames': [r'sys\d+'],
             'partitions': [
                 {

tools/convert_config.py

@@ -0,0 +1,32 @@
+# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
+# ReFrame Project Developers. See the top-level LICENSE file for details.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+import os
+import sys
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+
+import reframe.core.config as config    # noqa: F401, F403
+
+
+if __name__ == '__main__':
+    try:
+        old_config = sys.argv[1]
+    except IndexError:
+        print(f'{sys.argv[0]}: too few arguments', file=sys.stderr)
+        print(f'Usage: {sys.argv[0]} OLD_CONFIG_FILE', file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        new_config = config.convert_old_config(old_config)
+    except Exception as e:
+        print(f'{sys.argv[0]}: could not convert file: {e}',
+              file=sys.stderr)
+        sys.exit(1)
+
+    print(
+        f"Conversion successful! "
+        f"Please find the converted file at '{new_config}'."
+    )

unittests/resources/settings.py

@@ -66,7 +66,7 @@ class ReframeSettings:
             },
             'sys0': {
                 # System used for dependency checking
-                'descr': 'System for test dependencies unit tests',
+                'descr': 'System for checking test dependencies',
                 'hostnames': [r'sys\d+'],
                 'partitions': {
                     'p0': {