Merge branch 'master' into docs/refine-docs

Author: Vasileios Karakasis

ci-scripts/ci-runner.bash

@@ -158,7 +158,7 @@ elif [ $CI_TUTORIAL -eq 1 ]; then
     # Run tutorial checks
     # Find modified or added tutorial checks
     tutorialchecks=( $(git diff origin/master...HEAD --name-only --oneline --no-merges | \
-                       grep -e '^tutorial/(?!config/).*\.py') )
+                       grep -e '^tutorial/.*\.py') )
 
     if [ ${#tutorialchecks[@]} -ne 0 ]; then
         tutorialchecks_path=""

docs/config_reference.rst

@@ -1009,7 +1009,7 @@ General Configuration
 .. js:attribute:: .general[].check_search_recursive
 
    :required: No
-   :default: ``true``
+   :default: ``false``
 
    Search directories in the `search path <#.general[].check_search_path>`__ recursively.
 

docs/migration_2_to_3.rst

@@ -24,18 +24,24 @@ As soon as it detects an old-style configuration file, it will convert it to the
    ./bin/reframe: the syntax of the configuration file 'unittests/resources/settings_old_syntax.py' is deprecated
    ./bin/reframe: configuration file has been converted to the new syntax here: '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/tmph5n8u3kf.py'
 
-Alternatively, you can convert any old configuration file using the conversion tool ``tools/convert_config.py``:
+Alternatively, you can convert any old configuration file using the conversion tool |convert_config|_:
+
+.. |convert_config| replace:: :obj:`convert-config`
+.. _convert_config: https://github.com/eth-cscs/reframe/blob/master/tools/convert-config
 
 .. code-block:: none
 
-   $ ./tools/convert-config unittests/resources/settings_old_syntax.py
-   Conversion successful! The converted file can be found at '/var/folders/h7/k7cgrdl13r996m4dmsvjq7v80000gp/T/tmpz4f6yer4.py'.
+   $ ./tools/convert-config unittests/resources/settings_old_syntax.py new_config.py
+   Conversion successful! The converted file can be found at 'new_config.py'.
 
 
 Another important change is that default locations for looking up a configuration file has changed (see `Configuring ReFrame for Your Site <configure.html>`__ for more details).
 That practically means that if you were relying on ReFrame loading your ``reframe/settings.py`` by default, this is no longer true.
 You have to move it to any of the default settings locations or set the corresponding command line option or environment variable.
 
+.. note::
+   The conversion tool will create a JSON configuration file if the extension of the target file is ``.json``.
+
 
 Automatic conversion limitations
 ================================

reframe/core/config.py

@@ -381,16 +381,14 @@ def convert_old_config(filename, newfilename=None):
         'environments': [],
         'logging': [],
     }
-    perflogdir = None
+    perflogdir = {}
     old_systems = old_config.site_configuration['systems'].items()
     for sys_name, sys_spec in old_systems:
         sys_dict = {'name': sys_name}
 
-        # FIXME: We pick the perflogdir that we first find we use it as
-        # filelog's basedir for all systems. This is not correct since
-        # per-system customizations of perflogdir will be lost
-        if perflogdir is None:
-            perflogdir = sys_spec.pop('perflogdir', None)
+        system_perflogdir = sys_spec.pop('perflogdir', None)
+        perflogdir.setdefault(system_perflogdir, [])
+        perflogdir[system_perflogdir].append(sys_name)
 
         sys_dict.update(sys_spec)
 
@@ -482,34 +480,40 @@ def convert_old_config(filename, newfilename=None):
 
                 converted['modes'].append(new_mode)
 
-    def handler_list(handler_config):
+    def handler_list(handler_config, basedir=None):
         ret = []
         for h in handler_config:
-            new_h = h
+            new_h = h.copy()
             new_h['level'] = h['level'].lower()
             if h['type'] == 'graylog':
                 # `host` and `port` attribute are converted to `address`
                 new_h['address'] = h['host']
                 if 'port' in h:
                     new_h['address'] += ':' + h['port']
-            elif h['type'] == 'filelog' and perflogdir is not None:
-                new_h['basedir'] = perflogdir
+            elif h['type'] == 'filelog' and basedir is not None:
+                new_h['basedir'] = basedir
 
             ret.append(new_h)
 
         return ret
 
-    converted['logging'].append(
-        {
-            'level': old_config.logging_config['level'].lower(),
-            'handlers': handler_list(
-                old_config.logging_config['handlers']
-            ),
-            'handlers_perflog': handler_list(
-                old_config.perf_logging_config['handlers']
-            )
-        }
-    )
+    for basedir, target_systems in perflogdir.items():
+        converted['logging'].append(
+            {
+                'level': old_config.logging_config['level'].lower(),
+                'handlers': handler_list(
+                    old_config.logging_config['handlers']
+                ),
+                'handlers_perflog': handler_list(
+                    old_config.perf_logging_config['handlers'],
+                    basedir=basedir
+                ),
+                'target_systems': target_systems
+            }
+        )
+        if basedir is None:
+            del converted['logging'][-1]['target_systems']
+
     converted['general'] = [{}]
     if hasattr(old_config, 'checks_path'):
         converted['general'][0][
@@ -528,9 +532,16 @@ def handler_list(handler_config):
                 f"by ReFrame based on '{filename}'.\n#\n\n"
                 f"site_configuration = {util.ppretty(converted)}\n")
 
+    contents = '\n'.join(l if len(l) < 80 else f'{l}  # noqa: E501'
+                         for l in contents.split('\n'))
+
     if newfilename:
         with open(newfilename, 'w') as fp:
-            fp.write(contents)
+            if newfilename.endswith('.json'):
+                json.dump(converted, fp, indent=4)
+            else:
+                fp.write(contents)
+
     else:
         with tempfile.NamedTemporaryFile(mode='w', suffix='.py',
                                          delete=False) as fp:

reframe/frontend/cli.py

@@ -332,8 +332,7 @@ def main():
     logging.getlogger().colorize = site_config.get('general/0/colorize')
     printer = PrettyPrinter()
     printer.colorize = site_config.get('general/0/colorize')
-    if options.verbose:
-        printer.inc_verbosity(options.verbose)
+    printer.inc_verbosity(site_config.get('general/0/verbose'))
 
     # Now configure ReFrame according to the user configuration file
     try:
@@ -360,6 +359,7 @@ def main():
 
     logging.getlogger().colorize = site_config.get('general/0/colorize')
     printer.colorize = site_config.get('general/0/colorize')
+    printer.inc_verbosity(site_config.get('general/0/verbose'))
     try:
         runtime.init_runtime(site_config)
     except ConfigError as e:
@@ -436,7 +436,7 @@ def print_infoline(param, value):
                    f"{os_ext.osuser() or '<unknown>'}@{socket.gethostname()}")
     print_infoline('working directory', repr(os.getcwd()))
     print_infoline('check search path',
-                   f"{'(R)' if loader.recurse else ''} "
+                   f"{'(R) ' if loader.recurse else ''}"
                    f"{':'.join(loader.load_path)!r}")
     print_infoline('stage directory', repr(rt.stage_prefix))
     print_infoline('output directory', repr(rt.output_prefix))

reframe/frontend/statistics.py

@@ -165,7 +165,7 @@ def performance_report(self):
         line_width = 78
         report_start = line_width * '='
         report_title = 'PERFORMANCE REPORT'
-        report_end = line_width * '_'
+        report_end = line_width * '-'
         report_body = []
         previous_name = ''
         previous_part = ''

schemas/config.json

@@ -386,7 +386,7 @@
         "environments/ldflags": [],
         "environments/target_systems": ["*"],
         "general/check_search_path": ["${RFM_INSTALL_PREFIX}/checks/"],
-        "general/check_search_recursive": true,
+        "general/check_search_recursive": false,
         "general/colorize": true,
         "general/ignore_check_conflicts": false,
         "general/keep_stage_files": false,

tools/convert_config.py

@@ -20,6 +20,9 @@
         print(f'{sys.argv[0]}: too few arguments', file=sys.stderr)
         print(f'Usage: {sys.argv[0]} OLD_CONFIG_FILE [NEW_CONFIG_FILE]',
               file=sys.stderr)
+        print('       Use the extension of NEW_CONFIG_FILE to specify\n'
+              '       python (.py) or json (.json) format.',
+              file=sys.stderr)
         sys.exit(1)
 
     try:

tutorial/config/settings.py

@@ -43,7 +43,7 @@
                     ],
                     'container_platforms': [
                         {
-                            'name': 'Singularity',
+                            'type': 'Singularity',
                             'modules': ['Singularity']
                         }
                     ],
@@ -64,7 +64,7 @@
                     ],
                     'container_platforms': [
                         {
-                            'name': 'Singularity',
+                            'type': 'Singularity',
                             'modules': ['Singularity']
                         }
                     ],
@@ -131,7 +131,6 @@
     'general': [
         {
             'check_search_path': ['tutorial/'],
-            'check_search_recursive': True
         }
     ]
 }

unittests/test_argparser.py

@@ -156,7 +156,7 @@ def test_option_with_config(extended_parser):
             ['--timestamp=%FT%T', '--nocolor']
         )
         options.update_config(site_config)
-        assert site_config.get('general/0/check_search_recursive') is True
+        assert site_config.get('general/0/check_search_recursive') is False
         assert site_config.get('general/0/timestamp_dirs') == '%FT%T'
         assert site_config.get('general/0/non_default_craype') is True
         assert site_config.get('systems/0/prefix') == '.'