Merge branch 'master' into nodelist

Author: Vasileios Karakasis

README.md

@@ -13,7 +13,7 @@
 [![Slack](https://reframe-slack.herokuapp.com/badge.svg)](https://reframe-slack.herokuapp.com/)<br/>
 [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause)
 [![DOI](https://zenodo.org/badge/89384186.svg)](https://zenodo.org/badge/latestdoi/89384186)<br/>
-![Twitter Follow](https://img.shields.io/twitter/follow/ReFrameHPC?style=social)
+[![Twitter Follow](https://img.shields.io/twitter/follow/ReFrameHPC?style=social)](https://twitter.com/ReFrameHPC)
 
 # ReFrame in a Nutshell
 

cscs-checks/system/slurm/slurm.py

@@ -257,3 +257,36 @@ def reference_meminfo(self):
             'pilatus:mc': 250,
         }
         return reference_meminfo[self.current_partition.fullname]
+
+
+@rfm.simple_test
+class slurm_response_check(rfm.RunOnlyRegressionTest):
+    command = parameter(['squeue', 'sacct'])
+    descr = 'Slurm command test'
+    valid_systems = ['daint:login', 'dom:login']
+    valid_prog_environs = ['builtin']
+    num_tasks = 1
+    num_tasks_per_node = 1
+    reference = {
+        'squeue': {
+            'real_time': (0.02, None, 0.1, 's')
+        },
+        'sacct': {
+            'real_time': (0.1, None, 0.1, 's')
+        }
+    }
+    executable = 'time -p'
+    tags = {'diagnostic', 'health'}
+    maintainers = ['CB', 'VH']
+
+    @rfm.run_before('run')
+    def set_exec_opts(self):
+        self.executable_opts = [self.command]
+
+    @rfm.run_before('sanity')
+    def set_sanity(self):
+        self.sanity_patterns = sn.assert_eq(self.job.exitcode, 0)
+        self.perf_patterns = {
+            'real_time': sn.extractsingle(r'real (?P<real_time>\S+)',
+                                          self.stderr, 'real_time', float)
+        }

docs/_static/img/favicon.png

@@ -319,7 +319,7 @@ def finalize(self):
                 jsonext.dump(self.check, fp, indent=2)
         except OSError as e:
             logging.getlogger().warning(
-                f'could not dump test case {self.case}: {e}'
+                f'could not dump test case {self.testcase}: {e}'
             )
 
         self._current_stage = 'finalize'

reframe/frontend/executors/__init__.py

@@ -14,7 +14,7 @@
 
 import argparse                         # noqa: F401, F403
 import pytest                           # noqa: F401, F403
-import unittests.fixtures as fixtures   # noqa: F401, F403
+import unittests.utility as test_util   # noqa: F401, F403
 
 
 if __name__ == '__main__':
@@ -39,9 +39,9 @@
         '--rfm-help', action='help', help='Print this help message and exit.'
     )
     options, rem_args = parser.parse_known_args()
-    fixtures.USER_CONFIG_FILE = options.rfm_user_config
-    fixtures.USER_SYSTEM = options.rfm_user_system
-    fixtures.init_runtime()
+    test_util.USER_CONFIG_FILE = options.rfm_user_config
+    test_util.USER_SYSTEM = options.rfm_user_system
+    test_util.init_runtime()
 
     # If no positional argument is specified, use the `unittests` directory,
     # so as to avoid any automatic discovery of random unit tests from the

test_reframe.py

@@ -0,0 +1,76 @@
+# Copyright 2016-2021 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
+# ReFrame Project Developers. See the top-level LICENSE file for details.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+#
+# unittests/conftest.py -- pytest fixtures used in multiple unit tests
+#
+
+import contextlib
+import pytest
+
+import reframe.core.runtime as rt
+
+from .utility import TEST_CONFIG_FILE
+
+
+class _ExecutionContext:
+    def __init__(self, config_file=TEST_CONFIG_FILE,
+                 system=None, options=None):
+        self.config_file = config_file
+        self.system = system
+        self.options = options
+        self.__ctx = None
+
+    def started(self):
+        return self.__ctx is not None
+
+    def start(self):
+        self.__ctx = self._make_rt()
+        next(self.__ctx)
+
+    def _make_rt(self):
+        with rt.temp_runtime(self.config_file, self.system, self.options):
+            yield
+
+    def shutdown(self):
+        with contextlib.suppress(StopIteration):
+            next(self.__ctx)
+
+
+@pytest.fixture
+def make_exec_ctx(tmp_path):
+    '''Fixture to create a temporary execution context for the framework.'''
+
+    ctx = _ExecutionContext()
+
+    def _make_exec_ctx(config_file=TEST_CONFIG_FILE,
+                       system=None, options=None):
+        ctx.config_file = config_file
+        ctx.system = system
+        ctx.options = options or {}
+        ctx.options.update({'systems/prefix': str(tmp_path)})
+        ctx.start()
+        return ctx
+
+    yield _make_exec_ctx
+
+    # The execution context may have not been started, in case the test is
+    # skipped; so skip the shutdown
+    if ctx.started():
+        ctx.shutdown()
+
+
+@pytest.fixture
+def make_exec_ctx_g(make_exec_ctx):
+    '''Same as ``make_exec_ctx_g`` except that it is a generator.
+
+    You should use this fixture if you want to pass it to ``yield from``
+    expressions.
+    '''
+    def _make_exec_ctx(*args, **kwargs):
+        ctx = make_exec_ctx(*args, **kwargs)
+        yield ctx
+
+    yield _make_exec_ctx

unittests/conftest.py

@@ -3,4 +3,4 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-import _foo # noqa: F401
+import _foo  # noqa: F401

unittests/resources/checks/bad/notacheck.py

@@ -6,5 +6,5 @@
 #
 # module to test reframe's loader with relative imports
 #
-from .. import hellocheck          # noqa: F401
-from ..hellocheck import HelloTest # noqa: F401
+from .. import hellocheck           # noqa: F401
+from ..hellocheck import HelloTest  # noqa: F401

unittests/resources/checks/bad/relative_imports.py

@@ -9,6 +9,7 @@
 
 import os
 import signal
+import sys
 import time
 
 import reframe as rfm

unittests/resources/checks/frontend_checks.py

@@ -6,14 +6,19 @@
 import pytest
 
 import reframe.core.runtime as rt
-import unittests.fixtures as fixtures
+import unittests.utility as test_util
 from reframe.frontend.argparse import ArgumentParser
 
 
 @pytest.fixture
-def argparser():
-    with rt.temp_runtime(fixtures.TEST_CONFIG_FILE):
-        return ArgumentParser()
+def default_exec_ctx(make_exec_ctx_g):
+    yield from make_exec_ctx_g(test_util.BUILTIN_CONFIG_FILE)
+
+
+@pytest.fixture
+def argparser(make_exec_ctx):
+    make_exec_ctx()
+    return ArgumentParser()
 
 
 @pytest.fixture
@@ -122,59 +127,56 @@ def extended_parser():
     return parser
 
 
-def test_option_precedence(extended_parser):
-    with rt.temp_runtime(fixtures.BUILTIN_CONFIG_FILE):
-        with rt.temp_environment(variables={
-                'RFM_TIMESTAMP': '%F',
-                'RFM_NON_DEFAULT_CRAYPE': 'yes',
-                'RFM_MODULES_PRELOAD': 'a,b,c',
-                'RFM_CHECK_SEARCH_PATH': 'x:y:z'
-
-        }):
-            options = extended_parser.parse_args(
-                ['--timestamp=%FT%T', '--nocolor']
-            )
-            assert options.recursive is None
-            assert options.timestamp == '%FT%T'
-            assert options.non_default_craype is True
-            assert options.config_file is None
-            assert options.prefix is None
-            assert options.stagedir == '/foo'
-            assert options.module == ['a', 'b', 'c']
-            assert options.check_path == ['x', 'y', 'z']
-            assert options.colorize is False
-
-
-def test_option_with_config(extended_parser):
-    with rt.temp_runtime(fixtures.BUILTIN_CONFIG_FILE):
-        with rt.temp_environment(variables={
-                'RFM_TIMESTAMP': '%F',
-                'RFM_NON_DEFAULT_CRAYPE': 'yes',
-                'RFM_MODULES_PRELOAD': 'a,b,c',
-                'RFM_KEEP_STAGE_FILES': 'no'
-        }):
-            site_config = rt.runtime().site_config
-            options = extended_parser.parse_args(
-                ['--timestamp=%FT%T', '--nocolor']
-            )
-            options.update_config(site_config)
-            assert site_config.get('general/0/check_search_recursive') is False
-            assert site_config.get('general/0/timestamp_dirs') == '%FT%T'
-            assert site_config.get('general/0/non_default_craype') is True
-            assert site_config.get('systems/0/prefix') == '.'
-            assert site_config.get('general/0/colorize') is False
-            assert site_config.get('general/0/keep_stage_files') is False
-
-            # Defaults specified in parser override those in configuration file
-            assert site_config.get('systems/0/stagedir') == '/foo'
-
-
-def test_option_envvar_conversion_error(extended_parser):
-    with rt.temp_runtime(fixtures.BUILTIN_CONFIG_FILE):
-        with rt.temp_environment(variables={
-                'RFM_NON_DEFAULT_CRAYPE': 'foo',
-        }):
-            site_config = rt.runtime().site_config
-            options = extended_parser.parse_args(['--nocolor'])
-            errors = options.update_config(site_config)
-            assert len(errors) == 1
+def test_option_precedence(default_exec_ctx, extended_parser):
+    with rt.temp_environment(variables={
+            'RFM_TIMESTAMP': '%F',
+            'RFM_NON_DEFAULT_CRAYPE': 'yes',
+            'RFM_MODULES_PRELOAD': 'a,b,c',
+            'RFM_CHECK_SEARCH_PATH': 'x:y:z'
+
+    }):
+        options = extended_parser.parse_args(
+            ['--timestamp=%FT%T', '--nocolor']
+        )
+        assert options.recursive is None
+        assert options.timestamp == '%FT%T'
+        assert options.non_default_craype is True
+        assert options.config_file is None
+        assert options.prefix is None
+        assert options.stagedir == '/foo'
+        assert options.module == ['a', 'b', 'c']
+        assert options.check_path == ['x', 'y', 'z']
+        assert options.colorize is False
+
+
+def test_option_with_config(default_exec_ctx, extended_parser, tmp_path):
+    with rt.temp_environment(variables={
+            'RFM_TIMESTAMP': '%F',
+            'RFM_NON_DEFAULT_CRAYPE': 'yes',
+            'RFM_MODULES_PRELOAD': 'a,b,c',
+            'RFM_KEEP_STAGE_FILES': 'no'
+    }):
+        site_config = rt.runtime().site_config
+        options = extended_parser.parse_args(
+            ['--timestamp=%FT%T', '--nocolor']
+        )
+        options.update_config(site_config)
+        assert site_config.get('general/0/check_search_recursive') is False
+        assert site_config.get('general/0/timestamp_dirs') == '%FT%T'
+        assert site_config.get('general/0/non_default_craype') is True
+        assert site_config.get('systems/0/prefix') == str(tmp_path)
+        assert site_config.get('general/0/colorize') is False
+        assert site_config.get('general/0/keep_stage_files') is False
+
+        # Defaults specified in parser override those in configuration file
+        assert site_config.get('systems/0/stagedir') == '/foo'
+
+
+def test_option_envvar_conversion_error(default_exec_ctx, extended_parser):
+    with rt.temp_environment(variables={
+            'RFM_NON_DEFAULT_CRAYPE': 'foo',
+    }):
+        site_config = rt.runtime().site_config
+        options = extended_parser.parse_args(['--nocolor'])
+        errors = options.update_config(site_config)
+        assert len(errors) == 1