Merge branch 'master' into tutorial/eb

Author: Vasileios Karakasis

ci-scripts/ci-runner.bash

@@ -125,10 +125,7 @@ if [ "X${MODULEUSE}" != "X" ]; then
     module use ${MODULEUSE}
 fi
 
-parallel_opts="--workers=auto --forked"
-if [[ $(hostname) =~ tsa|uan ]]; then
-    parallel_opts=""
-fi
+parallel_opts=""
 
 # Bootstrap ReFrame
 ./bootstrap.sh

config/cscs.py

@@ -717,6 +717,7 @@
                         'cpeIntel'
                     ],
                     'max_jobs': 100,
+                    'access': ['-Cmc', f'--account={osext.osgroup()}'],
                     'resources': [
                         {
                             'name': 'switches',
@@ -753,7 +754,6 @@
                         'PrgEnv-cray',
                         'PrgEnv-gnu',
                         'PrgEnv-intel',
-                        'PrgEnv-nvidia',
                         'cpeAMD',
                         'cpeCray',
                         'cpeGNU',
@@ -781,7 +781,6 @@
                         'PrgEnv-cray',
                         'PrgEnv-gnu',
                         'PrgEnv-intel',
-                        'PrgEnv-nvidia',
                         'cpeAMD',
                         'cpeCray',
                         'cpeGNU',

docs/config_reference.rst

@@ -341,9 +341,13 @@ System Partition Configuration
    :default: ``{}``
 
    Processor information for this partition stored in a `processor info object <#processor-info>`__.
+   If not set, ReFrame will try to auto-detect this information (see :ref:`proc-autodetection` for more information).
 
    .. versionadded:: 3.5.0
 
+   .. versionchanged:: 3.7.0
+      ReFrame is now able to detect the processor information automatically.
+
 
 .. js:attribute:: .systems[].partitions[].devices
 
@@ -1201,6 +1205,28 @@ General Configuration
    The command-line option sets the configuration option to ``false``.
 
 
+.. js:attribute:: .general[].remote_detect
+
+   :required: No
+   :default: ``false``
+
+   Try to auto-detect processor information of remote partitions as well.
+   This may slow down the initialization of the framework, since it involves submitting auto-detection jobs to the remote partitions.
+   For more information on how ReFrame auto-detects processor information, you may refer to :ref:`proc-autodetection`.
+
+   .. versionadded:: 3.7.0
+
+
+.. js:attribute:: .general[].remote_workdir
+
+   :required: No
+   :default: ``"."``
+
+   The temporary directory prefix that will be used to create a fresh ReFrame clone, in order to auto-detect the processor information of a remote partition.
+
+   .. versionadded:: 3.7.0
+
+
 .. js:attribute:: .general[].ignore_check_conflicts
 
    :required: No

docs/configure.rst

@@ -397,3 +397,43 @@ Let's see some concrete examples:
      "CC"
 
   If you explicitly query a configuration value which is not defined in the configuration file, ReFrame will print its default value.
+
+
+.. _proc-autodetection:
+
+Auto-detecting processor information
+------------------------------------
+
+.. versionadded:: 3.7.0
+
+.. |devices| replace:: :attr:`devices`
+.. _devices: config_reference.html#.systems[].partitions[].devices
+.. |processor| replace:: :attr:`processor`
+.. _processor: config_reference.html#.systems[].partitions[].processor
+.. |detect_remote_system_topology| replace:: :attr:`detect_remote_system_topology`
+.. _detect_remote_system_topology: config_reference.html#.general[].detect_remote_system_topology
+
+ReFrame is able to detect the processor topology of both local and remote partitions automatically.
+The processor and device information are made available to the tests through the corresponding attributes of the :attr:`~reframe.core.pipeline.RegressionTest.current_partition` allowing a test to modify its behavior accordingly.
+Currently, ReFrame supports auto-detection of the local or remote processor information only.
+It does not support auto-detection of devices, in which cases users should explicitly specify this information using the |devices|_ configuration option.
+The processor information auto-detection works as follows:
+
+#. If the |processor|_ configuration is option is defined, then no auto-detection is attempted.
+
+#. If the |processor|_ configuration option is not defined, ReFrame will look for a processor configuration metadata file in ``{configdir}/_meta/{system}-{part}/processor.json`` or in ``~/.reframe/topology/{system}-{part}/processor.json`` in case of the builtin configuration file.
+   If the file is found, the topology information is loaded from there.
+   These files are generated automatically by ReFrame from previous runs.
+
+#. If the corresponding metadata files are not found, the processor information will be auto-detected.
+   If the system partition is local (i.e., ``local`` scheduler + ``local`` launcher), the processor information is auto-detected unconditionally and stored in the corresponding metadata file for this partition.
+   If the partition is remote, ReFrame will not try to auto-detect it unless the :envvar:`RFM_REMOTE_DETECT` or the |detect_remote_system_topology|_ configuration option is set.
+   In that case, the steps to auto-detect the remote processor information are the following:
+
+     a. ReFrame creates a fresh clone of itself in a temporary directory created under ``.`` by default.
+        This temporary directory prefix can be changed by setting the :envvar:`RFM_REMOTE_WORKDIR` environment variable.
+     b. ReFrame changes to that directory and launches a job that will first bootstrap the fresh clone and then run that clone with ``{launcher} ./bin/reframe --detect-host-topology=topo.json``.
+        The :option:`--detect-host-topology` option causes ReFrame to detect the topology of the current host,
+        which in this case would be the remote compute nodes.
+
+   In case of errors during auto-detection, ReFrame will simply issue a warning and continue.

docs/manpage.rst

@@ -576,6 +576,16 @@ Miscellaneous options
 
    This option can also be set using the :envvar:`RFM_SYSTEM` environment variable.
 
+.. _--detect-host-topology:
+
+.. option:: --detect-host-topology[=FILE]
+
+   Detect the local host processor topology, store it to ``FILE`` and exit.
+   If no ``FILE`` is specified, the standard output will be used.
+
+   .. versionadded:: 3.7.0
+
+
 .. option:: --failure-stats
 
    Print failure statistics at the end of the run.
@@ -698,6 +708,36 @@ Here is an alphabetical list of the environment variables recognized by ReFrame:
       ================================== ==================
 
 
+.. envvar:: RFM_REMOTE_DETECT
+
+   Auto-detect processor information of remote partitions as well.
+
+   .. table::
+      :align: left
+
+      ================================== ==================
+      Associated command line option     N/A
+      Associated configuration parameter :js:attr:`remote_detect` general configuration parameter
+      ================================== ==================
+
+.. versionadded:: 3.7.0
+
+
+.. envvar:: RFM_REMOTE_WORKDIR
+
+   The temporary directory prefix that will be used to create a fresh ReFrame clone, in order to auto-detect the processor information of a remote partition.
+
+   .. table::
+      :align: left
+
+      ================================== ==================
+      Associated command line option     N/A
+      Associated configuration parameter :js:attr:`remote_workdir` general configuration parameter
+      ================================== ==================
+
+.. versionadded:: 3.7.0
+
+
 .. envvar:: RFM_GRAYLOG_ADDRESS
 
    The address of the Graylog server to send performance logs.
@@ -920,7 +960,7 @@ Here is an alphabetical list of the environment variables recognized by ReFrame:
       :align: left
 
       ================================== ==================
-      Associated command line option     n/a
+      Associated command line option     N/A
       Associated configuration parameter :js:attr:`resolve_module_conflicts` general configuration parameter
       ================================== ==================
 

docs/requirements.txt

@@ -1,3 +1,4 @@
+archspec==0.1.2
 docutils==0.16 # https://github.com/sphinx-doc/sphinx/issues/9001
 jsonschema==3.2.0
 semver==2.13.0

reframe/core/config.py

@@ -101,6 +101,11 @@ def __getitem__(self, key):
     def __getattr__(self, attr):
         return getattr(self._pick_config(), attr)
 
+    @property
+    def schema(self):
+        '''Configuration schema'''
+        return self._schema
+
     def add_sticky_option(self, option, value):
         self._sticky_options[option] = value
 

reframe/core/decorators.py

@@ -91,6 +91,17 @@ def _validate_test(cls):
         raise ValueError(f'decorated test ({cls.__qualname__!r}) has one or '
                          f'more undefined parameters')
 
+    conditions = [VersionValidator(v) for v in cls._rfm_required_version]
+    if (cls._rfm_required_version and
+        not any(c.validate(osext.reframe_version()) for c in conditions)):
+
+        getlogger().warning(f"skipping incompatible test "
+                            f"'{cls.__qualname__}': not valid for ReFrame "
+                            f"version {osext.reframe_version().split('-')[0]}")
+        return False
+
+    return True
+
 
 def simple_test(cls):
     '''Class decorator for registering tests with ReFrame.
@@ -101,10 +112,9 @@ def simple_test(cls):
 
     .. versionadded:: 2.13
     '''
-    _validate_test(cls)
-
-    for _ in cls.param_space:
-        _register_test(cls)
+    if _validate_test(cls):
+        for _ in cls.param_space:
+            _register_test(cls)
 
     return cls
 
@@ -139,14 +149,14 @@ def parameterized_test(*inst):
     )
 
     def _do_register(cls):
-        _validate_test(cls)
-        if not cls.param_space.is_empty():
-            raise ValueError(
-                f'{cls.__qualname__!r} is already a parameterized test'
-            )
+        if _validate_test(cls):
+            if not cls.param_space.is_empty():
+                raise ValueError(
+                    f'{cls.__qualname__!r} is already a parameterized test'
+                )
 
-        for args in inst:
-            _register_test(cls, args)
+            for args in inst:
+                _register_test(cls, args)
 
         return cls
 
@@ -187,6 +197,12 @@ def required_version(*versions):
        These should be written as ``3.5.0`` and ``3.5.0-dev.0``.
 
     '''
+    warn.user_deprecation_warning(
+        "the '@required_version' decorator is deprecated; please set "
+        "the 'require_version' parameter in the class definition instead",
+        from_version='3.7.0'
+    )
+
     if not versions:
         raise ValueError('no versions specified')
 
@@ -198,8 +214,10 @@ def _skip_tests(cls):
             mod.__rfm_skip_tests = set()
 
         if not any(c.validate(osext.reframe_version()) for c in conditions):
-            getlogger().info('skipping incompatible test defined'
-                             ' in class: %s' % cls.__name__)
+            getlogger().warning(
+                f"skipping incompatible test '{cls.__qualname__}': not valid "
+                f"for ReFrame version {osext.reframe_version().split('-')[0]}"
+            )
             mod.__rfm_skip_tests.add(cls)
 
         return cls

reframe/core/meta.py

@@ -176,6 +176,7 @@ def __prepare__(metacls, name, bases, **kwargs):
         namespace['variable'] = variables.TestVar
         namespace['required'] = variables.Undefined
 
+        # Utility decorators
         def bind(fn, name=None):
             '''Directive to bind a free function to a class.
 
@@ -186,7 +187,14 @@ def bind(fn, name=None):
             namespace[inst.__name__] = inst
             return inst
 
+        def final(fn):
+            '''Indicate that a function is final and cannot be overridden.'''
+
+            fn._rfm_final = True
+            return fn
+
         namespace['bind'] = bind
+        namespace['final'] = final
 
         # Hook-related functionality
         def run_before(stage):
@@ -205,8 +213,6 @@ def run_before(stage):
 
             return hooks.attach_to('pre_' + stage)
 
-        namespace['run_before'] = run_before
-
         def run_after(stage):
             '''Decorator for attaching a test method to a pipeline stage.
 
@@ -228,6 +234,7 @@ def run_after(stage):
 
             return hooks.attach_to('post_' + stage)
 
+        namespace['run_before'] = run_before
         namespace['run_after'] = run_after
         namespace['require_deps'] = hooks.require_deps
 
@@ -259,7 +266,8 @@ class was created or even at the instance level (e.g. doing
 
         directives = [
             'parameter', 'variable', 'bind', 'run_before', 'run_after',
-            'require_deps', 'required', 'deferrable', 'sanity_function'
+            'require_deps', 'required', 'deferrable', 'sanity_function',
+            'final'
         ]
         for b in directives:
             namespace.pop(b, None)
@@ -326,13 +334,12 @@ def __init__(cls, name, bases, namespace, **kwargs):
                               if hasattr(v, '_rfm_final')}
 
         # Add the final functions from its parents
-        cls._final_methods.update(*(b._final_methods for b in bases
-                                    if hasattr(b, '_final_methods')))
+        bases_w_final = [b for b in bases if hasattr(b, '_final_methods')]
+        cls._final_methods.update(*(b._final_methods for b in bases_w_final))
 
-        if getattr(cls, '_rfm_special_test', None):
+        if getattr(cls, '_rfm_override_final', None):
             return
 
-        bases_w_final = [b for b in bases if hasattr(b, '_final_methods')]
         for v in namespace.values():
             for b in bases_w_final:
                 if callable(v) and v.__name__ in b._final_methods: