Merge branch 'master' into feature/improve-error-handling

Author: Vasileios Karakasis

config/cscs-ci.py

@@ -24,6 +24,7 @@
                 {
                     'name': 'gpu',
                     'scheduler': 'slurm',
+                    'time_limit': '10m',
                     'access': [
                         '--constraint=gpu',
                         '--partition=cscsci',
@@ -58,6 +59,7 @@
                 {
                     'name': 'slurm',
                     'scheduler': 'slurm',
+                    'time_limit': '10m',
                     'access': [
                         '--constraint=gpu',
                         f'--account={osext.osgroup()}'
@@ -80,6 +82,7 @@
                 {
                     'name': 'pbs',
                     'scheduler': 'pbs',
+                    'time_limit': '10m',
                     'access': [
                         'proc=gpu',
                         f'-A {osext.osgroup()}'
@@ -94,6 +97,7 @@
                 {
                     'name': 'torque',
                     'scheduler': 'torque',
+                    'time_limit': '10m',
                     'access': [
                         '-l proc=gpu',
                         f'-A {osext.osgroup()}'

config/cscs.py

@@ -182,6 +182,7 @@
                 },
                 {
                     'name': 'gpu',
+                    'time_limit': '10m',
                     'scheduler': 'slurm',
                     'container_platforms': [
                         {
@@ -230,6 +231,7 @@
                 {
                     'name': 'mc',
                     'scheduler': 'slurm',
+                    'time_limit': '10m',
                     'container_platforms': [
                         {
                             'type': 'Sarus',
@@ -348,6 +350,7 @@
                 {
                     'name': 'gpu',
                     'scheduler': 'slurm',
+                    'time_limit': '10m',
                     'container_platforms': [
                         {
                             'type': 'Sarus',
@@ -389,6 +392,7 @@
                 {
                     'name': 'mc',
                     'scheduler': 'slurm',
+                    'time_limit': '10m',
                     'container_platforms': [
                         {
                             'type': 'Sarus',

cscs-checks/system/nvidia/nvidia_smi_check.py

@@ -0,0 +1,50 @@
+# Copyright 2016-2021 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
+# ReFrame Project Developers. See the top-level LICENSE file for details.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+import reframe as rfm
+import reframe.utility.sanity as sn
+import reframe.utility.typecheck as typ
+
+
+@rfm.simple_test
+class nvidia_smi_check(rfm.RunOnlyRegressionTest):
+    gpu_mode = parameter(['accounting', 'compute', 'ecc'])
+    valid_systems = ['daint:gpu', 'dom:gpu']
+    valid_prog_environs = ['builtin']
+    executable = 'nvidia-smi'
+    executable_opts = ['-a', '-d']
+    num_tasks = 1
+    num_tasks_per_node = 1
+    exclusive = True
+    tags = {'maintenance', 'production'}
+    maintainers = ['VH']
+    mode_values = variable(typ.Dict[str, str], value={
+        'accounting': 'Enabled',
+        'compute': 'Exclusive_Process',
+        'ecc': 'Enabled'
+    })
+
+    @rfm.run_before('run')
+    def set_display_opt(self):
+        self.executable_opts.append(self.gpu_mode.upper())
+
+    @rfm.run_before('sanity')
+    def set_sanity(self):
+        modeval = self.mode_values[self.gpu_mode]
+        if self.gpu_mode == 'ecc':
+            patt = rf'Current\s+: {modeval}'
+        else:
+            patt = rf'{self.gpu_mode.capitalize()} Mode\s+: {modeval}'
+
+        num_gpus_detected = sn.count(sn.findall(patt, self.stdout))
+        num_gpus_all = self.num_tasks
+
+        # We can't an use f-string here, because it will misinterpret the
+        # placeholders for the sanity function message
+        errmsg = ('{0} out of {1} GPU(s) have the correct %s mode' %
+                  self.gpu_mode)
+        self.sanity_patterns = sn.assert_eq(
+            num_gpus_detected, num_gpus_all, errmsg
+        )

docs/config_reference.rst

@@ -287,6 +287,15 @@ System Partition Configuration
   A list of `environment module objects <#module-objects>`__ to be loaded before running a regression test on this partition.
 
 
+.. js:attribute:: .systems[].partitions[].time_limit
+
+   :required: No
+   :default: ``null``
+
+   The time limit for the jobs submitted on this partition.
+   When the value is ``null``, no time limit is applied.
+
+
 .. js:attribute:: .systems[].partitions[].variables
 
    :required: No

docs/regression_test_api.rst

@@ -48,7 +48,7 @@ This provides the ReFrame internals with further control over the user's input,
 In essence, these builtins exert control over the test creation, and they allow adding and/or modifying certain attributes of the regression test.
 
 
-.. py:function:: reframe.core.pipeline.RegressionTest.parameter(values=None, inherit_params=False, filter_params=None)
+.. py:function:: RegressionTest.parameter(values=None, inherit_params=False, filter_params=None)
 
   Inserts or modifies a regression test parameter.
   If a parameter with a matching name is already present in the parameter space of a parent class, the existing parameter values will be combined with those provided by this method following the inheritance behavior set by the arguments ``inherit_params`` and ``filter_params``.
@@ -67,7 +67,7 @@ In essence, these builtins exert control over the test creation, and they allow
             else:
                 do_other()
 
-  One of the most powerful features about these built-in functions is that they store their input information at the class level. 
+  One of the most powerful features about these built-in functions is that they store their input information at the class level.
   However, a parameter may only be accessed from the class instance and accessing it directly from the class body is disallowed.
   With this approach, extending or specializing an existing parametrized regression test becomes straightforward, since the test attribute additions and modifications made through built-in functions in the parent class are automatically inherited by the child test.
   For instance, continuing with the example above, one could override the :func:`__init__` method in the :class:`Foo` regression test as follows:
@@ -83,7 +83,7 @@ In essence, these builtins exert control over the test creation, and they allow
 
   Note that this built-in parameter function provides an alternative method to parameterize a test to :func:`reframe.core.decorators.parameterized_test`, and the use of both approaches in the same test is currently disallowed.
   The two main advantages of the built-in :func:`parameter` over the decorated approach reside in the parameter inheritance across classes and the handling of large parameter sets.
-  As shown in the example above, the parameters declared with the built-in :func:`parameter` are automatically carried over into derived tests through class inheritance, whereas tests using the decorated approach would have to redefine the parameters on every test. 
+  As shown in the example above, the parameters declared with the built-in :func:`parameter` are automatically carried over into derived tests through class inheritance, whereas tests using the decorated approach would have to redefine the parameters on every test.
   Similarly, parameters declared through the built-in :func:`parameter` are regarded as fully independent from each other and ReFrame will automatically generate as many tests as available parameter combinations. This is a major advantage over the decorated approach, where one would have to manually expand the parameter combinations.
   This is illustrated in the example below, consisting of a case with two parameters, each having two possible values.
 
@@ -114,11 +114,11 @@ In essence, these builtins exert control over the test creation, and they allow
      This only has an effect if used with ``inherit_params=True``.
 
 
-.. py:function:: reframe.core.pipeline.RegressionTest.variable(*types, value=None)
+.. py:function:: RegressionTest.variable(*types, value=None)
 
   Inserts a new regression test variable.
   Declaring a test variable through the :func:`variable` built-in allows for a more robust test implementation than if the variables were just defined as regular test attributes (e.g. ``self.a = 10``).
-  Using variables declared through the :func:`variable` built-in guarantees that these regression test variables will not be redeclared by any child class, while also ensuring that any values that may be assigned to such variables comply with its original declaration. 
+  Using variables declared through the :func:`variable` built-in guarantees that these regression test variables will not be redeclared by any child class, while also ensuring that any values that may be assigned to such variables comply with its original declaration.
   In essence, by using test variables, the user removes any potential test errors that might be caused by accidentally overriding a class attribute. See the example below.