modernize slurm test

Author: rsarm

cscs-checks/system/slurm/slurm.py

@@ -11,57 +11,54 @@
 class SlurmSimpleBaseCheck(rfm.RunOnlyRegressionTest):
     '''Base class for Slurm simple binary tests'''
 
-    def __init__(self):
-        self.valid_systems = ['daint:gpu', 'daint:mc',
-                              'dom:gpu', 'dom:mc',
-                              'arolla:cn', 'arolla:pn',
-                              'tsa:cn', 'tsa:pn',
-                              'daint:xfer', 'eiger:mc',
-                              'pilatus:mc']
-        self.valid_prog_environs = ['PrgEnv-cray']
-        self.tags = {'slurm', 'maintenance', 'ops',
-                     'production', 'single-node'}
-        self.num_tasks_per_node = 1
+    valid_systems = ['daint:gpu', 'daint:mc',
+                     'dom:gpu', 'dom:mc',
+                     'arolla:cn', 'arolla:pn',
+                     'tsa:cn', 'tsa:pn',
+                     'daint:xfer', 'eiger:mc',
+                     'pilatus:mc']
+    valid_prog_environs = ['PrgEnv-cray']
+    tags = {'slurm', 'maintenance', 'ops',
+            'production', 'single-node'}
+    num_tasks_per_node = 1
+    maintainers = ['RS', 'VH']
+
+    @run_after('init')
+    def customize_systems(self):
         if self.current_system.name in ['arolla', 'tsa']:
             self.valid_prog_environs = ['PrgEnv-gnu', 'PrgEnv-pgi']
             self.exclusive_access = True
 
-        self.maintainers = ['RS', 'VH']
-
 
 class SlurmCompiledBaseCheck(rfm.RegressionTest):
     '''Base class for Slurm tests that require compiling some code'''
 
-    def __init__(self):
-        self.valid_systems = ['daint:gpu', 'daint:mc',
-                              'dom:gpu', 'dom:mc']
-        self.valid_prog_environs = ['PrgEnv-cray']
-        self.tags = {'slurm', 'maintenance', 'ops',
-                     'production', 'single-node'}
-        self.num_tasks_per_node = 1
-
-        self.maintainers = ['RS', 'VH']
+    valid_systems = ['daint:gpu', 'daint:mc',
+                     'dom:gpu', 'dom:mc']
+    valid_prog_environs = ['PrgEnv-cray']
+    tags = {'slurm', 'maintenance', 'ops',
+            'production', 'single-node'}
+    num_tasks_per_node = 1
+    maintainers = ['RS', 'VH']
 
 
 @rfm.simple_test
 class HostnameCheck(SlurmSimpleBaseCheck):
-    def __init__(self):
-        super().__init__()
-        self.executable = '/bin/hostname'
-        self.valid_prog_environs = ['builtin']
-        self.hostname_patt = {
-            'arolla:cn': r'^arolla-cn\d{3}$',
-            'arolla:pn': r'^arolla-pp\d{3}$',
-            'tsa:cn': r'^tsa-cn\d{3}$',
-            'tsa:pn': r'^tsa-pp\d{3}$',
-            'daint:gpu': r'^nid\d{5}$',
-            'daint:mc': r'^nid\d{5}$',
-            'daint:xfer': r'^datamover\d{2}.cscs.ch$',
-            'dom:gpu': r'^nid\d{5}$',
-            'dom:mc': r'^nid\d{5}$',
-            'eiger:mc': r'^nid\d{6}$',
-            'pilatus:mc': r'^nid\d{6}$'
-        }
+    executable = '/bin/hostname'
+    valid_prog_environs = ['builtin']
+    hostname_patt = {
+        'arolla:cn': r'^arolla-cn\d{3}$',
+        'arolla:pn': r'^arolla-pp\d{3}$',
+        'tsa:cn': r'^tsa-cn\d{3}$',
+        'tsa:pn': r'^tsa-pp\d{3}$',
+        'daint:gpu': r'^nid\d{5}$',
+        'daint:mc': r'^nid\d{5}$',
+        'daint:xfer': r'^datamover\d{2}.cscs.ch$',
+        'dom:gpu': r'^nid\d{5}$',
+        'dom:mc': r'^nid\d{5}$',
+        'eiger:mc': r'^nid\d{6}$',
+        'pilatus:mc': r'^nid\d{6}$'
+    }
 
     @run_before('sanity')
     def set_sanity_patterns(self):
@@ -74,45 +71,48 @@ def set_sanity_patterns(self):
 
 @rfm.simple_test
 class EnvironmentVariableCheck(SlurmSimpleBaseCheck):
-    def __init__(self):
-        super().__init__()
-        self.num_tasks = 2
-        self.valid_systems = ['daint:gpu', 'daint:mc',
-                              'dom:gpu', 'dom:mc',
-                              'arolla:cn', 'arolla:pn',
-                              'tsa:cn', 'tsa:pn',
-                              'eiger:mc', 'pilatus:mc']
-        self.executable = '/bin/echo'
-        self.executable_opts = ['$MY_VAR']
-        self.variables = {'MY_VAR': 'TEST123456!'}
-        self.tags.remove('single-node')
+    num_tasks = 2
+    valid_systems = ['daint:gpu', 'daint:mc',
+                     'dom:gpu', 'dom:mc',
+                     'arolla:cn', 'arolla:pn',
+                     'tsa:cn', 'tsa:pn',
+                     'eiger:mc', 'pilatus:mc']
+    executable = '/bin/echo'
+    executable_opts = ['$MY_VAR']
+    variables = {'MY_VAR': 'TEST123456!'}
+    tags.remove('single-node')
+
+    @sanity_function
+    def assert_num_tasks(self):
         num_matches = sn.count(sn.findall(r'TEST123456!', self.stdout))
-        self.sanity_patterns = sn.assert_eq(self.num_tasks, num_matches)
+        return sn.assert_eq(self.num_tasks, num_matches)
 
 
 @rfm.simple_test
 class RequiredConstraintCheck(SlurmSimpleBaseCheck):
-    def __init__(self):
-        super().__init__()
-        self.valid_systems = ['daint:login', 'dom:login']
-        self.executable = 'srun'
-        self.executable_opts = ['-A', osext.osgroup(), 'hostname']
-        self.sanity_patterns = sn.assert_found(
+    valid_systems = ['daint:login', 'dom:login']
+    executable = 'srun'
+    executable_opts = ['-A', osext.osgroup(), 'hostname']
+
+    @sanity_function
+    def assert_found_missing_constraint(self):
+        return sn.assert_found(
             r'ERROR: you must specify -C with one of the following: mc,gpu',
             self.stderr
         )
 
 
 @rfm.simple_test
 class RequestLargeMemoryNodeCheck(SlurmSimpleBaseCheck):
-    def __init__(self):
-        super().__init__()
-        self.valid_systems = ['daint:mc']
-        self.executable = '/usr/bin/free'
-        self.executable_opts = ['-h']
+    valid_systems = ['daint:mc']
+    executable = '/usr/bin/free'
+    executable_opts = ['-h']
+
+    @sanity_function
+    def assert_memory_is_bounded(self):
         mem_obtained = sn.extractsingle(r'Mem:\s+(?P<mem>\S+)G',
                                         self.stdout, 'mem', float)
-        self.sanity_patterns = sn.assert_bounded(mem_obtained, 122.0, 128.0)
+        return sn.assert_bounded(mem_obtained, 122.0, 128.0)
 
     @run_before('run')
     def set_memory_limit(self):
@@ -121,55 +121,57 @@ def set_memory_limit(self):
 
 @rfm.simple_test
 class DefaultRequestGPU(SlurmSimpleBaseCheck):
-    def __init__(self):
-        super().__init__()
-        self.valid_systems = ['daint:gpu', 'dom:gpu',
-                              'arolla:cn', 'tsa:cn']
-        self.executable = 'nvidia-smi'
-        self.sanity_patterns = sn.assert_found(
-            r'NVIDIA-SMI.*Driver Version.*', self.stdout)
+    valid_systems = ['daint:gpu', 'dom:gpu',
+                     'arolla:cn', 'tsa:cn']
+    executable = 'nvidia-smi'
+
+    @sanity_function
+    def asser_found_nvidia_driver_version(self):
+        return sn.assert_found(r'NVIDIA-SMI.*Driver Version.*',
+                               self.stdout)
 
 
 @rfm.simple_test
 class DefaultRequestGPUSetsGRES(SlurmSimpleBaseCheck):
-    def __init__(self):
-        super().__init__()
-        self.valid_systems = ['daint:gpu', 'dom:gpu']
-        self.executable = 'scontrol show job ${SLURM_JOB_ID}'
-        self.sanity_patterns = sn.assert_found(
+    valid_systems = ['daint:gpu', 'dom:gpu']
+    executable = 'scontrol show job ${SLURM_JOB_ID}'
+
+    @sanity_function
+    def assert_found_resources(self):
+        return sn.assert_found(
             r'.*(TresPerNode|Gres)=.*gpu:1.*', self.stdout)
 
 
 @rfm.simple_test
 class DefaultRequestMC(SlurmSimpleBaseCheck):
-    def __init__(self):
-        super().__init__()
-        self.valid_systems = ['daint:mc', 'dom:mc']
-        # This is a basic test that should return the number of CPUs on the
-        # system which, on a MC node should be 72
-        self.executable = 'lscpu -p |grep -v "^#" -c'
-        self.sanity_patterns = sn.assert_found(r'72', self.stdout)
+    valid_systems = ['daint:mc', 'dom:mc']
+    # This is a basic test that should return the number of CPUs on the
+    # system which, on a MC node should be 72
+    executable = 'lscpu -p |grep -v "^#" -c'
+
+    @sanity_function
+    def assert_found_num_cpus(self):
+        return sn.assert_found(r'72', self.stdout)
 
 
 @rfm.simple_test
 class ConstraintRequestCabinetGrouping(SlurmSimpleBaseCheck):
-    def __init__(self):
-        super().__init__()
-        self.valid_systems = ['daint:gpu', 'daint:mc',
-                              'dom:gpu', 'dom:mc']
-        self.executable = 'cat /proc/cray_xt/cname'
-        self.cabinets = {
-            'daint:gpu': 'c0-1',
-            'daint:mc': 'c1-0',
-
-            # Numbering is inverse in Dom
-            'dom:gpu': 'c0-0',
-            'dom:mc': 'c0-1',
-        }
+    valid_systems = ['daint:gpu', 'daint:mc',
+                     'dom:gpu', 'dom:mc']
+    executable = 'cat /proc/cray_xt/cname'
+    cabinets = {
+        'daint:gpu': 'c0-1',
+        'daint:mc': 'c1-0',
+        # Numbering is inverse in Dom
+        'dom:gpu': 'c0-0',
+        'dom:mc': 'c0-1',
+    }
 
+    @sanity_function
+    def assert_found_cabinet(self):
         # We choose a default pattern that will cause assert_found() to fail
         cabinet = self.cabinets.get(self.current_system.name, r'$^')
-        self.sanity_patterns = sn.assert_found(fr'{cabinet}.*', self.stdout)
+        return sn.assert_found(fr'{cabinet}.*', self.stdout)
 
     @run_before('run')
     def set_slurm_constraint(self):
@@ -180,16 +182,16 @@ def set_slurm_constraint(self):
 
 @rfm.simple_test
 class MemoryOverconsumptionCheck(SlurmCompiledBaseCheck):
-    def __init__(self):
-        super().__init__()
-        self.time_limit = '1m'
-        self.valid_systems += ['eiger:mc', 'pilatus:mc']
-        self.sourcepath = 'eatmemory.c'
-        self.tags.add('mem')
-        self.executable_opts = ['4000M']
-        self.sanity_patterns = sn.assert_found(
-            r'(exceeded memory limit)|(Out Of Memory)', self.stderr
-        )
+    time_limit = '1m'
+    valid_systems += ['eiger:mc', 'pilatus:mc']
+    tags.add('mem')
+    sourcepath = 'eatmemory.c'
+    executable_opts = ['4000M']
+
+    @sanity_function
+    def assert_found_exceeded_memory(self):
+        return sn.assert_found(r'(exceeded memory limit)|(Out Of Memory)',
+                               self.stderr)
 
     @run_before('run')
     def set_memory_limit(self):
@@ -198,25 +200,27 @@ def set_memory_limit(self):
 
 @rfm.simple_test
 class MemoryOverconsumptionMpiCheck(SlurmCompiledBaseCheck):
-    def __init__(self):
-        super().__init__()
-        self.maintainers = ['JG']
-        self.valid_systems += ['eiger:mc', 'pilatus:mc']
-        self.time_limit = '5m'
-        self.sourcepath = 'eatmemory_mpi.c'
-        self.tags.add('mem')
-        self.executable_opts = ['100%']
-        self.sanity_patterns = sn.assert_found(r'(oom-kill)|(Killed)',
-                                               self.stderr)
-        # {{{ perf
+    maintainers = ['JG']
+    valid_systems += ['eiger:mc', 'pilatus:mc']
+    time_limit = '5m'
+    sourcepath = 'eatmemory_mpi.c'
+    tags.add('mem')
+    executable_opts = ['100%']
+
+    @sanity_function
+    def assert_found_oom(self):
+        return sn.assert_found(r'(oom-kill)|(Killed)',
+                               self.stderr)
+
+    @run_before('performance')
+    def set_perf_patterns(self):
         regex = (r'^Eating \d+ MB\/mpi \*\d+mpi = -\d+ MB memory from \/proc\/'
                  r'meminfo: total: \d+ GB, free: \d+ GB, avail: \d+ GB, using:'
                  r' (\d+) GB')
         self.perf_patterns = {
             'max_cn_memory': sn.getattr(self, 'reference_meminfo'),
-            'max_allocated_memory': sn.max(
-                sn.extractall(regex, self.stdout, 1, int)
-            ),
+            'max_allocated_memory': sn.max(sn.extractall(regex, self.stdout,
+                                                         1, int)),
         }
         no_limit = (0, None, None, 'GB')
         self.reference = {
@@ -227,9 +231,7 @@ def __init__(self):
                 ),
             }
         }
-        # }}}
 
-    # {{{ hooks
     @run_before('run')
     def set_tasks(self):
         tasks_per_node = {
@@ -244,10 +246,9 @@ def set_tasks(self):
         self.num_tasks_per_node = tasks_per_node[partname]
         self.num_tasks = self.num_tasks_per_node
         self.job.launcher.options = ['-u']
-    # }}}
 
     @property
-    @sn.sanity_function
+    @deferrable
     def reference_meminfo(self):
         reference_meminfo = {
             'dom:gpu': 62,
@@ -285,9 +286,12 @@ class slurm_response_check(rfm.RunOnlyRegressionTest):
     def set_exec_opts(self):
         self.executable_opts = [self.command]
 
-    @run_before('sanity')
-    def set_sanity(self):
-        self.sanity_patterns = sn.assert_eq(self.job.exitcode, 0)
+    @sanity_function
+    def assert_exitcode_zero(self):
+        return sn.assert_eq(self.job.exitcode, 0)
+
+    @run_before('performance')
+    def set_perf_patterns(self):
         self.perf_patterns = {
             'real_time': sn.extractsingle(r'real (?P<real_time>\S+)',
                                           self.stderr, 'real_time', float)