Merge branch 'master' into bugfix/autodetect_modules

Author: Vasileios Karakasis

cscs-checks/apps/greasy/greasy_check.py

@@ -16,79 +16,76 @@ def to_seconds(str):
             datetime.strptime('00:00:00', '%H:%M:%S')).total_seconds()
 
 
-@rfm.parameterized_test(
-    ['serial',     'gpu', 24, 12, 1, 1],
-    ['serial',     'mc',  72, 36, 1, 1],
-    ['openmp',     'gpu', 24,  3, 1, 4],
-    ['openmp',     'mc',  72,  9, 1, 4],
-    ['mpi',        'gpu', 24,  4, 3, 1],
-    ['mpi',        'mc',  72, 12, 3, 1],
-    ['mpi+openmp', 'gpu', 24,  3, 2, 2],
-    ['mpi+openmp', 'mc',  72,  6, 3, 2]
-)
+@rfm.simple_test
 class GREASYCheck(rfm.RegressionTest):
-    def __init__(self, variant, partition, num_greasy_tasks, nworkes_per_node,
-                 nranks_per_worker, ncpus_per_worker):
-        self.valid_systems = ['daint:' + partition, 'dom:' + partition]
+    configuration = parameter([('serial', 'gpu', 24, 12, 1, 1),
+                               ('serial', 'mc',  72, 36, 1, 1),
+                               ('openmp', 'gpu', 24,  3, 1, 4),
+                               ('openmp', 'mc',  72,  9, 1, 4),
+                               ('mpi', 'gpu', 24,  4, 3, 1),
+                               ('mpi', 'mc',  72, 12, 3, 1),
+                               ('mpi+openmp', 'gpu', 24,  3, 2, 2),
+                               ('mpi+openmp', 'mc',  72,  6, 3, 2)])
+    variant = variable(str)
+    partition = variable(str)
+    num_greasy_tasks = variable(int)
+    workers_per_node = variable(int)
+    ranks_per_worker = variable(int)
+    cpus_per_worker = variable(int)
+    valid_prog_environs = ['PrgEnv-gnu']
+    sourcepath = 'tasks_mpi_openmp.c'
+    build_system = 'SingleSource'
+    executable = 'tasks_mpi_openmp.x'
+    tasks_file = variable(str, value='tasks.txt')
+    greasy_logfile = variable(str, value='greasy.log')
+    nnodes = variable(int, value=2)
 
-        self.valid_prog_environs = ['PrgEnv-gnu']
-        self.sourcepath = 'tasks_mpi_openmp.c'
-        self.build_system = 'SingleSource'
+    # sleep enough time to distinguish if the files are running in parallel
+    # or not
+    sleep_time = variable(int, value=60)
+    use_multithreading = False
+    modules = ['GREASY']
+    maintainers = ['VH', 'SK']
+    tags = {'production'}
 
-        # sleep enough time to distinguish if the files are running in parallel
-        # or not
-        self.sleep_time = 60
+    @run_after('init')
+    def unpack_configuration_parameter(self):
+        self.variant, self.partition = self.configuration[0:2]
+        self.num_greasy_tasks, self.workers_per_node = self.configuration[2:4]
+        self.ranks_per_worker, self.cpus_per_worker = self.configuration[4:6]
+
+    @run_after('init')
+    def set_valid_systems(self):
+        self.valid_systems = [f'daint:{self.partition}',
+                              f'dom:{self.partition}']
+
+    @run_before('compile')
+    def setup_build_system(self):
         self.build_system.cflags = [f'-DSLEEP_TIME={self.sleep_time:d}']
-        self.variant = variant
-        if variant == 'openmp':
+        if self.variant == 'openmp':
             self.build_system.cflags += ['-fopenmp']
-        elif variant == 'mpi':
+        elif self.variant == 'mpi':
             self.build_system.cflags += ['-D_MPI']
-        elif variant == 'mpi+openmp':
+        elif self.variant == 'mpi+openmp':
             self.build_system.cflags += ['-fopenmp', '-D_MPI']
 
-        self.executable = 'tasks_mpi_openmp.x'
-        self.tasks_file = 'tasks.txt'
+    @run_before('run')
+    def setup_greasy_run(self):
         self.executable_opts = [self.tasks_file]
-        self.greasy_logfile = 'greasy.log'
         self.keep_files = [self.tasks_file, self.greasy_logfile]
-        nnodes = 2
-        self.use_multithreading = False
-        self.num_greasy_tasks = num_greasy_tasks
-        self.nworkes_per_node = nworkes_per_node
-        self.nranks_per_worker = nranks_per_worker
-        self.num_tasks_per_node = nranks_per_worker * nworkes_per_node
-        self.num_tasks = self.num_tasks_per_node * nnodes
-        self.num_cpus_per_task = ncpus_per_worker
-        self.sanity_patterns = self.eval_sanity()
+        self.num_tasks_per_node = self.ranks_per_worker * self.workers_per_node
+        self.num_tasks = self.num_tasks_per_node * self.nnodes
+        self.num_cpus_per_task = self.cpus_per_worker
 
-        # Reference value is system agnostic
-        # Adding 10 secs of slowdown per greasy tasks
-        # this is to compensate for whenever the systems are full and srun gets
-        # slightly slower
-        refperf = (
-            (self.sleep_time+10)*num_greasy_tasks / nworkes_per_node / nnodes
-        )
-        self.reference = {
-            '*': {
-                'time': (refperf, None, 0.5, 's')
-            }
-        }
-        self.perf_patterns = {
-            'time': sn.extractsingle(r'Total time: (?P<perf>\S+)',
-                                     self.greasy_logfile,
-                                     'perf', to_seconds)
-        }
+    @run_before('run')
+    def set_environment_variables(self):
         # On SLURM there is no need to set OMP_NUM_THREADS if one defines
         # num_cpus_per_task, but adding for completeness and portability
         self.variables = {
             'OMP_NUM_THREADS': str(self.num_cpus_per_task),
-            'GREASY_NWORKERS_PER_NODE': str(nworkes_per_node),
+            'GREASY_NWORKERS_PER_NODE': str(self.workers_per_node),
             'GREASY_LOGFILE': self.greasy_logfile
         }
-        self.modules = ['GREASY']
-        self.maintainers = ['VH', 'SK']
-        self.tags = {'production'}
 
     @run_before('run')
     def generate_tasks_file(self):
@@ -114,7 +111,7 @@ def daint_dom_gpu_specific_workaround(self):
                 }
             }
         elif self.current_partition.fullname in ['daint:mc']:
-            if self.variant != 'serial':
+            if 'serial' not in self.variant:
                 self.extra_resources = {
                     'gres': {
                         'gres': 'craynetwork:72'
@@ -133,17 +130,19 @@ def set_launcher(self):
         # make calls to srun
         self.job.launcher = getlauncher('local')()
 
-    @sn.sanity_function
-    def eval_sanity(self):
+    @sanity_function
+    def assert_success(self):
         output_files = []
         output_files = [file for file in os.listdir(self.stagedir)
                         if file.startswith('output-')]
         num_greasy_tasks = len(output_files)
         failure_msg = (f'Requested {self.num_greasy_tasks} task(s), but '
                        f'executed only {num_greasy_tasks} tasks(s)')
-        sn.evaluate(sn.assert_eq(num_greasy_tasks, self.num_greasy_tasks,
-                                 msg=failure_msg))
-        num_tasks = sn.getattr(self, 'nranks_per_worker')
+        sn.evaluate(
+            sn.assert_eq(num_greasy_tasks, self.num_greasy_tasks,
+                         msg=failure_msg)
+        )
+        num_tasks = sn.getattr(self, 'ranks_per_worker')
         num_cpus_per_task = sn.getattr(self, 'num_cpus_per_task')
 
         def tid(match):
@@ -184,7 +183,7 @@ def num_ranks(match):
                         lambda x: sn.assert_lt(
                             rank(x), num_ranks(x),
                             msg=(f'Rank id {rank(x)} is not lower than the '
-                                 f'number of ranks {self.nranks_per_worker} '
+                                 f'number of ranks {self.ranks_per_worker} '
                                  f'in output file')
                         ), result
                     ),
@@ -217,7 +216,7 @@ def num_ranks(match):
                         lambda x: sn.assert_eq(
                             num_ranks(x), num_tasks,
                             msg=(f'Number of ranks {num_ranks(x)} is not '
-                                 f'equal to {self.nranks_per_worker} in '
+                                 f'equal to {self.ranks_per_worker} in '
                                  f'output file {output_file}')
                         ), result
                     )
@@ -234,3 +233,24 @@ def num_ranks(match):
         ))
 
         return True
+
+    @run_before('performance')
+    def set_reference(self):
+        # Reference value is system agnostic
+        # Adding 10 secs of slowdown per greasy tasks
+        # this is to compensate for whenever the systems are full and srun gets
+        # slightly slower
+        refperf = (
+            (self.sleep_time + 10) * self.num_greasy_tasks /
+            self.workers_per_node / self.nnodes
+        )
+        self.reference = {
+            '*': {
+                'time': (refperf, None, 0.5, 's')
+            }
+        }
+
+    @performance_function('s')
+    def time(self):
+        return sn.extractsingle(r'Total time: (?P<perf>\S+)',
+                                self.greasy_logfile, 'perf', to_seconds)

cscs-checks/libraries/petsc/petsc_helloworld.py

@@ -7,30 +7,39 @@
 import reframe.utility.sanity as sn
 
 
-@rfm.parameterized_test(['dynamic'], ['static'])
+@rfm.simple_test
 class PetscPoisson2DCheck(rfm.RegressionTest):
-    def __init__(self, linkage):
+    linkage = parameter(['dynamic', 'static'])
+    valid_systems = ['daint:gpu', 'daint:mc', 'dom:gpu', 'dom:mc']
+    valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu', 'PrgEnv-intel']
+    modules = ['cray-petsc']
+    build_system = 'SingleSource'
+    sourcepath = 'poisson2d.c'
+    num_tasks = 16
+    num_tasks_per_node = 8
+    executable_opts = ['-da_grid_x 4', '-da_grid_y 4', '-ksp_monitor']
+    tags = {'production', 'craype'}
+    maintainers = ['AJ', 'CB']
+
+    @run_after('init')
+    def set_description(self):
         self.descr = (f'Compile/run PETSc 2D Poisson example with cray-petsc '
-                      f'({linkage} linking)')
-        self.valid_systems = ['daint:gpu', 'daint:mc', 'dom:gpu', 'dom:mc']
-        self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu',
-                                    'PrgEnv-intel']
-        self.sourcepath = 'poisson2d.c'
-        self.modules = ['cray-petsc']
-        self.num_tasks = 16
-        self.num_tasks_per_node = 8
-        self.build_system = 'SingleSource'
+                      f'({self.linkage} linking)')
+
+    @run_before('compile')
+    def set_variables(self):
+        self.variables = {'CRAYPE_LINK_TYPE': self.linkage}
+
+    @run_before('compile')
+    def intel_workaround(self):
         # FIXME: static compilation yields a link error in case of
         # PrgEnv-intel (Cray Bug #255701) workaround use C++ compiler
-        if linkage == 'static':
+        if self.linkage == 'static':
             self.build_system.cc = 'CC'
 
-        self.variables = {'CRAYPE_LINK_TYPE': linkage}
-        self.executable_opts = ['-da_grid_x 4', '-da_grid_y 4', '-ksp_monitor']
-
+    @sanity_function
+    def assert_convergence(self):
         # Check the final residual norm for convergence
         norm = sn.extractsingle(r'\s+\d+\s+KSP Residual norm\s+(?P<norm>\S+)',
                                 self.stdout, 'norm', float, -1)
-        self.sanity_patterns = sn.assert_lt(norm, 1.0e-5)
-        self.tags = {'production', 'craype'}
-        self.maintainers = ['AJ', 'CB']
+        return sn.assert_lt(norm, 1.0e-5)

cscs-checks/microbenchmarks/gpu/gpu_burn/gpu_burn_test.py

@@ -3,10 +3,11 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-import os
 
 import reframe as rfm
 import reframe.utility.sanity as sn
+import reframe.utility.osext as osext
+from reframe.core.exceptions import SanityError
 
 from hpctestlib.microbenchmarks.gpu.gpu_burn import GpuBurn
 import cscstests.microbenchmarks.gpu.hooks as hooks
@@ -24,30 +25,29 @@ class gpu_burn_check(GpuBurn):
     num_tasks = 0
     reference = {
         'dom:gpu': {
-            'perf': (4115, -0.10, None, 'Gflop/s'),
+            'min_perf': (4115, -0.10, None, 'Gflop/s'),
         },
         'daint:gpu': {
-            'perf': (4115, -0.10, None, 'Gflop/s'),
+            'min_perf': (4115, -0.10, None, 'Gflop/s'),
         },
         'arolla:cn': {
-            'perf': (5861, -0.10, None, 'Gflop/s'),
+            'min_perf': (5861, -0.10, None, 'Gflop/s'),
         },
         'tsa:cn': {
-            'perf': (5861, -0.10, None, 'Gflop/s'),
+            'min_perf': (5861, -0.10, None, 'Gflop/s'),
         },
         'ault:amda100': {
-            'perf': (15000, -0.10, None, 'Gflop/s'),
+            'min_perf': (15000, -0.10, None, 'Gflop/s'),
         },
         'ault:amdv100': {
-            'perf': (5500, -0.10, None, 'Gflop/s'),
+            'min_perf': (5500, -0.10, None, 'Gflop/s'),
         },
         'ault:intelv100': {
-            'perf': (5500, -0.10, None, 'Gflop/s'),
+            'min_perf': (5500, -0.10, None, 'Gflop/s'),
         },
         'ault:amdvega': {
-            'perf': (3450, -0.10, None, 'Gflop/s'),
+            'min_perf': (3450, -0.10, None, 'Gflop/s'),
         },
-        '*': {'temp': (0, None, None, 'degC')}
     }
 
     maintainers = ['AJ', 'TM']
@@ -63,16 +63,25 @@ def set_num_gpus_per_node(self):
         hooks.set_num_gpus_per_node(self)
 
     @run_before('performance')
-    def report_nid_with_smallest_flops(self):
-        regex = r'\[(\S+)\] GPU\s+\d\(OK\): (\d+) GF/s'
-        rptf = os.path.join(self.stagedir, sn.evaluate(self.stdout))
-        self.nids = sn.extractall(regex, rptf, 1)
-        self.flops = sn.extractall(regex, rptf, 2, float)
+    def report_slow_nodes(self):
+        '''Report the base perf metrics and also all the slow nodes.'''
+
+        # Only report the nodes that don't meet the perf reference
+        with osext.change_dir(self.stagedir):
+            key = f'{self.current_partition.fullname}:min_perf'
+            if key in self.reference:
+                regex = r'\[(\S+)\] GPU\s+\d\(OK\): (\d+) GF/s'
+                nids = set(sn.extractall(regex, self.stdout, 1))
+
+                # Get the references
+                ref, lt, ut, *_ = self.reference[key]
+
+                # Flag the slow nodes
+                for nid in nids:
+                    try:
+                        node_perf = self.min_perf(nid)
+                        val = node_perf.evaluate(cache=True)
+                        sn.assert_reference(val, ref, lt, ut).evaluate()
+                    except SanityError:
+                        self.perf_variables[nid] = node_perf
 
-        # Find index of smallest flops and update reference dictionary to
-        # include our patched units
-        index = self.flops.evaluate().index(min(self.flops))
-        unit = f'GF/s ({self.nids[index]})'
-        for key, ref in self.reference.items():
-            if not key.endswith(':temp'):
-                self.reference[key] = (*ref[:3], unit)