Merge branch 'master' into feature/min_flex_nodes

Author: vkarak

README.md

@@ -61,4 +61,4 @@ For unsubscribing, you may send an empty message to [[email protected]
 
 ### Slack
 
-You may also reach the community through Slack at [reframetalk.slack.com](https://reframetalk.slack.com/join/signup). Currently, you may join the Slack workspace by invitation only, which you will get as soon as you subscribe to the mailing list.
+You may also reach the community through Slack [here](https://reframe-slack.herokuapp.com).

cscs-checks/microbenchmarks/kernel_latency/kernel_latency.py

@@ -0,0 +1,84 @@
+import reframe as rfm
+import reframe.utility.sanity as sn
+
+
+@rfm.required_version('>=2.16-dev0')
+@rfm.parameterized_test(['sync'], ['async'])
+class KernelLatencyTest(rfm.RegressionTest):
+    def __init__(self, kernel_version):
+        super().__init__()
+        self.sourcepath = 'kernel_latency.cu'
+        self.build_system = 'SingleSource'
+        self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn']
+        self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi']
+        self.num_tasks = 0
+        self.num_tasks_per_node = 1
+
+        if self.current_system.name in {'dom', 'daint'}:
+            self.num_gpus_per_node = 1
+            gpu_arch = '60'
+            self.modules = ['craype-accel-nvidia60']
+            self.valid_prog_environs += ['PrgEnv-gnu']
+        else:
+            self.num_gpus_per_node = 16
+            self.modules = ['craype-accel-nvidia35']
+            gpu_arch = '37'
+
+        self.build_system.cxxflags = ['-arch=compute_%s' % gpu_arch,
+                                      '-code=sm_%s' % gpu_arch, '-std=c++11']
+
+        if kernel_version == 'sync':
+            self.build_system.cppflags = ['-D SYNCKERNEL=1']
+        else:
+            self.build_system.cppflags = ['-D SYNCKERNEL=0']
+
+        self.sanity_patterns = sn.all([
+            sn.assert_eq(
+                sn.count(sn.findall(r'\[\S+\] Found \d+ gpu\(s\)',
+                                    self.stdout)),
+                self.num_tasks_assigned),
+            sn.assert_eq(
+                sn.count(sn.findall(r'\[\S+\] \[gpu \d+\] Kernel launch '
+                                    r'latency: \S+ us', self.stdout)),
+                self.num_tasks_assigned * self.num_gpus_per_node)
+        ])
+
+        self.perf_patterns = {
+            'latency': sn.max(sn.extractall(
+                r'\[\S+\] \[gpu \d+\] Kernel launch latency: '
+                r'(?P<latency>\S+) us', self.stdout, 'latency', float))
+        }
+        self.sys_reference = {
+            'sync': {
+                'dom:gpu': {
+                    'latency': (6.6, None, 0.10, 's')
+                },
+                'daint:gpu': {
+                    'latency': (6.6, None, 0.10, 'us')
+                },
+                'kesch:cn': {
+                    'latency': (12.0, None, 0.10, 'us')
+                },
+            },
+            'async': {
+                'dom:gpu': {
+                    'latency': (2.2, None, 0.10, 'us')
+                },
+                'daint:gpu': {
+                    'latency': (2.2, None, 0.10, 's')
+                },
+                'kesch:cn': {
+                    'latency': (5.7, None, 0.10, 'us')
+                },
+            },
+        }
+
+        self.reference = self.sys_reference[kernel_version]
+
+        self.maintainers = ['TM']
+        self.tags = {'benchmark', 'diagnostic'}
+
+    @property
+    @sn.sanity_function
+    def num_tasks_assigned(self):
+        return self.job.num_tasks

cscs-checks/microbenchmarks/kernel_latency/src/kernel_latency.cu

@@ -0,0 +1,59 @@
+#include <iostream>
+#include <chrono>
+#include <ratio>
+#include <unistd.h>
+#include <cuda.h>
+
+__global__ void null_kernel() {
+};
+
+int main(int argc, char* argv[]) {
+
+    char hostname[256];
+    hostname[255]='\0';
+    gethostname(hostname, 255);
+
+    cudaError_t error;
+    int gpu_count = 0;
+
+    error = cudaGetDeviceCount(&gpu_count);
+
+    if (error == cudaSuccess) {
+        if (gpu_count <= 0) {
+            std::cout << "[" << hostname << "] " << "Could not find any gpu\n";
+            return 1;
+        }
+        std::cout << "[" << hostname << "] " << "Found " << gpu_count << " gpu(s)\n";
+    }
+    else{
+        std::cout << "[" << hostname << "] " << "Error getting gpu count, exiting...\n";
+        return 1;
+    }
+
+    for (int i = 0; i < gpu_count; i++) {
+
+        cudaSetDevice(i);
+        // Single kernel launch to initialize cuda runtime
+        null_kernel<<<1, 1>>>();
+
+        auto t_start = std::chrono::system_clock::now();
+        const int kernel_count = 1000;
+
+        for (int i = 0; i < kernel_count; ++i) {
+            null_kernel<<<1, 1>>>();
+            #if SYNCKERNEL == 1
+            cudaDeviceSynchronize();
+            #endif
+        }
+
+        #if SYNCKERNEL != 1
+        cudaDeviceSynchronize();
+        #endif
+
+        auto t_end = std::chrono::system_clock::now();
+        std::cout << "[" << hostname << "] " << "[gpu " << i << "] " <<  "Kernel launch latency: " << std::chrono::duration_cast<std::chrono::duration<double, std::micro>>(t_end - t_start).count() / kernel_count << " us\n";
+    }
+
+    return 0;
+}
+

cscs-checks/prgenv/mpi.py

@@ -0,0 +1,64 @@
+import reframe as rfm
+import reframe.utility.sanity as sn
+
+
+@rfm.required_version('>=2.14')
+@rfm.parameterized_test(['single'], ['funneled'], ['serialized'], ['multiple'])
+class MpiInitTest(rfm.RegressionTest):
+    """This test checks the value returned by calling MPI_Init_thread.
+
+    Output should look the same for every prgenv (cray, gnu, intel, pgi)
+    (mpi_thread_multiple seems to be not supported):
+
+    # 'single':
+    ['mpi_thread_supported=MPI_THREAD_SINGLE
+      mpi_thread_queried=MPI_THREAD_SINGLE 0'],
+
+    # 'funneled':
+    ['mpi_thread_supported=MPI_THREAD_FUNNELED
+      mpi_thread_queried=MPI_THREAD_FUNNELED 1'],
+
+    # 'serialized':
+    ['mpi_thread_supported=MPI_THREAD_SERIALIZED
+      mpi_thread_queried=MPI_THREAD_SERIALIZED 2'],
+
+    # 'multiple':
+    ['mpi_thread_supported=MPI_THREAD_SERIALIZED
+      mpi_thread_queried=MPI_THREAD_SERIALIZED 2']
+
+    """
+
+    def __init__(self, required_thread):
+        super().__init__()
+        self.valid_systems = ['daint:gpu', 'daint:mc', 'dom:gpu', 'dom:mc']
+        self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu',
+                                    'PrgEnv-intel', 'PrgEnv-pgi']
+        self.build_system = 'SingleSource'
+        self.sourcepath = 'mpi_init_thread.cpp'
+        self.cppflags = {
+            'single':     ['-D_MPI_THREAD_SINGLE'],
+            'funneled':   ['-D_MPI_THREAD_FUNNELED'],
+            'serialized': ['-D_MPI_THREAD_SERIALIZED'],
+            'multiple':   ['-D_MPI_THREAD_MULTIPLE']
+        }
+        self.build_system.cppflags = self.cppflags[required_thread]
+        self.build_system.cppflags += ['-static']
+        self.time_limit = (0, 1, 0)
+        found_mpithread = sn.extractsingle(
+            r'^mpi_thread_required=\w+\s+mpi_thread_supported=\w+'
+            r'\s+mpi_thread_queried=\w+\s+(?P<result>\d)$',
+            self.stdout, 1, int)
+        self.mpithread_version = {
+            'single':     0,
+            'funneled':   1,
+            'serialized': 2,
+            'multiple':   2
+        }
+        self.sanity_patterns = sn.all([
+            sn.assert_found(r'tid=0 out of 1 from rank 0 out of 1',
+                            self.stdout),
+            sn.assert_eq(found_mpithread,
+                         self.mpithread_version[required_thread])
+        ])
+        self.maintainers = ['JG']
+        self.tags = {'production'}

cscs-checks/prgenv/src/mpi_init_thread.cpp

@@ -0,0 +1,82 @@
+// testing MPI_Init_thread
+#include <iostream>
+#include <stdio.h>  
+#include <mpi.h>   
+using namespace std;
+
+int main(int argc, char **argv) {
+  int rank, size, mpiversion, mpisubversion;
+  int resultlen = -1, mpi_thread_supported=-1;
+  char mpilibversion[MPI_MAX_LIBRARY_VERSION_STRING];
+
+// --------------------------------------------------------------------------
+// int MPI_Init_thread( int *argc, char ***argv, int required, int *provided )
+//
+// { MPI_THREAD_SINGLE}
+// Only one thread will execute.
+// 
+// { MPI_THREAD_FUNNELED}
+// The process may be multi-threaded, but only the main thread will make MPI calls
+// (all MPI calls are funneled to the main thread).
+// 
+// { MPI_THREAD_SERIALIZED}
+// The process may be multi-threaded, and multiple threads may make MPI calls, but
+// only one at a time: MPI calls are not made concurrently from two distinct
+// threads (all MPI calls are serialized).
+// 
+// { MPI_THREAD_MULTIPLE}
+// Multiple threads may call MPI, with no restrictions.
+// --------------------------------------------------------------------------
+
+#if defined(_MPI_THREAD_SINGLE)
+  cout << "mpi_thread_required=MPI_THREAD_SINGLE ";
+  int ev = MPI_Init_thread( 0,0, MPI_THREAD_SINGLE,     &mpi_thread_supported );
+#elif defined(_MPI_THREAD_FUNNELED)
+  cout << "mpi_thread_required=MPI_THREAD_FUNNELED ";
+  int ev = MPI_Init_thread( 0,0, MPI_THREAD_FUNNELED,   &mpi_thread_supported );
+#elif defined(_MPI_THREAD_SERIALIZED)
+  cout << "mpi_thread_required=MPI_THREAD_SERIALIZED ";
+  int ev = MPI_Init_thread( 0,0, MPI_THREAD_SERIALIZED, &mpi_thread_supported );
+#elif defined(_MPI_THREAD_MULTIPLE)
+  cout << "mpi_thread_required=MPI_THREAD_MULTIPLE ";
+  int ev = MPI_Init_thread( 0,0, MPI_THREAD_MULTIPLE,   &mpi_thread_supported );
+#else
+  cout << "mpi_thread_required=none ";
+  int ev = MPI_Init(0,0);
+#endif
+
+  switch ( mpi_thread_supported )
+    {
+        case MPI_THREAD_SINGLE:     cout << "mpi_thread_supported=MPI_THREAD_SINGLE" ;    break;
+        case MPI_THREAD_FUNNELED:   cout << "mpi_thread_supported=MPI_THREAD_FUNNELED" ;  break;
+        case MPI_THREAD_SERIALIZED: cout << "mpi_thread_supported=MPI_THREAD_SERIALIZED" ;break;
+        case MPI_THREAD_MULTIPLE:   cout << "mpi_thread_supported=MPI_THREAD_MULTIPLE" ;  break;
+        default:                    cout << "mpi_thread_supported=UNKNOWN" ;
+    }
+
+  // Return the level of thread support provided by the MPI library:
+  int mpi_thread_required=-1;
+  MPI_Query_thread( &mpi_thread_required );
+  switch ( mpi_thread_supported )
+    {
+        case MPI_THREAD_SINGLE:     cout << " mpi_thread_queried=MPI_THREAD_SINGLE "    << mpi_thread_required << std::endl; break;
+        case MPI_THREAD_FUNNELED:   cout << " mpi_thread_queried=MPI_THREAD_FUNNELED "  << mpi_thread_required << std::endl; break;
+        case MPI_THREAD_SERIALIZED: cout << " mpi_thread_queried=MPI_THREAD_SERIALIZED "<< mpi_thread_required << std::endl; break;
+        case MPI_THREAD_MULTIPLE:   cout << " mpi_thread_queried=MPI_THREAD_MULTIPLE "  << mpi_thread_required << std::endl; break;
+        default:                    cout << " mpi_thread_queried=UNKNOWN "              << mpi_thread_required << std::endl; 
+    }
+
+  MPI_Get_version( &mpiversion, &mpisubversion );
+  MPI_Get_library_version(mpilibversion, &resultlen);
+  printf( "# MPI-%d.%d = %s", mpiversion, mpisubversion, mpilibversion);
+
+  rank = MPI::COMM_WORLD.Get_rank();
+  size = MPI::COMM_WORLD.Get_size();
+  cout << "tid=0 out of 1 from rank " << rank << " out of " << size << "\n";
+
+  //std::cout << " mpi_thread_queried=" << mpi_thread_required << std::endl;
+
+  MPI::Finalize();
+
+  return 0;
+} /* end func main */

reframe/core/decorators.py

@@ -7,16 +7,26 @@
 
 import collections
 import inspect
+import sys
+import traceback
 
 import reframe
-from reframe.core.exceptions import ReframeSyntaxError
+from reframe.core.exceptions import ReframeSyntaxError, user_frame
 from reframe.core.logging import getlogger
 from reframe.core.pipeline import RegressionTest
 from reframe.utility.versioning import Version, VersionValidator
 
 
 def _register_test(cls, args=None):
-    def _instantiate():
+    def _instantiate(cls, args):
+        if isinstance(args, collections.Sequence):
+            return cls(*args)
+        elif isinstance(args, collections.Mapping):
+            return cls(**args)
+        elif args is None:
+            return cls()
+
+    def _instantiate_all():
         ret = []
         for cls, args in mod.__rfm_test_registry:
             try:
@@ -26,18 +36,21 @@ def _instantiate():
             except AttributeError:
                 mod.__rfm_skip_tests = set()
 
-            if isinstance(args, collections.Sequence):
-                ret.append(cls(*args))
-            elif isinstance(args, collections.Mapping):
-                ret.append(cls(**args))
-            elif args is None:
-                ret.append(cls())
+            try:
+                ret.append(_instantiate(cls, args))
+            except Exception as e:
+                frame = user_frame(sys.exc_info()[2])
+                msg = "skipping test due to errors: %s: " % cls.__name__
+                msg += "use `-v' for more information\n"
+                msg += "  FILE: %s:%s" % (frame.filename, frame.lineno)
+                getlogger().warning(msg)
+                getlogger().verbose(traceback.format_exc())
 
         return ret
 
     mod = inspect.getmodule(cls)
     if not hasattr(mod, '_rfm_gettests'):
-        mod._rfm_gettests = _instantiate
+        mod._rfm_gettests = _instantiate_all
 
     try:
         mod.__rfm_test_registry.append((cls, args))