Merge pull request #150 from eth-cscs/frontend/prgenv_pgi_kesch

Add regression tests for GPU direct.

Author: vkarak

config/cscs.py

@@ -107,20 +107,23 @@ class ReframeSettings:
                 'partitions': {
                     'login': {
                         'scheduler': 'local',
-                        'environs': ['PrgEnv-gnu', 'PrgEnv-cray', 'PrgEnv-gdr'],
+                        'environs': ['PrgEnv-gnu', 'PrgEnv-cray',
+                                     'PrgEnv-pgi', 'PrgEnv-gnu-gdr'],
                         'descr': 'Kesch login nodes',
                     },
                     'pn': {
                         'scheduler': 'nativeslurm',
                         'access': ['--partition=pn-regression'],
-                        'environs': ['PrgEnv-gnu', 'PrgEnv-cray', 'PrgEnv-gdr'],
+                        'environs': ['PrgEnv-gnu', 'PrgEnv-cray',
+                                     'PrgEnv-pgi', 'PrgEnv-gnu-gdr'],
                         'descr': 'Kesch post-processing nodes'
                     },
 
                     'cn': {
                         'scheduler': 'nativeslurm',
                         'access': ['--partition=cn-regression'],
-                        'environs': ['PrgEnv-gnu', 'PrgEnv-cray', 'PrgEnv-gdr'],
+                        'environs': ['PrgEnv-gnu', 'PrgEnv-cray',
+                                     'PrgEnv-pgi', 'PrgEnv-gnu-gdr'],
                         'descr': 'Kesch compute nodes',
                         'resources': {
                             '_rfm_gpu': ['--gres=gpu:{num_gpus_per_node}']
@@ -199,7 +202,14 @@ class ReframeSettings:
                     'cxx': 'mpicxx',
                     'ftn': 'mpif90',
                 },
-                'PrgEnv-gdr': {
+                'PrgEnv-pgi': {
+                    'type': 'ProgEnvironment',
+                    'modules': ['/apps/common/regression/prgenv_pgi_17.10_aj'],
+                    'cc': 'mpicc',
+                    'cxx': 'mpicxx',
+                    'ftn': 'mpif90',
+                },
+                'PrgEnv-gnu-gdr': {
                     'type': 'ProgEnvironment',
                     'modules': ['gmvapich2/17.02_cuda_8.0_gdr'],
                     'cc': 'mpicc',

cscs-checks/microbenchmarks/mch/g2g_meteoswiss_check.py

@@ -13,7 +13,7 @@ def __init__(self, g2g, **kwargs):
         self.descr = 'G2G Meteoswiss check with G2G=%s' % g2g
         self.strict_check = False
         self.valid_systems = ['kesch:cn']
-        self.valid_prog_environs = ['PrgEnv-gdr']
+        self.valid_prog_environs = ['PrgEnv-gnu-gdr']
         self.executable = 'src/$EXECUTABLE'
         self.sourcesdir = ('https://github.com/MeteoSwiss-APN/'
                            'comm_overlap_bench.git')

cscs-checks/prgenv/gpu_direct_acc.py

@@ -0,0 +1,47 @@
+import os
+
+import reframe.utility.sanity as sn
+from reframe.core.pipeline import RegressionTest
+
+class GpuDirectAccCheck(RegressionTest):
+    def __init__(self, **kwargs):
+        super().__init__('gpu_direct_acc_check',
+                         os.path.dirname(__file__), **kwargs)
+        self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn']
+        self.valid_prog_environs = ['PrgEnv-cray']
+        if self.current_system.name in ['daint', 'dom']:
+            self.modules = ['craype-accel-nvidia60']
+            self._pgi_flags = '-acc -ta=tesla:cc60'
+            self.variables = {'MPICH_RDMA_ENABLED_CUDA': '1'}
+        elif self.current_system.name in ['kesch']:
+            self.modules = ['craype-accel-nvidia35']
+            self._pgi_flags = '-acc -ta=tesla:cc35'
+            self.variables = {'MPICH_RDMA_ENABLED_CUDA': '1',
+                              'MV2_USE_CUDA': '1',
+                              'MV2_USE_GPUDIRECT': '1',
+                              'G2G': '1',
+                              'MPICH_G2G_PIPELINE': '1'}
+
+        self.num_tasks = 2
+        self.num_gpus_per_node = 1
+        self.sourcepath = 'gpu_direct_acc.f90'
+        self.num_tasks_per_node = 1
+
+        result = sn.extractsingle(r'Result :\s+(?P<result>\d+\.?\d*)',
+            self.stdout, 'result', float)
+        self.sanity_patterns = sn.assert_reference(result, 1., -1e-5, 1e-5)
+
+        self.maintainers = ['AJ', 'VK']
+        self.tags = {'production'}
+
+    def setup(self, partition, environ, **job_opts):
+        if environ.name == 'PrgEnv-cray':
+            environ.fflags = '-hacc -hnoomp'
+        elif environ.name == 'PrgEnv-pgi':
+            environ.fflags = self._pgi_flags
+
+        super().setup(partition, environ, **job_opts)
+
+
+def _get_checks(**kwargs):
+    return [GpuDirectAccCheck(**kwargs)]

cscs-checks/prgenv/gpu_direct_cuda.py

@@ -0,0 +1,50 @@
+import os
+
+import reframe.utility.sanity as sn
+from reframe.core.pipeline import RegressionTest
+
+class GpuDirectCudaCheck(RegressionTest):
+    def __init__(self, **kwargs):
+        super().__init__('gpu_direct_cuda_check',
+                         os.path.dirname(__file__), **kwargs)
+        self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn']
+        self.valid_prog_environs = ['PrgEnv-gnu']
+        if self.current_system.name in ['daint', 'dom']:
+            self.variables = {'MPICH_RDMA_ENABLED_CUDA': '1'}
+        elif self.current_system.name in ['kesch']:
+            self.valid_prog_environs = ['PrgEnv-gnu-gdr']
+            self.variables = {'MPICH_RDMA_ENABLED_CUDA': '1',
+                              'MV2_USE_CUDA': '1',
+                              'MV2_USE_GPUDIRECT': '1',
+                              'MPICH_G2G_PIPELINE': '1',
+                              'G2G': '1'}
+
+        self.num_tasks = 2
+        self.num_gpus_per_node = 1
+        self.sourcepath = 'gpu_direct_cuda.cu'
+        self.num_tasks_per_node = 1
+
+        self.modules = ['cudatoolkit']
+
+        result = sn.extractsingle(r'Result :\s+(?P<result>\d+\.?\d*)',
+            self.stdout, 'result', float)
+        self.sanity_patterns = sn.assert_reference(result, 1., -1e-5, 1e-5)
+
+        self.maintainers = ['AJ', 'VK']
+        self.tags = {'production'}
+
+    def compile(self):
+        # Set nvcc flags
+        nvidia_sm = '60'
+        cpp_compiler = 'CC'
+        if self.current_system.name == 'kesch':
+            nvidia_sm = '37'
+            cpp_compiler = 'mpicxx'
+        self.current_environ.cxxflags = ('-ccbin %s -lcublas -lcudart '
+                                         '-arch=sm_%s' % 
+                                         (cpp_compiler, nvidia_sm))
+        super().compile()
+
+
+def _get_checks(**kwargs):
+    return [GpuDirectCudaCheck(**kwargs)]

cscs-checks/prgenv/openacc_checks.py

@@ -3,19 +3,31 @@
 import reframe.utility.sanity as sn
 from reframe.core.pipeline import RegressionTest
 
-
 class OpenACCFortranCheck(RegressionTest):
-    def __init__(self, **kwargs):
-        super().__init__('openacc_fortran_check',
+    def __init__(self, num_tasks, **kwargs):
+        if num_tasks == 1:
+            check_name = 'openacc_fortran_check'
+        else:
+            check_name = 'openacc_mpi_fortran_check'
+        super().__init__(check_name,
                          os.path.dirname(__file__), **kwargs)
         self.valid_systems = ['daint:gpu', 'dom:gpu', 'kesch:cn']
         self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi']
         if self.current_system.name in ['daint', 'dom']:
             self.modules = ['craype-accel-nvidia60']
-
-        self.sourcepath = 'vecAdd_openacc.f90'
+            self._pgi_flags = '-acc -ta=tesla:cc60'
+        elif self.current_system.name in ['kesch']:
+            self.modules = ['craype-accel-nvidia35']
+            self._pgi_flags = '-acc -ta=tesla:cc35'
+
+        self.num_tasks = num_tasks
+        if self.num_tasks == 1:
+            self.sourcepath = 'vecAdd_openacc.f90'
+        else:
+            self.sourcepath = 'vecAdd_openacc_mpi.f90'
         self.num_gpus_per_node = 1
         self.executable = self.name
+        self.num_tasks_per_node = 1
 
         result = sn.extractsingle(r'final result:\s+(?P<result>\d+\.?\d*)',
             self.stdout, 'result', float)
@@ -28,10 +40,11 @@ def setup(self, partition, environ, **job_opts):
         if environ.name == 'PrgEnv-cray':
             environ.fflags = '-hacc -hnoomp'
         elif environ.name == 'PrgEnv-pgi':
-            environ.fflags = '-acc -ta=tesla:cc60'
+            environ.fflags = self._pgi_flags
 
         super().setup(partition, environ, **job_opts)
 
 
 def _get_checks(**kwargs):
-    return [OpenACCFortranCheck(**kwargs)]
+    return [OpenACCFortranCheck(1, **kwargs),
+            OpenACCFortranCheck(2, **kwargs)]

cscs-checks/prgenv/src/gpu_direct_acc.f90

@@ -0,0 +1,41 @@
+program GpuDirectAcc
+    implicit none
+
+    include 'mpif.h'
+
+    integer :: ierr, status
+    integer :: mpi_size, mpi_rank
+    integer(8) :: mydata(1)
+
+    call MPI_Init(ierr)
+
+    call MPI_Comm_size(MPI_COMM_WORLD, mpi_size, ierr)
+    call MPI_Comm_rank(MPI_COMM_WORLD, mpi_rank, ierr)
+
+    if (mpi_size.ne.2) then
+        if (mpi_rank.eq.0) write (*,*) "2 MPI ranks required"
+        call MPI_Finalize(ierr);
+        stop
+    end if
+
+    mydata(1) = mpi_rank
+
+!$acc data copy(mydata)
+    if (mpi_rank.eq.0) then
+!$acc host_data use_device(mydata)
+        call MPI_Recv(mydata, 1, MPI_INTEGER8, 1, 0, MPI_COMM_WORLD, status, ierr)
+!$acc end host_data
+    else
+!$acc host_data use_device(mydata)
+        call MPI_Send(mydata, 1, MPI_INTEGER8, 0, 0, MPI_COMM_WORLD, ierr)
+!$acc end host_data
+    end if
+!$acc end data
+
+    if (mpi_rank.eq.0) then
+        write (*,*) "Result : ", mydata
+    end if
+
+    call MPI_Finalize(ierr);
+
+end program GpuDirectAcc

cscs-checks/prgenv/src/gpu_direct_cuda.cu

@@ -0,0 +1,45 @@
+#include <iostream>
+#include <mpi.h>
+
+using std::cout;
+using std::endl;
+
+int main(int argc, char** argv){
+    MPI_Status status;
+    int mpi_size, mpi_rank;
+    int host_data, *device_data;
+
+    MPI_Init(&argc, &argv);
+
+    MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
+
+    if (mpi_size!=2){
+        if (mpi_rank==0) cout << "2 MPI ranks required" << endl;
+        MPI_Finalize();
+        return(1);
+    }
+
+    host_data = mpi_rank;
+    cudaMalloc((void **)&device_data, sizeof(int));
+
+    cudaMemcpy(device_data, &host_data, sizeof(int), cudaMemcpyHostToDevice);
+
+    if (mpi_rank==0){
+        MPI_Recv(device_data, 1, MPI_INT, 1, 0, MPI_COMM_WORLD, &status);
+    }else{
+        MPI_Send(device_data, 1, MPI_INT, 0, 0, MPI_COMM_WORLD);
+    }
+
+    cudaMemcpy(&host_data, device_data, sizeof(int), cudaMemcpyDeviceToHost);
+
+    cudaFree(device_data);
+
+    if (mpi_rank==0){
+        cout << "Result : " << host_data << endl;
+    }
+
+    MPI_Finalize();
+
+    return(0);
+}

cscs-checks/prgenv/src/vecAdd_openacc_mpi.f90

@@ -0,0 +1,60 @@
+      program main
+      include 'mpif.h'
+ 
+      ! Size of vectors
+      integer :: n = 100000
+  
+      ! Input vectors
+      real(8),dimension(:),allocatable :: a
+      real(8),dimension(:),allocatable :: b  
+      ! Output vector
+      real(8),dimension(:),allocatable :: c
+   
+      integer :: i
+      real(8) :: sum
+
+      call MPI_Init(ierr)
+      call MPI_Comm_size(MPI_COMM_WORLD, isize, ierr)
+      call MPI_Comm_rank(MPI_COMM_WORLD, irank, ierr)
+   
+      ! Allocate memory for each vector
+      allocate(a(n))
+      allocate(b(n))
+      allocate(c(n))
+   
+      ! Initialize content of input vectors, vector a[i] = sin(i)^2 vector b[i] = cos(i)^2
+      do i=1,n
+          a(i) = sin(i*1D0)*sin(i*1D0)
+          b(i) = cos(i*1D0)*cos(i*1D0)  
+      enddo
+   
+      ! Sum component wise and save result into vector c
+   
+      !$acc kernels copyin(a(1:n),b(1:n)), copyout(c(1:n))
+      do i=1,n
+          c(i) = a(i) + b(i)
+      enddo
+      !$acc end kernels
+   
+      sum = 0d0
+      ! Sum up vector c and print result divided by n, this should equal 1 within error
+      do i=1,n
+          sum = sum +  c(i)
+      enddo
+      sum = sum/n/isize
+
+      if (irank.eq.0) then
+          call MPI_Reduce(MPI_IN_PLACE, sum, 1, MPI_REAL8, MPI_SUM, 0, MPI_COMM_WORLD, ierr)
+          print *, 'final result: ', sum
+      else
+          call MPI_Reduce(sum, sum, 1, MPI_REAL8, MPI_SUM, 0, MPI_COMM_WORLD, ierr)
+      end if
+   
+      ! Release memory
+      deallocate(a)
+      deallocate(b)
+      deallocate(c)
+
+      call MPI_Finalize(ierr)
+  
+      end program