Merge pull request #648 from lattice/hotfix/p2p_performance_rank

Add support for P2P performance ranks

Author: maddyscientist

lib/comm_mpi.cpp

@@ -150,11 +150,23 @@ void comm_peer2peer_init(const char* hostname_recv_buf)
 	  cudaDeviceCanAccessPeer(&canAccessPeer[0], gpuid, neighbor_gpuid);
 	  cudaDeviceCanAccessPeer(&canAccessPeer[1], neighbor_gpuid, gpuid);
 
+      // this was introduced with CUDA 8
+#if CUDA_VERSION >= 8000
+      int accessRank[2];
+      cudaDeviceGetP2PAttribute(&accessRank[0], cudaDevP2PAttrPerformanceRank, gpuid, neighbor_gpuid);
+      cudaDeviceGetP2PAttribute(&accessRank[1], cudaDevP2PAttrPerformanceRank, neighbor_gpuid, gpuid);
+#endif
+
 	  if(canAccessPeer[0]*canAccessPeer[1]){
 	    peer2peer_enabled[dir][dim] = true;
 	    if (getVerbosity() > QUDA_SILENT)
-	      printf("Peer-to-peer enabled for rank %d (gpu=%d) with neighbor %d (gpu=%d) dir=%d, dim=%d\n",
-		     comm_rank(), gpuid, neighbor_rank, neighbor_gpuid, dir, dim);
+	      printf("Peer-to-peer enabled for rank %d (gpu=%d) with neighbor %d (gpu=%d) dir=%d, dim=%d, performance rank = (%d, %d)\n",
+#if CUDA_VERSION >= 8000
+                  comm_rank(), gpuid, neighbor_rank, neighbor_gpuid, dir, dim, accessRank[0], accessRank[1]);
+#else
+                  // default to 0 for CUDA < 8
+                  comm_rank(), gpuid, neighbor_rank, neighbor_gpuid, dir, dim, 0, 0);
+#endif
 	  }
 	} // on the same node
       } // different dimensions - x, y, z, t