Merge pull request #302 from JuliaParallel/sb/cuda

CUDA-aware MPI with tests

Author: simonbyrne

.gitlab-ci.yml

@@ -20,12 +20,16 @@ include:
     - popd
   script:
     - export JULIA_MPI_PATH="${HOME}/mpi"
+    - export JULIA_PROJECT="test/cudaenv"
     - ${JULIA_MPI_PATH}/bin/ompi_info
     - julia -e 'using InteractiveUtils;
                 versioninfo()'
-    - julia --project -e 'using Pkg;
-                          Pkg.build();
-                          Pkg.test(coverage=true)'
+    - julia --color=yes -e 'using Pkg;
+                            Pkg.develop(PackageSpec(path=pwd()));
+                            Pkg.instantiate();
+                            Pkg.build()'
+    - julia --color=yes test/runtests.jl
+
 .gputest:
   extends: .projecttest
   variables:

Project.toml

@@ -7,6 +7,7 @@ version = "0.9.0"
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
 
@@ -15,9 +16,8 @@ julia = "1"
 
 [extras]
 DoubleFloats = "497a8b3b-efae-58df-a0af-a86822472b78"
-LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["DoubleFloats", "LinearAlgebra", "Pkg", "Test"]
+test = ["DoubleFloats", "Pkg", "Test"]

docs/src/usage.md

@@ -22,6 +22,13 @@ Hello world, I am rank 2 of 3
 Hello world, I am rank 1 of 3
 ```
 
+## CUDA-aware MPI support
+
+If your MPI implementation has been compiled with CUDA support, then `CuArray`s (from the
+[CuArrays.jl](https://github.com/JuliaGPU/CuArrays.jl) package) can be passed directly as
+send and receive buffers for point-to-point and collective operations (they may also work
+with one-sided operations, but these are not often supported).
+
 ## Finalizers
 
 In order to ensure MPI routines are called in the correct order at finalization time,

src/MPI.jl

@@ -1,6 +1,7 @@
 module MPI
 
 using Libdl, Serialization
+using Requires
 
 macro mpichk(expr)
     @assert expr isa Expr && expr.head == :call && expr.args[1] == :ccall
@@ -21,10 +22,14 @@ function deserialize(x)
     Serialization.deserialize(s)
 end
 
-primitive type SentinelPtr
-    Sys.WORD_SIZE
+primitive type SentinelPtr Sys.WORD_SIZE
 end
-Base.cconvert(::Type{Ptr{T}}, sptr::SentinelPtr) where {T} = reinterpret(Ptr{T}, sptr)
+
+primitive type MPIPtr Sys.WORD_SIZE
+end
+Base.cconvert(::Type{MPIPtr}, x::SentinelPtr) = x
+Base.unsafe_convert(::Type{MPIPtr}, x::SentinelPtr) = reinterpret(MPIPtr, x)
+
 
 function _doc_external(fname)
 """
@@ -65,6 +70,8 @@ function __init__()
     if filesize(dlpath(libmpi)) != libmpi_size
         error("MPI library has changed, re-run Pkg.build(\"MPI\")")
     end
+
+    @require CuArrays="3a865a2d-5b23-5a0f-bc46-62713ec82fae" include("cuda.jl")
 end
 
 end