Use consts for known ABIs (#361)

Adds a mechanism to detect the ABI via MPI_LIBRARY_VERSION_STRING, and use that to detect the appropriate consts file, falling back on the C generator only when that fails.

Other changes:
 - add to Travis test matrix for C generator fallback
 - makes `MPI_Comm` etc aliases for `Cint` or `Ptr{Cvoid}` instead of custom types
 - defines `mpiexec` function consistent with how BinaryBuilder packages wrap executables, and adds support for `JULIA_MPIEXEC_ARGS` everywhere
 - `--oversubscribe` is no longer always applied in `runtests.jl` (use `JULIA_MPIEXEC_ARGS` instead).
 - `JULIA_MPIEXEC_ARGS` uses `Base.shell_split` instead of plain `split` to deal with quoted strings

Author: simonbyrne

.travis.yml

@@ -29,7 +29,13 @@ cache:
 env:
   - MPI_IMPL=mpich
   - MPI_IMPL=openmpi
+    JULIA_MPIEXEC_ARGS="--oversubscribe"
   - MPI_IMPL=intelmpi
+  - MPI_IMPL=mpich
+    JULIA_MPI_ABI=UnknownABI
+  - MPI_IMPL=openmpi
+    JULIA_MPI_ABI=UnknownABI
+    JULIA_MPIEXEC_ARGS="--oversubscribe"
 
 matrix:
   allow_failures: # issue 262

deps/build.jl

@@ -1,5 +1,14 @@
-using Libdl
+_doc_external(fname) = ""
 
+include(joinpath("..","src","paths.jl"))
+include(joinpath("..","src","implementations.jl"))
+
+if MPI_LIBRARY_ABI != UnknownABI
+    # constants provided under src/consts/
+    exit(0)
+end
+
+@info "Uknown MPI ABI: building constants file"
 
 MPI_PATH = get(ENV, "JULIA_MPI_PATH", nothing)
 MPI_LIBRARY_PATH = get(ENV, "JULIA_MPI_LIBRARY_PATH") do
@@ -8,6 +17,7 @@ end
 MPI_INCLUDE_PATH = get(ENV, "JULIA_MPI_INCLUDE_PATH") do
     MPI_PATH !== nothing ? joinpath(MPI_PATH,"include") : nothing
 end
+
 mpicc = get(ENV, "JULIA_MPICC") do
     if MPI_PATH !== nothing
         joinpath(MPI_PATH,"bin","mpicc")
@@ -16,27 +26,6 @@ mpicc = get(ENV, "JULIA_MPICC") do
     end
 end
 
-const mpiexec_path = get(ENV, "JULIA_MPIEXEC") do
-    if MPI_PATH !== nothing && Sys.isexecutable(joinpath(MPI_PATH,"bin","mpiexec"))
-        joinpath(MPI_PATH,"bin","mpiexec")
-    else
-        Sys.which("mpiexec")
-    end
-end
-
-const mpiexec_args = split(get(ENV, "JULIA_MPIEXEC_ARGS", ""))
-
-const libmpi = get(ENV, "JULIA_MPI_LIBRARY") do
-    libmpi = find_library(["libmpi", "libmpi_ibm", "msmpi", "libmpich"],
-                          MPI_LIBRARY_PATH !== nothing ? [MPI_LIBRARY_PATH] : [])
-    if libmpi == ""
-        error("No MPI library found.\nEnsure an MPI implementation is loaded, or set the `JULIA_MPI_PATH` variable.")
-    end
-    # expand paths
-    dlpath(libmpi)
-end
-
-
 if haskey(ENV, "JULIA_MPI_CFLAGS")
     CFLAGS = split(ENV["JULIA_MPI_CFLAGS"])
 else
@@ -53,65 +42,17 @@ else
     end
 end
 
-libsize = filesize(libmpi)
-
-@info "Using MPI library $libmpi"
-
-function Get_version()
-    major = Ref{Cint}()
-    minor = Ref{Cint}()
-    if Sys.iswindows()
-        ccall((:MPI_Get_version, libmpi), stdcall, Cint,
-              (Ptr{Cint}, Ptr{Cint}), major, minor)
-    else
-        ccall((:MPI_Get_version, libmpi), Cint,
-              (Ptr{Cint}, Ptr{Cint}), major, minor)
-    end
-    VersionNumber(major[], minor[])
-end
-
-MPI_VERSION = Get_version()
-
-@info "MPI version:\n$(MPI_VERSION)"
-
-function Get_library_version()
-    # There is no way to query at runtime what the length of the buffer should be.  
-    # https://github.com/mpi-forum/mpi-issues/issues/159    
-    # 8192 is the maximum value of MPI_MAX_LIBRARY_VERSION_STRING across known
-    # implementations.
-    buf = Array{UInt8}(undef, 8192)
-    buflen = Ref{Cint}()
-    if Sys.iswindows()
-        ccall((:MPI_Get_library_version, libmpi), stdcall, Cint, (Ptr{UInt8}, Ref{Cint}), buf, buflen)
-    else
-        ccall((:MPI_Get_library_version, libmpi), Cint, (Ptr{UInt8}, Ref{Cint}), buf, buflen)
-    end        
-    resize!(buf, buflen[])
-    return String(buf)
-end
+include("gen_consts.jl")
 
-MPI_LIBRARY_VERSION_STRING = Get_library_version()
+run(`$mpicc gen_consts.c -o gen_consts $CFLAGS`)
 
-@info "MPI library version:\n$(MPI_LIBRARY_VERSION_STRING)"
-
-open("deps.jl","w") do f
+open("consts.jl","w") do f
     println(f, "# This file is automatically generated")
     println(f, "# Do not edit")
     println(f)
-    println(f, :(const libmpi = $libmpi))
-    println(f, :(const libmpi_size = $libsize))
-    println(f, :(const MPI_VERSION = $MPI_VERSION))
-    println(f, :(const MPI_LIBRARY_VERSION_STRING = $MPI_LIBRARY_VERSION_STRING))
-    println(f, :(const mpiexec_path = $mpiexec_path))
-
-    if Sys.iswindows()
-        println(f, :(include("consts_msmpi.jl")))
-    else
-        include("gen_consts.jl")
+    println(f, :(MPI_LIBRARY_VERSION_STRING == $MPI_LIBRARY_VERSION_STRING || error("MPI library changed, re-run Pkg.build(\"MPI\")")))
+    println(f)
+end
 
-        run(`$mpicc gen_consts.c -o gen_consts $CFLAGS`)
-        run(`$mpiexec_path $mpiexec_args -n 1 ./gen_consts`)
+mpiexec(cmd -> run(`$cmd -n 1 ./gen_consts`))
 
-        println(f, :(include("consts.jl")))
-    end
-end

deps/gen_consts.jl

@@ -133,10 +133,8 @@ int main(int argc, char *argv[]) {
     MPI_Init(&argc, &argv);
 
     FILE *fptr;
-    fptr = fopen("consts.jl", "w");
+    fptr = fopen("consts.jl", "a");
 
-    fprintf(fptr, "# This file is automatically generated\\n");
-    fprintf(fptr, "# Do not edit\\n");
 """)
 
     println(f,"  fprintf(fptr, \"const MPI_Aint = Int%d\\n\", 8*(int)sizeof(MPI_Aint));")
@@ -149,17 +147,16 @@ int main(int argc, char *argv[]) {
     println(f,"  fprintf(fptr, \"const MPI_Status_Error_offset = %d\\n\", (int)offsetof(MPI_Status, MPI_ERROR));")
 
     for (T,constants) in MPI_handle
-        println(f,"  fprintf(fptr, \"primitive type $T %d end\\n\", (int)(sizeof($T) * 8));")
-
         T_f2c = T == :MPI_Datatype ? :MPI_Type_f2c : Symbol(T, :_f2c)
         T_c2f = T == :MPI_Datatype ? :MPI_Type_c2f : Symbol(T, :_c2f)
         if Libdl.dlsym_e(libptr, T_f2c) == C_NULL
-            println(f,"  fprintf(fptr, \"$T(c::Cint) = reinterpret($T,c)\\n\");")
+            println(f,"  fprintf(fptr, \"const $T = Cint\\n\");")
             for constant in constants
                 println(f,"  fprintf(fptr, \"const $constant = Cint(%i)\\n\", $constant);")
             end
         else
-            println(f,"  fprintf(fptr, \"$T(c::Cint) = ccall((:$T_f2c,libmpi),$T,(Cint,),c)\\n\");")
+            println(f,"  fprintf(fptr, \"const $T = Ptr{Cvoid}\\n\");")
+            println(f,"  fprintf(fptr, \"$(Symbol(T,:_f2c))(c::Cint) = ccall((:$T_f2c,libmpi),$T,(Cint,),c)\\n\");")
             for constant in constants
                 println(f,"  fprintf(fptr, \"const $constant = Cint(%i)\\n\", $T_c2f($constant));")
             end

docs/make.jl

@@ -46,6 +46,7 @@ makedocs(
         "usage.md",
         "Examples" => EXAMPLES,
         "Reference" => [
+            "library.md",
             "environment.md",
             "comm.md",
             "pointtopoint.md",

docs/src/environment.md

@@ -1,5 +1,11 @@
 # Environment
 
+## Launching MPI programs
+
+```@docs
+mpiexec
+```
+
 ## Functions
 
 ```@docs
@@ -9,5 +15,4 @@ MPI.Initialized
 MPI.Finalize
 MPI.Finalized
 MPI.universe_size
-MPI.has_cuda
 ```

docs/src/installation.md

@@ -2,39 +2,39 @@
 
 ## Requirements
 
-### Unix systems (Linux and MacOS)
+MPI.jl requires a shared library installation of a C MPI library, supporting the MPI 3.0 standard or later.
 
-MPI.jl requires:
+## Building
+
+The MPI.jl package can be installed via `add MPI` in the [Julia package
+manager](https://docs.julialang.org/en/v1/stdlib/Pkg/index.html). The package will attempt
+to find and identify the MPI installation.
 
-- A shared library MPI installation for C (supporting MPI standard 3.0), and
-- A C compiler available via the `mpicc` command: this is required as part of the build
-  process to determine the necessary type definitions and constants.
+The MPI standard doesn't specify the exact application binary interface (ABI), but the
+following implementations should work directly:
 
-This has been tested with:
 - [Open MPI](http://www.open-mpi.org/)
-- [MPICH](http://www.mpich.org/)
+- [MPICH](http://www.mpich.org/) (v3.1 or later)
 - [Intel MPI](https://software.intel.com/en-us/mpi-library)
+- [Microsoft MPI](https://docs.microsoft.com/en-us/message-passing-interface/microsoft-mpi)
+- [IBM Spectrum MPI](https://www.ibm.com/us-en/marketplace/spectrum-mpi)
 
-### Windows
-
-MPI.jl requires the [Microsoft MPI (MS-MPI)](https://docs.microsoft.com/en-us/message-passing-interface/microsoft-mpi) runtime to be installed.
+For other implementations, the build script will attempt to build a small C program to
+determine the appropriate type definitions and constants. This requires a compatible C
+compiler (`mpicc` by default).
 
-## Building
-
-The MPI.jl package can be installed via `add MPI` in the [Julia package manager](https://docs.julialang.org/en/v1/stdlib/Pkg/index.html). 
+## Environment variables
 
-The build script will attempt to find the shared library and constants: this can be
-controlled with the optional environment variables:
+The following optional environment variables can be used to control certain aspects of the
+build script and other library behaviour:
 
+- `JULIA_MPI_ABI`: the application binary interface, a string matching an [`MPIABI`](@ref) value.
 - `JULIA_MPI_PATH`: the top-level installation directory of MPI.
 - `JULIA_MPI_LIBRARY`: the path of the MPI shared library.
 - `JULIA_MPI_LIBRARY_PATH`: the directory containing the MPI library files.
 - `JULIA_MPI_INCLUDE_PATH`: the directory containing the MPI header files.
 - `JULIA_MPI_CFLAGS`: C flags passed to the constant generation build (default: `-lmpi`)
 - `JULIA_MPICC`: MPI C compiler (default: `mpicc`)
 - `JULIA_MPIEXEC`: MPI launcher command (default: `mpiexec`)
-- `JULIA_MPIEXEC_ARGS`: Additional arguments to be passed to MPI launcher (only used in the build step and tests).
+- `JULIA_MPIEXEC_ARGS`: Additional arguments to be passed to MPI launcher.
 - `JULIA_MPI_HAS_CUDA`: override the [`MPI.has_cuda`](@ref) function.
-
-If your MPI installation changes (e.g. it is upgraded by the system, or you switch
-libraries), you will need to re-run `build MPI` at the package prompt.

docs/src/library.md

@@ -0,0 +1,24 @@
+# Library information
+
+## Constants
+
+```@docs
+MPI.MPI_VERSION
+MPI.MPI_LIBRARY
+MPI.MPI_LIBRARY_VERSION
+MPI.MPI_LIBRARY_ABI
+MPI.MPI_LIBRARY_VERSION_STRING
+```
+
+## Enums
+
+```@docs
+MPI.MPIImpl
+MPI.MPIABI
+```
+
+## Functions
+
+```@docs
+MPI.has_cuda
+```

docs/src/usage.md

@@ -26,6 +26,8 @@ Hello world, I am rank 2 of 3
 Hello world, I am rank 1 of 3
 ```
 
+The [`mpiexec`](@ref) function is provided for launching MPI programs from Julia itself.
+
 ## CUDA-aware MPI support
 
 If your MPI implementation has been compiled with CUDA support, then `CuArray`s (from the

src/MPI.jl

@@ -4,6 +4,7 @@ using Libdl, Serialization
 using Requires
 using DocStringExtensions
 
+export mpiexec
 
 function serialize(x)
     s = IOBuffer()
@@ -31,7 +32,19 @@ function _doc_external(fname)
 """
 end    
 
-include(joinpath(@__DIR__, "..", "deps", "deps.jl"))
+include("paths.jl")
+include("implementations.jl")
+
+if MPI_LIBRARY_ABI == UnknownABI
+    include(joinpath(@__DIR__, "..", "deps", "consts.jl"))
+elseif MPI_LIBRARY_ABI == MPICHABI
+    include(joinpath("consts", "mpich.jl"))
+elseif MPI_LIBRARY_ABI == OpenMPIABI
+    include(joinpath("consts", "openmpi.jl"))
+elseif MPI_LIBRARY_ABI == MicrosoftMPIABI
+    include(joinpath("consts", "microsoftmpi.jl"))
+end
+
 include("error.jl")
 include("handle.jl")
 include("info.jl")
@@ -49,24 +62,30 @@ include("io.jl")
 include("deprecated.jl")
 
 function __init__()
+
     @static if Sys.isunix()
         # need to open libmpi with RTLD_GLOBAL flag for Linux, before
         # any ccall cannot use RTLD_DEEPBIND; this leads to segfaults
         # at least on Ubuntu 15.10
         Libdl.dlopen(libmpi, Libdl.RTLD_LAZY | Libdl.RTLD_GLOBAL)
     end
 
+    if MPI_LIBRARY_VERSION_STRING != Get_library_version()
+        # MPI library has changed, invalidate cache
+        cachefile = Base.compilecache(Base.PkgId(MPI))
+        rm(cachefile)
+        # TODO: figure out if we can reload package without erroring
+        # though that would probably trigger a race condition
+        error("MPI library has changed, please restart Julia")
+    end
+    
     # disable UCX memory hooks since it can mess up dlopen
     # https://github.com/openucx/ucx/issues/4001
     ENV["UCX_MEM_MMAP_RELOC"] = "no"
     ENV["UCX_MEM_MALLOC_HOOKS"] = "no"
     ENV["UCX_MEM_MALLOC_RELOC"] = "no"
     ENV["UCX_MEM_EVENTS"] = "no"
 
-    if filesize(dlpath(libmpi)) != libmpi_size
-        error("MPI library has changed, re-run Pkg.build(\"MPI\")")
-    end
-
     @require CuArrays="3a865a2d-5b23-5a0f-bc46-62713ec82fae" include("cuda.jl")
 end