Merge pull request #202 from barche/update-mpi

Upgrade linux mpich and open-mpi versions

Author: andreasnoack

.travis.yml

@@ -14,7 +14,7 @@ env:
   - MPI_IMPL=openmpi
 before_install:
   - sh ./conf/travis-install-mpi.sh $MPI_IMPL
-  - export PATH=$HOME/OpenMPI/bin:$PATH
+  - export PATH=$HOME/OpenMPI/bin:$HOME/MPICH/bin:$PATH
 # Work around OpenMPI attempting to create overly long temporary
 # file names - and erroring as a result
   - export TMPDIR=/tmp

conf/travis-install-mpi.sh

@@ -7,11 +7,14 @@ set -x
 
 MPI_IMPL="$1"
 os=`uname`
+OMPIVER=openmpi-3.0.0
+MPICHVER=mpich-3.2.1
 
 case "$os" in
     Darwin)
         brew update
         brew upgrade cmake
+        brew cask uninstall oclint # Prevent conflict with gcc
         case "$MPI_IMPL" in
             mpich|mpich3)
                 brew install mpich
@@ -36,16 +39,19 @@ case "$os" in
                 sudo apt-get install -y gfortran mpich2 libmpich2-3 libmpich2-dev
                 ;;
             mpich|mpich3)
-                sudo apt-get install -y gfortran libcr0 default-jdk hwloc libmpich10 libmpich-dev
-                wget -q http://de.archive.ubuntu.com/ubuntu/pool/universe/m/mpich/mpich_3.0.4-6ubuntu1_amd64.deb
-                sudo dpkg -i ./mpich_3.0.4-6ubuntu1_amd64.deb
-                # rm -f ./mpich_3.1-1ubuntu_amd64.deb
+                sudo apt-get install -y gfortran hwloc
+                wget http://www.mpich.org/static/downloads/3.2.1/$MPICHVER.tar.gz
+                tar -zxf $MPICHVER.tar.gz
+                cd $MPICHVER
+                sh ./configure --prefix=$HOME/MPICH --enable-shared > /dev/null
+                make -j > /dev/null
+                sudo make install > /dev/null
                 ;;
             openmpi)
                 sudo apt-get install -y gfortran
-                wget --no-check-certificate https://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-1.10.2.tar.gz
-                tar -zxf openmpi-1.10.2.tar.gz
-                cd openmpi-1.10.2
+                wget --no-check-certificate https://www.open-mpi.org/software/ompi/v3.0/downloads/$OMPIVER.tar.gz
+                tar -zxf $OMPIVER.tar.gz
+                cd $OMPIVER
                 sh ./configure --prefix=$HOME/OpenMPI > /dev/null
                 make -j > /dev/null
                 sudo make install > /dev/null

src/cman.jl

@@ -245,8 +245,12 @@ end
 
 # Kill a worker
 function kill(mgr::MPIManager, pid::Int, config::WorkerConfig)
-    # Do nothing, as the worker will self-terminate after calling MPI.Finalize
-    Base.set_worker_state(Base.Worker(pid), Base.W_TERMINATED)
+    # Exit the worker to avoid EOF errors on the workers
+    @spawnat pid begin
+        MPI.Finalize()
+        exit()
+    end
+    Distributed.set_worker_state(Distributed.Worker(pid), Distributed.W_TERMINATED)
 end
 
 # Set up a connection to a worker
@@ -418,14 +422,7 @@ end
 function stop_main_loop(mgr::MPIManager)
     if mgr.mode == TCP_TRANSPORT_ALL
         # Shut down all workers
-        for i in workers()
-            if i != myid()
-                @spawnat i begin
-                    MPI.Finalize()
-                    exit()
-                end
-            end
-        end
+        rmprocs(workers())
         # Poor man's flush of the send queue
         sleep(1)
         put!(mgr.initiate_shutdown, nothing)

test/runtests.jl

@@ -14,21 +14,34 @@ const coverage_opts =
                       JL_LOG_USER => "user",
                       JL_LOG_ALL => "all")
 
+# Files to run without mpiexec
+juliafiles = ["test_cman_julia.jl"]
+# Files to run with mpiexec -n 1
+singlefiles = []
+
 function runtests()
     nprocs = clamp(Sys.CPU_CORES, 2, 4)
     exename = joinpath(JULIA_HOME, Base.julia_exename())
     testdir = dirname(@__FILE__)
-    istest(f) = endswith(f, ".jl") && f != "runtests.jl"
+    istest(f) = endswith(f, ".jl") && startswith(f, "test_")
     testfiles = sort(filter(istest, readdir(testdir)))
+
+    extra_args = []
+    if contains(readlines(open(`mpiexec --version`)[1])[1], "OpenRTE")
+        push!(extra_args,"--oversubscribe")
+    end
+
     nfail = 0
     print_with_color(:white, "Running MPI.jl tests\n")
     for f in testfiles
         try
             coverage_opt = coverage_opts[Base.JLOptions().code_coverage]
-            if f == "test_cman_julia.jl"
+            if f ∈ singlefiles
+                run(`mpiexec $extra_args -n 1 $exename --code-coverage=$coverage_opt $(joinpath(testdir, f))`)
+            elseif f ∈ juliafiles
                 run(`$exename --code-coverage=$coverage_opt $(joinpath(testdir, f))`)
             else
-                run(`mpiexec -n $nprocs $exename --code-coverage=$coverage_opt $(joinpath(testdir, f))`)
+                run(`mpiexec $extra_args -n $nprocs $exename --code-coverage=$coverage_opt $(joinpath(testdir, f))`)
             end
             Base.with_output_color(:green,STDOUT) do io
                 println(io,"\tSUCCESS: $f")

test/test_cman_julia.jl

@@ -3,7 +3,11 @@ using MPI
 
 # Start workers via `mpiexec` that communicate among themselves via MPI;
 # communicate with the workers via TCP
-mgr = MPI.MPIManager(np=4)
+if contains(readlines(open(`mpiexec --version`)[1])[1], "OpenRTE")
+    mgr = MPI.MPIManager(np=4, mpirun_cmd=`mpiexec --oversubscribe -n 4`)
+else
+    mgr = MPI.MPIManager(np=4)
+end
 addprocs(mgr)
 
 refs = []