Enable GPU CI (#557)

Co-authored-by: Simon Byrne <[email protected]>

Author: vchuravy

.buildkite/pipeline.yml

@@ -50,14 +50,15 @@ steps:
     if: build.message !~ /\[skip tests\]/
     timeout_in_minutes: 60
     env:
-      JULIA_MPI_TEST_ARRAYTYPE: Array
+      JULIA_MPI_TEST_ARRAYTYPE: CuArray
       JULIA_MPI_TEST_NPROCS: 2
       JULIA_MPI_PATH: "${BUILDKITE_BUILD_CHECKOUT_PATH}/openmpi"
       OMPI_ALLOW_RUN_AS_ROOT: 1
       OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
       OMPI_MCA_btl_vader_single_copy_mechanism: 'none' # https://github.com/open-mpi/ompi/issues/4948
       OPAL_PREFIX: "${BUILDKITE_BUILD_CHECKOUT_PATH}/openmpi" # Should we set this for the user?
       JULIA_CUDA_MEMORY_POOL: "none"
+    soft_fail: true
     commands: |
       echo "--- Configure MPI"
       buildkite-agent artifact download --step "Build OpenMPI -- CUDA" openmpi.tar.gz .
@@ -90,14 +91,15 @@ steps:
     if: build.message !~ /\[skip tests\]/
     timeout_in_minutes: 60
     env:
-      JULIA_MPI_TEST_ARRAYTYPE: Array
+      JULIA_MPI_TEST_ARRAYTYPE: CuArray
       JULIA_MPI_TEST_NPROCS: 2
       JULIA_MPI_PATH: "${BUILDKITE_BUILD_CHECKOUT_PATH}/openmpi"
       OMPI_ALLOW_RUN_AS_ROOT: 1
       OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
       OMPI_MCA_btl_vader_single_copy_mechanism: 'none' # https://github.com/open-mpi/ompi/issues/4948
       OPAL_PREFIX: "${BUILDKITE_BUILD_CHECKOUT_PATH}/openmpi" # Should we set this for the user?
       JULIA_CUDA_MEMORY_POOL: "none"
+    soft_fail: true
     commands: |
       echo "--- Configure MPI"
       buildkite-agent artifact download --step "Build OpenMPI -- CUDA" openmpi.tar.gz .

test/runtests.jl

@@ -6,6 +6,8 @@ using MPIPreferences
 using DoubleFloats
 if get(ENV, "JULIA_MPI_TEST_ARRAYTYPE", "") == "CuArray"
     import CUDA
+    CUDA.version()
+    CUDA.precompile_runtime()
     ArrayType = CUDA.CuArray
 else
     ArrayType = Array

test/test_bcast.jl

@@ -15,10 +15,14 @@ MPI.Init()
 comm = MPI.COMM_WORLD
 
 root = 0
-Random.seed!(17)
 matsize = (17,17)
 
 for T in Base.uniontypes(MPI.MPIDatatype)
+    # This test depends on the stability of the rng and we have observed with
+    # CUDA.jl that it is not gurantueed that the same number of rand calls will
+    # occur on each rank. (This is a hypothesis). To be sure we shall seed the rng
+    # just before we call rand.
+    Random.seed!(17)
     A = ArrayType(rand(T, matsize))
     B = MPI.Comm_rank(comm) == root ? A : similar(A)
     MPI.Bcast!(B, comm; root=root)