Merge branch 'master' into sb/mpipref

Author: vchuravy

.github/workflows/Documenter.yml

@@ -5,7 +5,13 @@ on:
     branches:
       - master
     tags: '*'
+    paths:
+      - 'docs/**'
+      - 'src/**'
   pull_request:
+    paths:
+      - 'docs/**'
+      - 'src/**'
 
 jobs:
   docs-build:

.github/workflows/ShellCheck.yml

@@ -5,7 +5,11 @@ on:
     branches:
       - master
     tags: '*'
+    paths:
+      - 'bin/**'
   pull_request:
+    paths:
+      - 'bin/**'
 
 jobs:
   shellcheck:

.github/workflows/UnitTests.yml

@@ -1,11 +1,20 @@
-name: Unit Tests
+name: MPI.jl Unit Tests
 
 on:
   pull_request:
-
+    paths:
+      - 'bin/**'
+      - 'deps/**'
+      - 'src/**'
+      - 'test/**'
   push:
     branches:
       - master
+    paths:
+      - 'bin/**'
+      - 'deps/**'
+      - 'src/**'
+      - 'test/**'
 
 jobs:
   test-default:

docs/make.jl

@@ -18,6 +18,12 @@ for (example_title, example_md) in EXAMPLES
     example_jl = example_md[1:end-2]*"jl"
     @info "Building $example_md"
     open(joinpath(@__DIR__, "src", example_md), "w") do mdfile
+        println(mdfile, """
+            ```@meta
+            EditURL = "https://github.com/JuliaParallel/MPI.jl/blob/master/docs/$(example_jl)"
+            ```
+            """
+        )
         println(mdfile, "# $example_title")
         println(mdfile)
         println(mdfile, "```julia")

docs/src/configuration.md

@@ -6,13 +6,52 @@ By default, MPI.jl will download and link against the following MPI implementati
 
 This is suitable for most single-node use cases, but for larger systems, such as HPC
 clusters or multi-GPU machines, you will probably want to configure against a
-system-provided MPI implementation in order to exploit feature such as fast network
+system-provided MPI implementation in order to exploit features such as fast network
 interfaces and CUDA-aware MPI interfaces.
 
 MPI.jl will attempt to detect when you are running on a HPC cluster, and warn the user
 about this. To disable this warning, set the environment variable
 `JULIA_MPI_CLUSTER_WARN=n`.
 
+## Julia wrapper for `mpiexec`
+
+Since you can configure `MPI.jl` to use one of several MPI implementations, you
+may have different Julia projects using different implementation.  Thus, it may
+be cumbersome to find out which `mpiexec` executable is associated to a specific
+project.  To make this easy, on Unix-based systems `MPI.jl` comes with a thin
+project-aware wrapper around `mpiexec`, called `mpiexecjl`.
+
+### Installation
+
+You can install `mpiexecjl` with [`MPI.install_mpiexecjl()`](@ref).  The default
+destination directory is `joinpath(DEPOT_PATH[1], "bin")`, which usually
+translates to `~/.julia/bin`, but check the value on your system.  You can also
+tell `MPI.install_mpiexecjl` to install to a different directory.
+
+```sh
+$ julia
+julia> using MPI
+julia> MPI.install_mpiexecjl()
+```
+
+To quickly call this wrapper we recommend you to add the destination directory
+to your [`PATH`](https://en.wikipedia.org/wiki/PATH_(variable)) environment
+variable.
+
+### Usage
+
+`mpiexecjl` has the same syntax as the `mpiexec` binary that will be called, but
+it takes in addition a `--project` option to call the specific binary associated
+to the `MPI.jl` version in the given project.  If no `--project` flag is used,
+the `MPI.jl` in the global Julia environment will be used instead.
+
+After installing `mpiexecjl` and adding its directory to `PATH`, you can run it
+with:
+
+```sh
+$ mpiexecjl --project=/path/to/project -n 20 julia script.jl
+```
+
 ## Using a system-provided MPI
 
 ### Requirements
@@ -74,36 +113,3 @@ The test suite can also be modified by the following variables:
 - `JULIA_MPIEXEC_TEST_ARGS`: Additional arguments to be passed to the MPI launcher for the tests only.
 - `JULIA_MPI_TEST_ARRAYTYPE`: Set to `CuArray` to test the CUDA-aware interface with
   [`CUDA.CuArray](https://github.com/JuliaGPU/CUDA.jl) buffers.
-
-## Julia wrapper for `mpiexec`
-
-Since you can configure `MPI.jl` to use one of several MPI implementations, you
-may have different Julia projects using different implementation.  Thus, it may
-be cumbersome to find out which `mpiexec` executable is associated to a specific
-project.  To make this easy, on Unix-based systems `MPI.jl` comes with a thin
-project-aware wrapper around `mpiexec`, called `mpiexecjl`.
-
-### Installation
-
-You can install `mpiexecjl` with [`MPI.install_mpiexecjl()`](@ref).  The default
-destination directory is `joinpath(DEPOT_PATH[1], "bin")`, which usually
-translates to `~/.julia/bin`, but check the value on your system.  You can also
-tell `MPI.install_mpiexecjl` to install to a different directory.
-
-To quickly call this wrapper we recommend you to add the destination directory
-to your [`PATH`](https://en.wikipedia.org/wiki/PATH_(variable)) environment
-variable.
-
-### Usage
-
-`mpiexecjl` has the same syntax as the `mpiexec` binary that will be called, but
-it takes in addition a `--project` option to call the specific binary associated
-to the `MPI.jl` version in the given project.  If no `--project` flag is used,
-the `MPI.jl` in the global Julia environment will be used instead.
-
-After installing `mpiexecjl` and adding its directory to `PATH`, you can run it
-with:
-
-```
-$ mpiexecjl --project=/path/to/project -n 20 julia script.jl
-```

docs/src/knownissues.md

@@ -65,8 +65,51 @@ ENV["UCX_ERROR_SIGNALS"] = "SIGILL,SIGBUS,SIGFPE"
 ```
 at `__init__`. If set externally, it should be modified to exclude `SIGSEGV` from the list.
 
+## CUDA-aware MPI
+
+### Memory pool
+
+Using CUDA-aware MPI on multi-GPU nodes with recent CUDA.jl may trigger (see [here](https://github.com/JuliaGPU/CUDA.jl/issues/1053#issue-946826096))
+```
+The call to cuIpcGetMemHandle failed. This means the GPU RDMA protocol
+cannot be used.
+  cuIpcGetMemHandle return value:   1
+```
+in the MPI layer, or fail on a segmentation fault (see [here](https://discourse.julialang.org/t/cuda-aware-mpi-works-on-system-but-not-for-julia/75060)) with
+```
+[1642930332.032032] [gcn19:4087661:0] gdr_copy_md.c:122 UCX ERROR gdr_pin_buffer failed. length :65536 ret:22
+```
+This is due to the MPI implementation using legacy `cuIpc*` APIs, which are incompatible with stream-ordered allocator, now default in CUDA.jl, see [UCX issue #7110](https://github.com/openucx/ucx/issues/7110).
+
+To circumvent this, one has to ensure the CUDA memory pool to be set to `none`:
+```
+export JULIA_CUDA_MEMORY_POOL=none
+```
+_More about CUDA.jl [memory environment-variables](https://cuda.juliagpu.org/stable/usage/memory/#Memory-pool)._
+
+### Hints to ensure CUDA-aware MPI to be functional
+
+Make sure to:
+- Have MPI and CUDA on path (or module loaded) that were used to build the CUDA-aware MPI
+- Make sure to have:
+    ```
+    export JULIA_CUDA_MEMORY_POOL=none
+    export JULIA_MPI_BINARY=system
+    export JULIA_CUDA_USE_BINARYBUILDER=false
+    ```
+- Add CUDA and MPI packages in Julia. Build MPI.jl in verbose mode to check whether correct versions are built/used: 
+    ```
+    julia -e 'using Pkg; pkg"add CUDA"; pkg"add MPI"; Pkg.build("MPI"; verbose=true)'
+    ```
+- Then in Julia, upon loading MPI and CUDA modules, you can check
+  - CUDA version: `CUDA.versioninfo()`
+  - If MPI has CUDA: `MPI.has_cuda()`
+  - If you are using correct MPI implementation: `MPI.identify_implementation()`
+
+After that, it may be preferred to run the Julia MPI script (as suggested [here](https://discourse.julialang.org/t/cuda-aware-mpi-works-on-system-but-not-for-julia/75060/11)) launching it from a shell script (as suggested [here](https://discourse.julialang.org/t/cuda-aware-mpi-works-on-system-but-not-for-julia/75060/4)).
+
 ## Microsoft MPI
 
 ### Custom operators on 32-bit Windows
 
-It is not possible to use [custom operators with 32-bit Microsoft MPI](https://github.com/JuliaParallel/MPI.jl/issues/246), as it uses the `stdcall` calling convention, which is not supported by [Julia's C-compatible function pointers](https://docs.julialang.org/en/v1/manual/calling-c-and-fortran-code/index.html#Creating-C-Compatible-Julia-Function-Pointers-1).
+It is not possible to use [custom operators with 32-bit Microsoft MPI](https://github.com/JuliaParallel/MPI.jl/issues/246), as it uses the `stdcall` calling convention, which is not supported by [Julia's C-compatible function pointers](https://docs.julialang.org/en/v1/manual/calling-c-and-fortran-code/index.html#Creating-C-Compatible-Julia-Function-Pointers-1).

docs/src/topology.md

@@ -1,13 +1,12 @@
 # Topology
 
 ```@docs
-MPI.Cart_coords
-MPI.Cart_coords!
+MPI.Dims_create
 MPI.Cart_create
 MPI.Cart_get
+MPI.Cart_coords
 MPI.Cart_rank
 MPI.Cart_shift
 MPI.Cart_sub
 MPI.Cartdim_get
-MPI.Dims_create!
 ```

src/deprecated.jl

@@ -224,3 +224,14 @@ import Base: @deprecate
 @deprecate(recv(source::Integer, tag::Integer, comm::Comm), recv(comm, MPI.Status; source=source, tag=tag), false)
 @deprecate(Sendrecv!(sendbuf, dest::Integer, sendtag::Integer, recvbuf, source::Integer, recvtag::Integer, comm::Comm),
            Sendrecv!(sendbuf, recvbuf, comm, MPI.Status; dest=dest, sendtag=sendtag, source=source, recvtag=recvtag)[2], false)
+
+@deprecate(Cart_create(comm_old::Comm, ndims::Integer, dims::MPIBuffertype{Cint}, periods::MPIBuffertype{Cint}, reorder),
+           Cart_create(comm_old, dims; periodic=periods, reorder=reorder), false)
+@deprecate(Cart_create(comm_old::Comm, dims::AbstractArray{T}, periods::Array{T}, reorder) where {T <: Integer},
+           Cart_create(comm_old, dims; periodic=periods, reorder=reorder), false)
+@deprecate(Dims_create!(nnodes::Integer, ndims::Integer, dims::MPIBuffertype{T}) where {T<:Integer},
+           dims .= Dims_create(nnodes, dims), false)
+@deprecate(Dims_create!(nnodes::Integer, dims::AbstractArray{T,N}) where {T<:Integer, N},
+           dims .= Dims_create(nnodes, dims), false)
+@deprecate(Cart_coords!(comm::Comm, rank::Integer, coords::MPIBuffertype{Cint}),
+           coords .= Cart_coords(comm, rank), false)