recipe: Update build and test scripts

Author: dalcinl

recipe/build-mpi.sh

@@ -1,32 +1,37 @@
 #!/bin/bash
 
-# unset unused old fortran compiler vars
-unset F90 F77
-
 set -ex
 
-export FCFLAGS="$FFLAGS"
-
 # avoid absolute-paths in compilers
 export CC=$(basename "$CC")
 export CXX=$(basename "$CXX")
 export FC=$(basename "$FC")
 
-./autogen.pl --force
+# unset unused Fortran compiler variables
+unset FFLAGS F77 F90 F95
 
+# tweak compiler flags
+export LIBRARY_PATH="$PREFIX/lib"
 if [[ "$target_platform" == osx-* ]]; then
-    if [[ ! -z "$CONDA_BUILD_SYSROOT" ]]; then
+    if [[ -n "$CONDA_BUILD_SYSROOT" ]]; then
         export CFLAGS="$CFLAGS -isysroot $CONDA_BUILD_SYSROOT"
         export CXXFLAGS="$CXXFLAGS -isysroot $CONDA_BUILD_SYSROOT"
     fi
 fi
 
-if [[ -z $CUDA_HOME ]]; then
-    build_with_cuda=""
-else
+# UCX support
+build_with_ucx=""
+if [[ "$target_platform" == linux-* ]]; then
+    build_with_ucx="--with-ucx=$PREFIX"
+fi
+
+# CUDA support
+build_with_cuda=""
+if [[ -n "$CUDA_HOME" ]]; then
+    build_with_cuda="--with-cuda=yes"
     export CFLAGS="$CFLAGS -I$CUDA_HOME/include"
     export CXXFLAGS="$CXXFLAGS -I$CUDA_HOME/include"
-    build_with_cuda="--with-cuda --with-ucx=$PREFIX"
+    export LDFLAGS="$LDFLAGS -L$CUDA_HOME/lib64/stubs"
 fi
 
 if [[ $CONDA_BUILD_CROSS_COMPILATION == "1"  && $target_platform == osx-arm64 ]]; then
@@ -132,15 +137,6 @@ if [[ $CONDA_BUILD_CROSS_COMPILATION == "1"  && $target_platform == osx-arm64 ]]
     export ompi_cv_fortran_use_only=yes
 fi
 
-export LIBRARY_PATH="$PREFIX/lib"
-
-## Replaced by the patch from open-mpi/ompi#8361
-# if [[ "$target_platform" == *-64 ]]; then
-#     # -march=skylake-avx512 -march=nocona invalidates AVX512 flag. Remove -march flags and -mtune flags
-#     export CFLAGS=$(echo $CFLAGS | sed 's/-march=[a-z0-9\-]*//g')
-#     export CFLAGS=$(echo $CFLAGS | sed 's/-mtune=[a-z0-9\-]*//g')
-# fi
-
 ./configure --prefix=$PREFIX \
             --disable-dependency-tracking \
             --enable-mpi-fortran \
@@ -151,23 +147,32 @@ export LIBRARY_PATH="$PREFIX/lib"
             --with-wrapper-fcflags="-I$PREFIX/include" \
             --with-wrapper-ldflags="-L$PREFIX/lib -Wl,-rpath,$PREFIX/lib" \
             --with-sge \
-            $build_with_cuda || (cat config.log; false)
+            --with-hwloc=$PREFIX \
+            --with-libevent=$PREFIX \
+            --with-zlib=$PREFIX \
+            --enable-mca-dso \
+            $build_with_ucx \
+            $build_with_cuda \
+    || (cat config.log; false)
 
 make -j"${CPU_COUNT:-1}"
 make install
 
-if [ ! -z "$build_with_cuda" ]; then
-    echo "setting the mca opal_warn_on_missing_libcuda to 0..."
-    echo "opal_warn_on_missing_libcuda = 0" >> $PREFIX/etc/openmpi-mca-params.conf
-    echo "setting the mca opal_cuda_support to 0..."
-    echo "opal_cuda_support = 0" >> $PREFIX/etc/openmpi-mca-params.conf
-
-    echo "setting the mca pml to ^ucx..."
+POST_LINK=$PREFIX/bin/.openmpi-post-link.sh
+if [ -n "$build_with_ucx" ]; then
+    echo "setting MCA pml to ^ucx..."
     echo "pml = ^ucx" >> $PREFIX/etc/openmpi-mca-params.conf
-    echo "setting the mca osc to ^ucx..."
+    echo "setting MCA osc to ^ucx..."
     echo "osc = ^ucx" >> $PREFIX/etc/openmpi-mca-params.conf
-
-    POST_LINK=$PREFIX/bin/.openmpi-post-link.sh
-    cp $RECIPE_DIR/post-link.sh $POST_LINK
+    cat $RECIPE_DIR/post-link-ucx.sh >> $POST_LINK
+fi
+if [ -n "$build_with_cuda" ]; then
+    echo "setting MCA opal_warn_on_missing_libcuda to 0..."
+    echo "opal_warn_on_missing_libcuda = 0" >> $PREFIX/etc/openmpi-mca-params.conf
+    echo "setting MCA opal_cuda_support to 0..."
+    echo "opal_cuda_support = 0" >> $PREFIX/etc/openmpi-mca-params.conf
+    cat $RECIPE_DIR/post-link-cuda.sh >> $POST_LINK
+fi
+if [ -f $POST_LINK ]; then
     chmod +x $POST_LINK
 fi

recipe/conda_build_config.yaml

@@ -5,10 +5,13 @@ docker_image:  # [linux]
   - quay.io/condaforge/linux-anvil-cos7-cuda:10.2      # [linux64]
   - quay.io/condaforge/linux-anvil-ppc64le-cuda:11.0   # [ppc64le]
   - quay.io/condaforge/linux-anvil-aarch64-cuda:11.0   # [aarch64]
-cudatoolkit:  # [linux]
-  - 10.2      # [linux64]
-  - 11.0      # [ppc64le]
-  - 11.0      # [aarch64]
+enable_ucx:
+  - True       # [linux]
+  - False      # [not linux]
 enable_cuda:
-  - True      # [linux]
-  - False     # [not linux]
+  - True       # [linux]
+  - False      # [not linux]
+cuda_version:  # [linux]
+  - 10.2       # [linux64]
+  - 11.0       # [ppc64le]
+  - 11.0       # [aarch64]

recipe/meta.yaml

@@ -29,33 +29,43 @@ outputs:
       run_exports:
         - {{ pin_subpackage('openmpi', min_pin='x.x.x', max_pin='x') }}
       ignore_run_exports:
-        - ucx  # [enable_cuda]
+        - ucx  # [enable_ucx]
       script_env:
-        - CUDA_HOME     # [enable_cuda]
+        - CUDA_HOME  # [enable_cuda]
     requirements:
       build:
         - {{ compiler('c') }}
         - {{ compiler('cxx') }}
         - {{ compiler('fortran') }}
-        - autoconf  # [unix]
-        - automake  # [unix]
+        #- autoconf  # [unix]
+        #- automake  # [unix]
         - libtool   # [unix]
         - make  # [unix]
-        - perl 5.26.2
       host:
+        #- openpmix
+        #- prrte
+        - libhwloc
+        - libevent
         - zlib
-        - ucx                 # [enable_cuda]
-        - ucx-proc =*=gpu     # [enable_cuda]
+        - ucx  # [enable_ucx]
+        - cuda-version {{ cuda_version }}  # [enable_cuda]
       run:
-        - zlib
         - mpi 1.0 openmpi
+        #- openpmix
+        #- prrte
+        - libhwloc
+        - libevent
+        - zlib
       run_constrained:
-        - cudatoolkit  >= {{ cudatoolkit }}         # [enable_cuda]
-        - {{ pin_compatible("ucx", max_pin="x") }}  # [enable_cuda]
+        - {{ pin_compatible("ucx", max_pin="x.x") }}  # [enable_ucx]
+        # Open MPI only uses CUDA Driver APIs, set the minimal driver version
+        - __cuda  >= {{ cuda_version }}  # [enable_cuda]
+        # Ensure a consistent CUDA environment
+        - cuda-version  >= {{ cuda_version }}  # [enable_cuda]
+        - cudatoolkit   >= {{ cuda_version }}  # [enable_cuda]
     test:
       script: run_test.sh
       files:
-        - mpiexec.sh
         - tests/helloworld.sh
 
   - name: openmpi-mpicc
@@ -73,7 +83,6 @@ outputs:
     test:
       script: run_test.sh
       files:
-        - mpiexec.sh
         - tests/helloworld.c
 
   - name: openmpi-mpicxx
@@ -91,7 +100,6 @@ outputs:
     test:
       script: run_test.sh
       files:
-        - mpiexec.sh
         - tests/helloworld.cxx
 
   - name: openmpi-mpifort
@@ -108,7 +116,6 @@ outputs:
     test:
       script: run_test.sh
       files:
-        - mpiexec.sh
         - tests/helloworld.f
         - tests/helloworld.f90
   {% else %}

recipe/mpiexec.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+cat << EOF >> $PREFIX/.messages.txt
+
+On Linux, Open MPI is built with CUDA awareness but it is disabled by default.
+To enable it, please set the environment variable
+OMPI_MCA_opal_cuda_support=true
+before launching your MPI processes.
+Equivalently, you can set the MCA parameter in the command line:
+mpiexec --mca opal_cuda_support 1 ...
+Note that you might also need to set UCX_MEMTYPE_CACHE=n for CUDA awareness via
+UCX. Please consult UCX documentation for further details.
+
+EOF

recipe/post-link-cuda.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+cat << EOF >> $PREFIX/.messages.txt
+
+On Linux, Open MPI is built with UCX support but it is disabled by default.
+To enable it, first install UCX (conda install -c conda-forge ucx).
+Afterwards, set the environment variables
+OMPI_MCA_pml=ucx OMPI_MCA_osc=ucx
+before launching your MPI processes.
+Equivalently, you can set the MCA parameters in the command line:
+mpiexec --mca pml ucx --mca osc ucx ...
+
+EOF

recipe/post-link-ucx.sh

@@ -1,26 +1,34 @@
 #!/bin/bash
 set -ex
 
-export OMPI_MCA_plm=isolated
-export OMPI_MCA_btl_vader_single_copy_mechanism=none
-export OMPI_MCA_rmaps_base_oversubscribe=yes
-MPIEXEC="${PWD}/mpiexec.sh"
+export OMPI_MCA_pml=ob1
+export OMPI_MCA_btl=sm,self
+export OMPI_MCA_plm_ssh_agent=false
+export OMPI_MCA_rmaps_default_mapping_policy=:oversubscribe
+export OMPI_ALLOW_RUN_AS_ROOT=1
+export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+MPIEXEC="mpiexec"
 
 pushd "tests"
 
 if [[ $PKG_NAME == "openmpi" ]]; then
-  command -v ompi_info
-  ompi_info
 
-  if [[ ! -z "$(conda list | grep ucx)" ]]; then
+  if [[ -n "$(conda list | grep ucx)" ]]; then
     echo "Improper UCX dependency!"
     exit 1
   fi
-  if [[ ! -z "$(conda list | grep cudatoolkit)" ]]; then
-    echo "Improper cuda dependency!"
+
+  if [[ -n "$(conda list | grep cuda-version)" ]]; then
+    echo "Improper CUDA dependency!"
     exit 1
   fi
 
+  command -v ompi_info
+  ompi_info
+
+  command -v prte_info
+  prte_info
+
   command -v mpiexec
   $MPIEXEC --help
   $MPIEXEC -n 4 ./helloworld.sh