Various enhancements

* MPICH: Configure --with-device=ch4:ofi,ucx
* Remove references to source/build directories

Author: dalcinl

.github/workflows/cd-wheel.yml

@@ -174,7 +174,7 @@ jobs:
         CIBW_REPAIR_WHEEL_COMMAND_MACOS: >
           delocate-wheel
           --ignore-missing-dependencies
-          --exclude libmpi --exclude libpmpi
+          --exclude libmpi --exclude libpmpi --exclude libfabric
           --require-archs {delocate_archs}
           -w {dest_dir} -v {wheel}
 

bootstrap.sh

@@ -38,6 +38,35 @@ if test ! -d "$SOURCE"; then
     if test -f "$patch"; then
         patch -p1 -i "$patch" -d "$SOURCE"
     fi
+    if test "$mpiname" = "mpich"; then
+        if test "${version}" \> "4.2.2"; then
+            conf_args='s/\(ofi_subdir_args\)=\("--enable-embedded"\)/\1=""/'
+            configure="$SOURCE"/configure
+            sed -i.orig "$conf_args" "$configure"
+        fi
+        if test "${version}" \< "4.2.0"; then
+            disable_doc='s/^\(install-data-local:\s\+\)\$/\1#\$/'
+            Makefile="$SOURCE"/Makefile.in
+            sed -i.orig "$disable_doc" "$Makefile"
+        fi
+    fi
+    if test "$mpiname" = "openmpi"; then
+        makefiles=(
+            "$SOURCE"/3rd-party/openpmix/src/util/keyval/Makefile.in
+            "$SOURCE"/3rd-party/prrte/src/mca/rmaps/rank_file/Makefile.in
+            "$SOURCE"/3rd-party/prrte/src/util/hostfile/Makefile.in
+        )
+        variables=(
+            PRTE_CFLAGS_BEFORE_PICKY
+            PMIX_CFLAGS_BEFORE_PICKY
+        )
+        for makefile in "${makefiles[@]}"; do
+            test -f "$makefile" || continue
+            for variable in "${variables[@]}"; do
+                echo "$variable = @CFLAGS@" >> "$makefile"
+            done
+        done
+    fi
     if test "$mpiname-$(uname)" = "openmpi-Darwin"; then
         if test -d "$SOURCE"/3rd-party; then
             cd "$SOURCE"/3rd-party

build-wheel.sh

@@ -13,14 +13,14 @@ export CIBW_ARCHS=$ARCHLIST
 export CIBW_BEFORE_ALL='bash {project}/cibw-build-mpi.sh'
 export CIBW_TEST_COMMAND='bash {project}/cibw-check-mpi.sh'
 export CIBW_ENVIRONMENT_PASS='MPINAME RELEASE SOURCE WORKDIR DESTDIR'
-export CIBW_REPAIR_WHEEL_COMMAND_MACOS='delocate-wheel --ignore-missing-dependencies --exclude libmpi --exclude libpmpi --require-archs {delocate_archs -w {dest_dir} -v {wheel}'
+export CIBW_REPAIR_WHEEL_COMMAND_MACOS='delocate-wheel --ignore-missing-dependencies --exclude libmpi --exclude libpmpi --exclude libfabric --require-archs {delocate_archs -w {dest_dir} -v {wheel}'
 
 if test "$(uname)" = Linux; then
     containerengine=$(basename "$(command -v podman || command -v docker)")
     export CIBW_CONTAINER_ENGINE=$containerengine
     export SOURCE="/project/$SOURCE"
-    export WORKDIR="/host/$PWD/$WORKDIR"
-    export DESTDIR="/host/$PWD/$DESTDIR"
+    export WORKDIR="/host$PWD/$WORKDIR"
+    export DESTDIR="/host$PWD/$DESTDIR"
     platform=linux
 fi
 if test "$(uname)" = Darwin; then

check-wheel.sh

@@ -16,7 +16,9 @@ unzip -qq "$wheelfile" -d "$workdir"
 cd "$workdir"
 
 whlname=$(basename "$wheelfile")
-pkgname=${whlname%%-*}
+whlinfo=${whlname%%-py*}
+pkgname=${whlinfo%%-*}
+version=${whlinfo##*-}
 mpiname=${pkgname}
 
 data=$(ls -d "$pkgname"-*.data/data)
@@ -29,6 +31,9 @@ if test "$(uname)" = Linux; then
     libsdir=.libs
     print-runpath() { patchelf --print-rpath  "$1"; }
     print-needed()  { patchelf --print-needed "$1"; }
+    if test -f "$data"/lib/libfabric.so; then
+        runlibs=$runlibs'|libfabric'$soregex
+    fi
     if test -f "$data"/lib/libucp.so; then
         runlibs=$runlibs'|libuc(m|p|s|t)'$soregex
     fi
@@ -41,6 +46,9 @@ if test "$(uname)" = Darwin; then
     libsdir=.dylibs
     print-runpath() { otool -l "$1" | sed -n '/RPATH/{n;n;p;}'; }
     print-needed()  { otool -L "$1" | sed 1,1d; }
+    if test -f "$data"/lib/libfabric.dylib; then
+        runlibs=$runlibs'|libfabric'$soregex
+    fi
     if test -f "$data"/lib/libucp.dylib; then
         runlibs=$runlibs'|libuc(m|p|s|t)'$soregex
     fi
@@ -67,7 +75,12 @@ if test "$mpiname" = "mpich"; then
     libraries=(
         "$data"/lib/libmpi.*
     )
-    if test -d "$data"/lib/ucx; then
+    if ls "$data"/lib/libfabric.* > /dev/null 2>&1; then
+        libraries+=(
+            "$data"/lib/libfabric.*
+        )
+    fi
+    if ls "$data"/lib/libuct.* > /dev/null 2>&1; then
         libraries+=(
             "$data"/lib/libuc[mpst]*.*
             "$data"/lib/ucx/libuc*.*
@@ -96,14 +109,22 @@ if test "$mpiname" = "openmpi"; then
         "$data"/lib/libmpi.*
         "$data"/lib/libopen-*.*
     )
+    if test "${version%%.*}" -ge 5; then
+        libraries+=(
+            "$data"/lib/libevent*.*
+            "$data"/lib/libhwloc.*
+            "$data"/lib/libpmix.*
+            "$data"/lib/libprrte.*
+        )
+    fi
     runlibs+='|lib(z|util|event.*|hwloc)'$soregex
     runlibs+='|lib(open-(pal|rte)|pmix|prrte)'$soregex
 fi
 
 check-binary() {
     local dso=$1 out1="" out2=""
     echo checking "$dso"...
-    test -f "$dso" || printf "ERROR: file not found\n"
+    test -f "$dso" || (printf "ERROR: file not found\n"; exit 1)
     out1="$(print-runpath "$dso" | grep -vE "$runpath" || true)"
     test -z "$out1" || printf "ERROR: RUNPATH\n%s\n" "$out1"
     out2="$(print-needed  "$dso" | grep -vE "$runlibs" || true)"
@@ -114,15 +135,15 @@ check-binary() {
 for header in "${headers[@]-}"; do
     test -n "$header" || break
     echo checking "$header"...
-    test -f "$header"
+    test -f "$header" || (printf "ERROR: file not found\n"; exit 1)
     out=$(grep -E '^#include\s+"mpicxx\.h"' "$header" || true)
     test -z "$out" || printf "ERROR: include\n%s\n" "$out"
     test -z "$out"
 done
 for script in "${scripts[@]-}"; do
     test -n "$script" || break
     echo checking "$script"...
-    test -f "$script"
+    test -f "$script" || (printf "ERROR: file not found\n"; exit 1)
     out=$(grep -E "/opt/$mpiname" "$script" || true)
     test -z "$out" || printf "ERROR: prefix\n%s\n" "$out"
     test -z "$out"

cibw-build-mpi.sh

@@ -11,13 +11,31 @@ WORKDIR=${WORKDIR:-$PACKAGE/workdir}
 DESTDIR=${DESTDIR:-$PACKAGE/install}
 PREFIX=${PREFIX:-"/opt/$mpiname"}
 
+case $(uname) in
+    Linux)  njobs=$(nproc);;
+    Darwin) njobs=$(sysctl -n hw.physicalcpu);;
+esac
+
+if test "$(uname)" = Darwin; then
+    export MACOSX_DEPLOYMENT_TARGET="11.0"
+    if test "$(uname -m)" = x86_64; then
+        export MACOSX_DEPLOYMENT_TARGET="10.9"
+        export ac_cv_func_aligned_alloc="no" # macOS>=10.15
+    fi
+fi
+
 if test "$mpiname" = "mpich"; then
     version=$(sed -n 's/MPICH_VERSION=\(.*\)/\1/p' "$SOURCE"/maint/Version)
+    case $(uname)-$(uname -m) in
+        Linux-x86_64)  netmod=ofi,ucx ;;
+        Linux-aarch64) netmod=ofi,ucx ;;
+        *)             netmod=ofi ;;
+    esac
     options=(
         CC=cc
         CXX=c++
         --prefix="$PREFIX"
-        --with-device=ch4:"${variant:-ofi}"
+        --with-device="ch4:$netmod"
         --with-pm=hydra:gforker
         --with-libfabric=embedded
         --with-ucx=embedded
@@ -40,11 +58,16 @@ if test "$mpiname" = "mpich"; then
         export MPICH_MPICXX_LDFLAGS="-Wl,-rpath,$PREFIX/lib"
         export MPICH_MPIFORT_LDFLAGS="-Wl,-rpath,$PREFIX/lib"
     fi
-    disable_doc='s/^\(install-data-local:\s\+\)\$/\1#\$/'
-    sed -i.orig "$disable_doc" "$SOURCE"/Makefile.in
+    generated_files+=(src/include/mpichinfo.h)
+    generated_files+=(src/pm/hydra/hydra_config.h)
+    file_prefix_map+=("-ffile-prefix-map=$SOURCE=$mpiname-$version")
+    file_prefix_map+=("-ffile-prefix-map=$WORKDIR=$mpiname-$version")
+    export MPICHLIB_CFLAGS="${file_prefix_map[*]}"
 fi
 
 if test "$mpiname" = "openmpi"; then
+    # shellcheck disable=SC1091,SC2154
+    version=$(source "$SOURCE"/VERSION && echo "$major.$minor.$release")
     options=(
         CC=cc
         CXX=c++
@@ -67,27 +90,50 @@ if test "$mpiname" = "openmpi"; then
         --enable-mpi-fortran=mpifh
         --disable-dependency-tracking
     )
+    generated_files+=(ompi/tools/ompi_info/Makefile)
+    generated_files+=(oshmem/tools/oshmem_info/Makefile)
+    generated_files+=(3rd-party/openpmix/src/tools/*/Makefile)
+    generated_files+=(3rd-party/prrte/src/tools/*/Makefile)
+    generated_files+=(opal/include/opal_config.h)
+    generated_files+=(3rd-party/openpmix/src/include/pmix_config.h)
+    generated_files+=(3rd-party/prrte/src/include/prte_config.h)
+    file_prefix_map+=("-ffile-prefix-map=$SOURCE=$mpiname-$version")
+    file_prefix_map+=("-ffile-prefix-map=$WORKDIR=$mpiname-$version")
+    export CFLAGS="${file_prefix_map[*]}"
 fi
 
-if test "$(uname)" = Darwin; then
-    export MACOSX_DEPLOYMENT_TARGET="11.0"
-    if test "$(uname -m)" = x86_64; then
-        export MACOSX_DEPLOYMENT_TARGET="10.9"
-        export ac_cv_func_aligned_alloc="no" # macOS>=10.15
-    fi
-fi
-
-case $(uname) in
-    Linux)  njobs=$(nproc);;
-    Darwin) njobs=$(sysctl -n hw.physicalcpu);;
-esac
-
-mkdir -p "$WORKDIR"
-cd "$WORKDIR"
+mkdir -p "$WORKDIR" && cd "$WORKDIR"
 
 echo running configure
 "$SOURCE"/configure "${options[@]}" || cat config.log
 
+# shellcheck disable=SC2206
+generated_files=(${generated_files[@]:-})
+for filename in "${generated_files[@]}"; do
+    test -n "$filename" || continue
+    test -f "$filename" || continue
+    cflags="s|\s*${file_prefix_map[*]}\s*| |g"
+    source="s|$SOURCE|$mpiname-$version|g"
+    workdir="s|$WORKDIR|$mpiname-$version|g"
+    if test "$(basename "$filename")" = "Makefile"; then
+        echo removing CFLAGS from "$filename"
+        sed -i.orig "/-D.*_BUILD_CFLAGS=/$cflags" "$filename"
+        echo replacing SOURCE in "$filename"
+        sed -i.orig "/-D.*_BUILD_CPPFLAGS=/$source" "$filename"
+        sed -i.orig "/-D.*_BUILD_LIBS=/$source" "$filename"
+        echo replacing WORKDIR in "$filename"
+        sed -i.orig "/-D.*_BUILD_CPPFLAGS=/$workdir" "$filename"
+        sed -i.orig "/-D.*_BUILD_LIBS=/$workdir" "$filename"
+    else
+        echo removing CFLAGS from "$filename"
+        sed -i.orig "$cflags" "$filename"
+        echo replacing SOURCE in "$filename"
+        sed -i.orig "$source" "$filename"
+        echo replacing WORKDIR in "$filename"
+        sed -i.orig "$workdir" "$filename"
+    fi
+done
+
 echo running make with "${njobs:-1}" jobs
 make -j "${njobs:-1}" install DESTDIR="$DESTDIR"
 
@@ -97,7 +143,6 @@ cd "${DESTDIR}${PREFIX}"
 rm -f  include/*cxx.h
 rm -f  include/*.mod
 rm -f  include/*f.h
-rm -fr include/rdma
 rm -f  bin/mpif77
 rm -f  bin/mpif90
 rm -f  bin/mpifort
@@ -112,6 +157,12 @@ rm -f  lib/lib*mpifort.*
 rm -fr lib/pkgconfig
 rm -fr share
 
+cd "${DESTDIR}${PREFIX}"
+rm -fr include/rdma
+rm -f  bin/fi_*
+rm -f  lib/libfabric.a
+rm -f  lib/libfabric.la
+
 cd "${DESTDIR}${PREFIX}"
 rm -f  bin/io_demo
 rm -f  bin/ucx_read_profile
@@ -134,6 +185,7 @@ for script in "${scripts[@]}"; do
     test ! -L "$script" || continue
     # shellcheck disable=SC2016
     topdir='$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE:-$0}")/.." \&\& pwd)'
+    sed -i.orig s:"\s$WORKDIR/.*/lib\(.*\)\.la":" -l\1":g "$script"
     sed -i.orig s@^prefix=.*@prefix="$topdir"@ "$script"
     sed -i.orig s:"$PREFIX":\"\$\{prefix\}\":g "$script"
     sed -i.orig s:-Wl,-commons,use_dylibs::g "$script"
@@ -153,6 +205,19 @@ if test "$(uname)" = Linux; then
         mv "$(readlink "$libmpi")" "$libmpi"
         ln -sf "$libmpi" "${libmpi%.*}"
     fi
+    if test -f libfabric.so; then
+        patchelf --set-rpath "\$ORIGIN" "$libmpi"
+        for lib in libfabric.so.?; do
+            if test -f "$lib".*.*; then
+                mv "$(readlink "$lib")" "$lib"
+                ln -sf "$lib" "${lib%.*}"
+            fi
+            patchelf --set-rpath "\$ORIGIN" "$lib"
+            for exe in "${executables[@]}"; do
+                patchelf --remove-needed "$lib" "../bin/$exe"
+            done
+        done
+    fi
     if test -f libucp.so; then
         patchelf --set-rpath "\$ORIGIN" "$libmpi"
         for lib in libuc[mpst]*.so.?; do
@@ -182,16 +247,31 @@ if test "$(uname)" = Darwin; then
     libpmpi="libpmpi.12.dylib"
     cd "${DESTDIR}${PREFIX}/bin"
     for exe in "${executables[@]}"; do
-        install_name_tool -change "$libdir/$libmpi" "@rpath/$libmpi" "$exe"
-        install_name_tool -change "$libdir/$libpmpi" "@rpath/$libpmpi" "$exe"
         install_name_tool -add_rpath "@executable_path/../lib/" "$exe"
+        for lib in "$libmpi" "$libpmpi"; do
+            install_name_tool -change "$libdir/$lib" "@rpath/$lib" "$exe"
+        done
     done
     cd "${DESTDIR}${PREFIX}/lib"
     for lib in "$libmpi" "$libpmpi"; do
         install_name_tool -id "@rpath/$lib" "$lib"
         install_name_tool -add_rpath "@loader_path/" "$lib"
     done
     install_name_tool -change "$libdir/$libpmpi" "@rpath/$libpmpi" "$libmpi"
+    libfabric="libfabric.1.dylib"
+    if test -f "$libfabric"; then
+        install_name_tool -id "@rpath/$libfabric" "$libfabric"
+        install_name_tool -add_rpath "@loader_path/" "$libfabric"
+        oldname="$libdir/$libfabric"; newname="@rpath/$libfabric";
+        cd "${DESTDIR}${PREFIX}/bin"
+        for exe in "${executables[@]}"; do
+            install_name_tool -change "$oldname" "$newname" "$exe"
+        done
+        cd "${DESTDIR}${PREFIX}/lib"
+        for lib in "$libmpi" "$libpmpi"; do
+            install_name_tool -change "$oldname" "$newname" "$lib"
+        done
+    fi
 fi
 
 } # fixup-mpich()
@@ -365,3 +445,10 @@ done
 
 echo fixing install tree
 fixup-"$mpiname"
+
+echo checking install tree
+cd "${DESTDIR}${PREFIX}"
+echo cheking files with SOURCE
+! grep -lr "$SOURCE"
+echo cheking files with WORKDIR
+! grep -lr "$WORKDIR"

cibw-check-mpi.sh

@@ -68,3 +68,22 @@ RUN mpicxx helloworld.cxx -o helloworld-cxx
 RUN command -v mpiexec
 RUN mpiexec -n 3 ./helloworld-c
 RUN mpiexec -n 3 ./helloworld-cxx
+
+if test "$mpiname" = "mpich"; then
+    case $(uname)-$(uname -m) in
+        Linux-x86_64)  ch4netmods=(ofi ucx) ;;
+        Linux-aarch64) ch4netmods=(ofi ucx) ;;
+        *)             ch4netmods=(ofi) ;;
+    esac
+    export MPICH_CH4_OFI_CAPABILITY_DEBUG=1
+    export MPICH_CH4_UCX_CAPABILITY_DEBUG=1
+    for netmod in "${ch4netmods[@]}"; do
+        printf "testing ch4:%s ... " "$netmod"
+        export MPICH_CH4_NETMOD="$netmod"
+        ./helloworld-c | grep -i "$netmod" > /dev/null
+        printf "OK\n"
+    done
+    unset MPICH_CH4_OFI_CAPABILITY_DEBUG
+    unset MPICH_CH4_UCX_CAPABILITY_DEBUG
+    unset MPICH_CH4_NETMOD
+fi