MPICH: Build with non-embedded internal libfabric

Author: dalcinl

.github/workflows/cd-wheel.yml

@@ -174,7 +174,7 @@ jobs:
         CIBW_REPAIR_WHEEL_COMMAND_MACOS: >
           delocate-wheel
           --ignore-missing-dependencies
-          --exclude libmpi --exclude libpmpi
+          --exclude libmpi --exclude libpmpi --exclude libfabric
           --require-archs {delocate_archs}
           -w {dest_dir} -v {wheel}
 

build-wheel.sh

@@ -13,7 +13,7 @@ export CIBW_ARCHS=$ARCHLIST
 export CIBW_BEFORE_ALL='bash {project}/cibw-build-mpi.sh'
 export CIBW_TEST_COMMAND='bash {project}/cibw-check-mpi.sh'
 export CIBW_ENVIRONMENT_PASS='MPINAME RELEASE SOURCE WORKDIR DESTDIR'
-export CIBW_REPAIR_WHEEL_COMMAND_MACOS='delocate-wheel --ignore-missing-dependencies --exclude libmpi --exclude libpmpi --require-archs {delocate_archs -w {dest_dir} -v {wheel}'
+export CIBW_REPAIR_WHEEL_COMMAND_MACOS='delocate-wheel --ignore-missing-dependencies --exclude libmpi --exclude libpmpi --exclude libfabric --require-archs {delocate_archs -w {dest_dir} -v {wheel}'
 
 if test "$(uname)" = Linux; then
     containerengine=$(basename "$(command -v podman || command -v docker)")

check-wheel.sh

@@ -29,6 +29,9 @@ if test "$(uname)" = Linux; then
     libsdir=.libs
     print-runpath() { patchelf --print-rpath  "$1"; }
     print-needed()  { patchelf --print-needed "$1"; }
+    if test -f "$data"/lib/libfabric.so; then
+        runlibs=$runlibs'|libfabric'$soregex
+    fi
     if test -f "$data"/lib/libucp.so; then
         runlibs=$runlibs'|libuc(m|p|s|t)'$soregex
     fi
@@ -41,6 +44,9 @@ if test "$(uname)" = Darwin; then
     libsdir=.dylibs
     print-runpath() { otool -l "$1" | sed -n '/RPATH/{n;n;p;}'; }
     print-needed()  { otool -L "$1" | sed 1,1d; }
+    if test -f "$data"/lib/libfabric.dylib; then
+        runlibs=$runlibs'|libfabric'$soregex
+    fi
     if test -f "$data"/lib/libucp.dylib; then
         runlibs=$runlibs'|libuc(m|p|s|t)'$soregex
     fi
@@ -67,6 +73,11 @@ if test "$mpiname" = "mpich"; then
     libraries=(
         "$data"/lib/libmpi.*
     )
+    if ls "$data"/lib/libfabric.* > /dev/null 2>&1; then
+        libraries+=(
+            "$data"/lib/libfabric.*
+        )
+    fi
     if test -d "$data"/lib/ucx; then
         libraries+=(
             "$data"/lib/libuc[mpst]*.*

cibw-build-mpi.sh

@@ -82,8 +82,17 @@ case $(uname) in
     Darwin) njobs=$(sysctl -n hw.physicalcpu);;
 esac
 
-mkdir -p "$WORKDIR"
-cd "$WORKDIR"
+if test "$mpiname" = "mpich"; then
+    mkdir -p "$WORKDIR"/libfabric && cd "$WORKDIR"/libfabric
+    echo running configure on libfabric
+    "$SOURCE"/modules/libfabric/configure --prefix="$PREFIX"
+    echo running make with "${njobs:-1}" jobs on libfabric
+    make -j "${njobs:-1}" install DESTDIR="$DESTDIR"
+    withofi=--with-libfabric="${DESTDIR}${PREFIX}"
+    options=("${options[@]/--with-libfabric=embedded/$withofi}")
+fi
+
+mkdir -p "$WORKDIR" && cd "$WORKDIR"
 
 echo running configure
 "$SOURCE"/configure "${options[@]}" || cat config.log
@@ -97,7 +106,7 @@ cd "${DESTDIR}${PREFIX}"
 rm -f  include/*cxx.h
 rm -f  include/*.mod
 rm -f  include/*f.h
-rm -fr include/rdma
+rm -f  bin/fi_*
 rm -f  bin/mpif77
 rm -f  bin/mpif90
 rm -f  bin/mpifort
@@ -112,6 +121,12 @@ rm -f  lib/lib*mpifort.*
 rm -fr lib/pkgconfig
 rm -fr share
 
+cd "${DESTDIR}${PREFIX}"
+rm -fr include/rdma
+rm -f  bin/fi_*
+rm -f  lib/libfabric.a
+rm -f  lib/libfabric.la
+
 cd "${DESTDIR}${PREFIX}"
 rm -f  bin/io_demo
 rm -f  bin/ucx_read_profile
@@ -153,6 +168,19 @@ if test "$(uname)" = Linux; then
         mv "$(readlink "$libmpi")" "$libmpi"
         ln -sf "$libmpi" "${libmpi%.*}"
     fi
+    if test -f libfabric.so; then
+        patchelf --set-rpath "\$ORIGIN" "$libmpi"
+        for lib in libfabric.so.?; do
+            if test -f "$lib".*.*; then
+                mv "$(readlink "$lib")" "$lib"
+                ln -sf "$lib" "${lib%.*}"
+            fi
+            patchelf --set-rpath "\$ORIGIN" "$lib"
+            for exe in "${executables[@]}"; do
+                patchelf --remove-needed "$lib" "../bin/$exe"
+            done
+        done
+    fi
     if test -f libucp.so; then
         patchelf --set-rpath "\$ORIGIN" "$libmpi"
         for lib in libuc[mpst]*.so.?; do
@@ -180,18 +208,20 @@ if test "$(uname)" = Darwin; then
     libdir="$PREFIX/lib"
     libmpi="libmpi.12.dylib"
     libpmpi="libpmpi.12.dylib"
+    libfabric="libfabric.1.dylib"
     cd "${DESTDIR}${PREFIX}/bin"
     for exe in "${executables[@]}"; do
         install_name_tool -change "$libdir/$libmpi" "@rpath/$libmpi" "$exe"
         install_name_tool -change "$libdir/$libpmpi" "@rpath/$libpmpi" "$exe"
         install_name_tool -add_rpath "@executable_path/../lib/" "$exe"
     done
     cd "${DESTDIR}${PREFIX}/lib"
-    for lib in "$libmpi" "$libpmpi"; do
+    for lib in "$libmpi" "$libpmpi" "$libfabric"; do
         install_name_tool -id "@rpath/$lib" "$lib"
         install_name_tool -add_rpath "@loader_path/" "$lib"
     done
     install_name_tool -change "$libdir/$libpmpi" "@rpath/$libpmpi" "$libmpi"
+    install_name_tool -change "$libdir/$libfabric" "@rpath/$libfabric" "$libmpi"
 fi
 
 } # fixup-mpich()