MPICH: Build with non-embedded internal libfabric

Author: dalcinl

.github/workflows/cd-wheel.yml

@@ -174,7 +174,7 @@ jobs:
         CIBW_REPAIR_WHEEL_COMMAND_MACOS: >
           delocate-wheel
           --ignore-missing-dependencies
-          --exclude libmpi --exclude libpmpi
+          --exclude libmpi --exclude libpmpi --exclude libfabric
           --require-archs {delocate_archs}
           -w {dest_dir} -v {wheel}
 

build-wheel.sh

@@ -13,7 +13,7 @@ export CIBW_ARCHS=$ARCHLIST
 export CIBW_BEFORE_ALL='bash {project}/cibw-build-mpi.sh'
 export CIBW_TEST_COMMAND='bash {project}/cibw-check-mpi.sh'
 export CIBW_ENVIRONMENT_PASS='MPINAME RELEASE SOURCE WORKDIR DESTDIR'
-export CIBW_REPAIR_WHEEL_COMMAND_MACOS='delocate-wheel --ignore-missing-dependencies --exclude libmpi --exclude libpmpi --require-archs {delocate_archs -w {dest_dir} -v {wheel}'
+export CIBW_REPAIR_WHEEL_COMMAND_MACOS='delocate-wheel --ignore-missing-dependencies --exclude libmpi --exclude libpmpi --exclude libfabric --require-archs {delocate_archs -w {dest_dir} -v {wheel}'
 
 if test "$(uname)" = Linux; then
     containerengine=$(basename "$(command -v podman || command -v docker)")

check-wheel.sh

@@ -29,6 +29,9 @@ if test "$(uname)" = Linux; then
     libsdir=.libs
     print-runpath() { patchelf --print-rpath  "$1"; }
     print-needed()  { patchelf --print-needed "$1"; }
+    if test -f "$data"/lib/libfabric.so; then
+        runlibs=$runlibs'|libfabric'$soregex
+    fi
     if test -f "$data"/lib/libucp.so; then
         runlibs=$runlibs'|libuc(m|p|s|t)'$soregex
     fi
@@ -41,6 +44,9 @@ if test "$(uname)" = Darwin; then
     libsdir=.dylibs
     print-runpath() { otool -l "$1" | sed -n '/RPATH/{n;n;p;}'; }
     print-needed()  { otool -L "$1" | sed 1,1d; }
+    if test -f "$data"/lib/libfabric.dylib; then
+        runlibs=$runlibs'|libfabric'$soregex
+    fi
     if test -f "$data"/lib/libucp.dylib; then
         runlibs=$runlibs'|libuc(m|p|s|t)'$soregex
     fi
@@ -67,6 +73,11 @@ if test "$mpiname" = "mpich"; then
     libraries=(
         "$data"/lib/libmpi.*
     )
+    if ls "$data"/lib/libfabric.* > /dev/null 2>&1; then
+        libraries+=(
+            "$data"/lib/libfabric.*
+        )
+    fi
     if test -d "$data"/lib/ucx; then
         libraries+=(
             "$data"/lib/libuc[mpst]*.*

cibw-build-mpi.sh

@@ -82,12 +82,29 @@ case $(uname) in
     Darwin) njobs=$(sysctl -n hw.physicalcpu);;
 esac
 
-mkdir -p "$WORKDIR"
-cd "$WORKDIR"
+if test "$mpiname" = "mpich"; then
+    mkdir -p "$WORKDIR"/libfabric && cd "$WORKDIR"/libfabric
+    echo running configure on libfabric
+    "$SOURCE"/modules/libfabric/configure --prefix="$PREFIX"
+    echo running make with "${njobs:-1}" jobs on libfabric
+    make -j "${njobs:-1}" install DESTDIR="$DESTDIR"
+    withofi=--with-libfabric="${DESTDIR}${PREFIX}"
+    options=("${options[@]/--with-libfabric=embedded/$withofi}")
+    destdirfixfiles+=(src/include/mpichinfo.h)
+    destdirfixfiles+=(src/pm/hydra/hydra_config.h)
+fi
+
+mkdir -p "$WORKDIR" && cd "$WORKDIR"
 
 echo running configure
 "$SOURCE"/configure "${options[@]}" || cat config.log
 
+for hdr in "${destdirfixfiles[@]-}"; do
+    test -n "$hdr" || break
+    echo removing DESTDIR from "$hdr"
+    sed -i.orig "s|$DESTDIR||g" "$hdr"
+done
+
 echo running make with "${njobs:-1}" jobs
 make -j "${njobs:-1}" install DESTDIR="$DESTDIR"
 
@@ -97,7 +114,6 @@ cd "${DESTDIR}${PREFIX}"
 rm -f  include/*cxx.h
 rm -f  include/*.mod
 rm -f  include/*f.h
-rm -fr include/rdma
 rm -f  bin/mpif77
 rm -f  bin/mpif90
 rm -f  bin/mpifort
@@ -112,6 +128,12 @@ rm -f  lib/lib*mpifort.*
 rm -fr lib/pkgconfig
 rm -fr share
 
+cd "${DESTDIR}${PREFIX}"
+rm -fr include/rdma
+rm -f  bin/fi_*
+rm -f  lib/libfabric.a
+rm -f  lib/libfabric.la
+
 cd "${DESTDIR}${PREFIX}"
 rm -f  bin/io_demo
 rm -f  bin/ucx_read_profile
@@ -153,6 +175,19 @@ if test "$(uname)" = Linux; then
         mv "$(readlink "$libmpi")" "$libmpi"
         ln -sf "$libmpi" "${libmpi%.*}"
     fi
+    if test -f libfabric.so; then
+        patchelf --set-rpath "\$ORIGIN" "$libmpi"
+        for lib in libfabric.so.?; do
+            if test -f "$lib".*.*; then
+                mv "$(readlink "$lib")" "$lib"
+                ln -sf "$lib" "${lib%.*}"
+            fi
+            patchelf --set-rpath "\$ORIGIN" "$lib"
+            for exe in "${executables[@]}"; do
+                patchelf --remove-needed "$lib" "../bin/$exe"
+            done
+        done
+    fi
     if test -f libucp.so; then
         patchelf --set-rpath "\$ORIGIN" "$libmpi"
         for lib in libuc[mpst]*.so.?; do
@@ -182,16 +217,31 @@ if test "$(uname)" = Darwin; then
     libpmpi="libpmpi.12.dylib"
     cd "${DESTDIR}${PREFIX}/bin"
     for exe in "${executables[@]}"; do
-        install_name_tool -change "$libdir/$libmpi" "@rpath/$libmpi" "$exe"
-        install_name_tool -change "$libdir/$libpmpi" "@rpath/$libpmpi" "$exe"
         install_name_tool -add_rpath "@executable_path/../lib/" "$exe"
+        for lib in "$libmpi" "$libpmpi"; do
+            install_name_tool -change "$libdir/$lib" "@rpath/$lib" "$exe"
+        done
     done
     cd "${DESTDIR}${PREFIX}/lib"
     for lib in "$libmpi" "$libpmpi"; do
         install_name_tool -id "@rpath/$lib" "$lib"
         install_name_tool -add_rpath "@loader_path/" "$lib"
     done
     install_name_tool -change "$libdir/$libpmpi" "@rpath/$libpmpi" "$libmpi"
+    libfabric="libfabric.1.dylib"
+    if test -f "$libfabric"; then
+        install_name_tool -id "@rpath/$libfabric" "$libfabric"
+        install_name_tool -add_rpath "@loader_path/" "$libfabric"
+        oldname="$libdir/$libfabric"; newname="@rpath/$libfabric";
+        cd "${DESTDIR}${PREFIX}/bin"
+        for exe in "${executables[@]}"; do
+            install_name_tool -change "$oldname" "$newname" "$exe"
+        done
+        cd "${DESTDIR}${PREFIX}/lib"
+        for lib in "$libmpi" "$libpmpi"; do
+            install_name_tool -change "$oldname" "$newname" "$lib"
+        done
+    fi
 fi
 
 } # fixup-mpich()