Skip to content

Commit b2f86f9

Browse files
dalcinldalcinl
committed
Build UCX and OFI with psm2 and rdmacm-core on manylinux_2_28
1 parent f589195 commit b2f86f9

File tree

4 files changed

+111
-28
lines changed

4 files changed

+111
-28
lines changed

‎cibw-build-mpi.sh‎

Lines changed: 69 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,16 @@ case $(uname) in
1919
esac
2020

2121
if test "$(uname)" = Linux; then
22+
# shellcheck disable=SC1091
23+
PLATFORM_ID="$(source /etc/os-release && echo "${PLATFORM_ID:-}")"
2224
if test "${CIBUILDWHEEL}" = 1; then
2325
yum remove -y libatomic
26+
if test "$PLATFORM_ID" = "platform:el8"; then
27+
yum install -y rdma-core-devel
28+
if test "$(uname -m)" = x86_64; then
29+
yum install -y libpsm2-devel
30+
fi
31+
fi
2432
fi
2533
fi
2634
if test "$(uname)" = Darwin; then
@@ -84,6 +92,15 @@ if test -d "$MODSOURCE"/libfabric-*; then
8492
echo running configure for OFI
8593
oficonfigure="$OFISOURCE"/configure
8694
ofioptions=(--prefix="$PREFIX" --disable-static)
95+
if test -f /usr/lib*/librdmacm.so; then
96+
ofioptions+=(--enable-verbs=dl)
97+
fi
98+
if test -f /usr/lib*/libpsm2.so; then
99+
ofioptions+=(--enable-psm2=dl)
100+
fi
101+
if test -f /usr/lib*/libefa.so; then
102+
ofioptions+=(--enable-efa=dl)
103+
fi
87104
"$oficonfigure" "${ofioptions[@]}" \
88105
CFLAGS="${build_cflags[*]}" \
89106
LDFLAGS="${build_ldflags[*]}"
@@ -181,6 +198,7 @@ if test "$mpiname" = "openmpi"; then
181198
--prefix="$PREFIX"
182199
--without-ucx
183200
--without-ofi
201+
--without-psm2
184202
--without-cuda
185203
--without-rocm
186204
--with-hwloc=internal
@@ -284,6 +302,7 @@ rm -f lib/libuc[mpst]*.a
284302
rm -f lib/libuc[mpst]*.la
285303
rm -f lib/ucx/libuc[mt]_*.a
286304
rm -f lib/ucx/libuc[mt]_*.la
305+
rm -f lib/ucx/libucx_perftest_*.*
287306
rm -fr lib/cmake/ucx
288307
rm -f lib/pkgconfig/ucx*.pc
289308
rm -fr share/ucx
@@ -297,15 +316,18 @@ for lib in libuc[mpst]*.so.?; do
297316
fi
298317
patchelf --set-rpath "\$ORIGIN" "$lib"
299318
done
300-
patchelf --add-rpath "\$ORIGIN/ucx" libucm.so.?
301-
patchelf --add-rpath "\$ORIGIN/ucx" libuct.so.?
302-
for lib in ucx/libuc[mt]_*.so.?; do
319+
if test -d ucx; then
320+
patchelf --add-rpath "\$ORIGIN/ucx" libuc[mpst].so.?
321+
fi
322+
for lib in ucx/libuc[mpst]_*.so.?; do
323+
if test -f "$lib"; then
324+
patchelf --set-rpath "\$ORIGIN" "$lib"
325+
patchelf --add-rpath "\$ORIGIN/.." "$lib"
326+
fi
303327
if test -f "$lib".*; then
304328
mv "$(dirname "$lib")/$(readlink "$lib")" "$lib"
305329
ln -srf "$lib" "${lib%.*}"
306330
fi
307-
patchelf --set-rpath "\$ORIGIN" "$lib"
308-
patchelf --add-rpath "\$ORIGIN/.." "$lib"
309331
done
310332

311333
} # fixup-ucx()
@@ -317,6 +339,8 @@ rm -fr include/rdma
317339
rm -f bin/fi_*
318340
rm -f lib/libfabric.a
319341
rm -f lib/libfabric.la
342+
rm -f lib/libfabric/lib*-fi.a
343+
rm -f lib/libfabric/lib*-fi.la
320344
rm -f lib/pkgconfig/libfabric.pc
321345
rm -f share/man/man?/fabric.?
322346
rm -f share/man/man?/fi_*.?
@@ -330,6 +354,15 @@ for lib in libfabric.so.?; do
330354
fi
331355
patchelf --set-rpath "\$ORIGIN" "$lib"
332356
done
357+
if test -d libfabric; then
358+
patchelf --add-rpath "\$ORIGIN/libfabric" libfabric.so.?
359+
fi
360+
for lib in libfabric/lib*-fi.so; do
361+
if test -f "$lib"; then
362+
patchelf --set-rpath "\$ORIGIN" "$lib"
363+
patchelf --set-rpath "\$ORIGIN/.." "$lib"
364+
fi
365+
done
333366

334367
} # fixup-ofi()
335368

@@ -350,6 +383,7 @@ rm -f lib/lib*mpi.la
350383
rm -f lib/lib*mpich*.*
351384
rm -f lib/lib*mpicxx.*
352385
rm -f lib/lib*mpifort.*
386+
rm -fr lib/cmake
353387
rm -fr lib/pkgconfig
354388
rm -fr share
355389

@@ -398,10 +432,16 @@ if test "$(uname)" = Linux; then
398432
if test -f libfabric.so; then
399433
mkdir -p "$mpiname"
400434
mv libfabric.* "$mpiname"
435+
if test -d libfabric; then
436+
mv libfabric "$mpiname"
437+
fi
401438
fi
402439
if test -f libucp.so; then
403440
mkdir -p "$mpiname"
404-
mv libuc[mpst]*.* ucx "$mpiname"
441+
mv libuc[mpst]*.* "$mpiname"
442+
if test -d ucx; then
443+
mv ucx "$mpiname"
444+
fi
405445
fi
406446
cd "${DESTDIR}${PREFIX}/lib"
407447
for lib in lib*.so; do
@@ -416,6 +456,15 @@ if test "$(uname)" = Linux; then
416456
patchelf --add-rpath "\$ORIGIN/$mpiname" libmpi.so.*
417457
fi
418458
cd "${DESTDIR}${PREFIX}/lib"
459+
for dep in rdmacm ibverbs psm2 efa; do
460+
for ver in $(seq 1 2); do
461+
patchelf --remove-needed "lib$dep.so.$ver" libmpi.so.*
462+
for exe in "${executables[@]}"; do
463+
patchelf --remove-needed "lib$dep.so.$ver" "../bin/$exe"
464+
done
465+
done
466+
done
467+
cd "${DESTDIR}${PREFIX}/lib"
419468
find . -name '*.so' -type l -delete
420469
ln -s libmpi.so.* libmpi.so
421470
fi
@@ -536,14 +585,6 @@ for exe in 'mpirun' 'ompi*' 'pmix*' 'prte*' 'opal*' 'orte*'; do
536585
done < <(find . -name "$exe" -type f)
537586
done
538587

539-
cd "${DESTDIR}${PREFIX}/lib"
540-
unset libraries
541-
for lib in 'lib*.so.*' 'lib*.*.dylib'; do
542-
while IFS= read -r filename
543-
do libraries+=("$(basename "$filename")")
544-
done < <(find . -name "$lib" -type f)
545-
done
546-
547588
if test "$(uname)" = Linux; then
548589
cd "${DESTDIR}${PREFIX}/bin"
549590
for exe in "${executables[@]}"; do
@@ -561,10 +602,16 @@ if test "$(uname)" = Linux; then
561602
if test -f libfabric.so; then
562603
mkdir -p "$mpiname"
563604
mv libfabric.* "$mpiname"
605+
if test -d libfabric; then
606+
mv libfabric "$mpiname"
607+
fi
564608
fi
565609
if test -f libucp.so; then
566610
mkdir -p "$mpiname"
567-
mv libuc[mpst]*.* ucx "$mpiname"
611+
mv libuc[mpst]*.* "$mpiname"
612+
if test -d ucx; then
613+
mv ucx "$mpiname"
614+
fi
568615
fi
569616
for lib in lib*.so; do
570617
patchelf --set-rpath "\$ORIGIN" "$lib"
@@ -599,6 +646,13 @@ if test "$(uname)" = Linux; then
599646
patchelf --add-rpath "$libpath" libopen-*.so.*
600647
fi
601648
cd "${DESTDIR}${PREFIX}/lib"
649+
for dep in rdmacm ibverbs psm2 efa; do
650+
for ver in $(seq 1 2); do
651+
patchelf --remove-needed "lib$dep.so.$ver" libmpi.so.*
652+
patchelf --remove-needed "lib$dep.so.$ver" libopen-*.so.*
653+
done
654+
done
655+
cd "${DESTDIR}${PREFIX}/lib"
602656
find . -name '*.so' -type l -delete
603657
ln -s libmpi.so.* libmpi.so
604658
fi

‎patches/auditwheel.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,18 @@ def walk(topdir):
3131

3232
auditwheel.tools.walk = walk
3333

34+
exclude = (
35+
"rdmacm",
36+
"ibverbs",
37+
"mlx5",
38+
"psm2",
39+
"efa",
40+
)
41+
3442
if "repair" in sys.argv:
3543
sys.argv.append("--only-plat")
44+
for name in exclude:
45+
sys.argv.append("--exclude")
46+
sys.argv.append(f"lib{name}.so.*")
3647

3748
sys.exit(main())

‎wheel-build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ fi
3131
if test "$(uname)" = Darwin; then
3232
export MACOSX_DEPLOYMENT_TARGET=11.0
3333
export CIBW_BUILD_FRONTEND='build'
34-
export CIBW_BUILD='pp310-*'
34+
export CIBW_BUILD='pp311-*'
3535
export SOURCE="$PWD/$SOURCE"
3636
export WORKDIR="$PWD/$WORKDIR"
3737
export DESTDIR="$PWD/$DESTDIR"

‎wheel-check.sh‎

Lines changed: 30 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -73,19 +73,22 @@ if test "$mpiname" = "mpich"; then
7373
"$data"/bin/hydra_*
7474
)
7575
libraries=(
76-
"$data"/lib/lib*mpi.*
76+
"$data"/lib/lib*mpi.*.*
7777
)
78-
if ls "$data"/lib/*/libfabric.* > /dev/null 2>&1; then
78+
if ls "$data"/lib/*/libucp.* > /dev/null 2>&1; then
7979
libraries+=(
80-
"$data"/lib/*/libfabric.*
80+
"$data"/lib/*/libuc[mpst]*.*
8181
)
8282
fi
83-
if ls "$data"/lib/*/libucp.* > /dev/null 2>&1; then
83+
if ls "$data"/lib/*/libfabric.* > /dev/null 2>&1; then
8484
libraries+=(
85-
"$data"/lib/*/libuc[mpst]*.*
86-
"$data"/lib/*/ucx/libuc[mpst]*.*
85+
"$data"/lib/*/libfabric.*
8786
)
8887
fi
88+
plugins=(
89+
"$data"/lib/*/ucx/libuc[mpst]_*.*
90+
"$data"/lib/*/libfabric/lib*-fi.*
91+
)
8992
fi
9093

9194
if test "$mpiname" = "openmpi"; then
@@ -106,7 +109,7 @@ if test "$mpiname" = "openmpi"; then
106109
"$data"/bin/*_wrapper
107110
)
108111
libraries=(
109-
"$data"/lib/libmpi.*
112+
"$data"/lib/libmpi.*.*
110113
"$data"/lib/libopen-*.*
111114
)
112115
if test "${version%%.*}" -ge 5; then
@@ -117,28 +120,38 @@ if test "$mpiname" = "openmpi"; then
117120
"$data"/lib/openmpi/libprrte.*
118121
)
119122
fi
123+
runlibs+='|lib(z|util|event.*|hwloc)'$soregex
124+
runlibs+='|lib(open-(pal|rte)|pmix|prrte)'$soregex
120125
if ls "$data"/lib/*/libucp.* > /dev/null 2>&1; then
121126
libraries+=(
122127
"$data"/lib/*/libuc[mpst]*.*
123-
"$data"/lib/*/ucx/libuc[mpst]*.*
124128
)
125129
fi
126130
if ls "$data"/lib/*/libfabric.* > /dev/null 2>&1; then
127131
libraries+=(
128132
"$data"/lib/*/libfabric.*
129133
)
130134
fi
131-
runlibs+='|lib(z|util|event.*|hwloc)'$soregex
132-
runlibs+='|lib(open-(pal|rte)|pmix|prrte)'$soregex
135+
plugins=(
136+
"$data"/lib/*/ucx/libuc[mpst]_*.*
137+
"$data"/lib/*/libfabric/lib*-fi.*
138+
)
139+
fi
140+
141+
deplibs=$runlibs
142+
if test "$(uname)" = Linux; then
143+
deplibs+='|lib(psm2|uct_ib)'$soregex
144+
deplibs+='|lib(rdmacm|ibverbs|mlx5|efa)'$soregex
133145
fi
134146

135147
check-binary() {
136-
local dso=$1 out1="" out2=""
148+
local dso="$1" deplibs="${2:-$runlibs}"
149+
local out1="" out2=""
137150
echo checking "$dso"...
138151
test -f "$dso" || (printf "ERROR: file not found\n"; exit 1)
139152
out1="$(print-runpath "$dso" | grep -vE "$runpath" || true)"
140153
test -z "$out1" || printf "ERROR: RUNPATH\n%s\n" "$out1"
141-
out2="$(print-needed "$dso" | grep -vE "$runlibs" || true)"
154+
out2="$(print-needed "$dso" | grep -vE "$deplibs" || true)"
142155
test -z "$out2" || printf "ERROR: NEEDED\n%s\n" "$out2"
143156
test -z "$out1"
144157
test -z "$out2"
@@ -172,6 +185,11 @@ for lib in "${libraries[@]-}"; do
172185
test -n "$lib" || break
173186
check-binary "$lib"
174187
done
188+
for lib in "${plugins[@]-}"; do
189+
test -n "$lib" || break
190+
test -f "$lib" || break
191+
check-binary "$lib" "$deplibs"
192+
done
175193
if test -d "$pkgname$libsdir"; then
176194
echo checking "$pkgname$libsdir"...
177195
out=$(ls -A "$pkgname$libsdir")

0 commit comments

Comments
 (0)