Skip to content

Commit cca508e

Browse files
dalcinldalcinl
committed
Build UCX and OFI with psm2 and rdmacm-core on manylinux_2_28
1 parent f589195 commit cca508e

File tree

4 files changed

+96
-20
lines changed

4 files changed

+96
-20
lines changed

‎cibw-build-mpi.sh‎

Lines changed: 54 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,12 @@ case $(uname) in
1919
esac
2020

2121
if test "$(uname)" = Linux; then
22+
PLATFORM_ID="$(source /etc/os-release && echo "${PLATFORM_ID:-}")"
2223
if test "${CIBUILDWHEEL}" = 1; then
2324
yum remove -y libatomic
25+
if test "$PLATFORM_ID" = "platform:el8"; then
26+
yum install -y libpsm2-devel rdma-core-devel
27+
fi
2428
fi
2529
fi
2630
if test "$(uname)" = Darwin; then
@@ -84,6 +88,15 @@ if test -d "$MODSOURCE"/libfabric-*; then
8488
echo running configure for OFI
8589
oficonfigure="$OFISOURCE"/configure
8690
ofioptions=(--prefix="$PREFIX" --disable-static)
91+
if test -f /usr/lib*/libpsm2.so; then
92+
ofioptions+=(--enable-psm2=dl)
93+
fi
94+
if test -f /usr/lib*/librdmacm.so; then
95+
ofioptions+=(--enable-verbs=dl)
96+
fi
97+
if test -f /usr/lib*/libefa.so; then
98+
ofioptions+=(--enable-efa=dl)
99+
fi
87100
"$oficonfigure" "${ofioptions[@]}" \
88101
CFLAGS="${build_cflags[*]}" \
89102
LDFLAGS="${build_ldflags[*]}"
@@ -181,6 +194,7 @@ if test "$mpiname" = "openmpi"; then
181194
--prefix="$PREFIX"
182195
--without-ucx
183196
--without-ofi
197+
--without-psm2
184198
--without-cuda
185199
--without-rocm
186200
--with-hwloc=internal
@@ -284,6 +298,7 @@ rm -f lib/libuc[mpst]*.a
284298
rm -f lib/libuc[mpst]*.la
285299
rm -f lib/ucx/libuc[mt]_*.a
286300
rm -f lib/ucx/libuc[mt]_*.la
301+
rm -f lib/ucx/libucx_perftest_*.*
287302
rm -fr lib/cmake/ucx
288303
rm -f lib/pkgconfig/ucx*.pc
289304
rm -fr share/ucx
@@ -297,15 +312,18 @@ for lib in libuc[mpst]*.so.?; do
297312
fi
298313
patchelf --set-rpath "\$ORIGIN" "$lib"
299314
done
300-
patchelf --add-rpath "\$ORIGIN/ucx" libucm.so.?
301-
patchelf --add-rpath "\$ORIGIN/ucx" libuct.so.?
302-
for lib in ucx/libuc[mt]_*.so.?; do
315+
if test -d ucx; then
316+
patchelf --add-rpath "\$ORIGIN/ucx" libuc[mpst].so.?
317+
fi
318+
for lib in ucx/libuc[mpst]_*.so.?; do
319+
if test -f "$lib"; then
320+
patchelf --set-rpath "\$ORIGIN" "$lib"
321+
patchelf --add-rpath "\$ORIGIN/.." "$lib"
322+
fi
303323
if test -f "$lib".*; then
304324
mv "$(dirname "$lib")/$(readlink "$lib")" "$lib"
305325
ln -srf "$lib" "${lib%.*}"
306326
fi
307-
patchelf --set-rpath "\$ORIGIN" "$lib"
308-
patchelf --add-rpath "\$ORIGIN/.." "$lib"
309327
done
310328

311329
} # fixup-ucx()
@@ -317,6 +335,8 @@ rm -fr include/rdma
317335
rm -f bin/fi_*
318336
rm -f lib/libfabric.a
319337
rm -f lib/libfabric.la
338+
rm -f lib/libfabric/lib*-fi.a
339+
rm -f lib/libfabric/lib*-fi.la
320340
rm -f lib/pkgconfig/libfabric.pc
321341
rm -f share/man/man?/fabric.?
322342
rm -f share/man/man?/fi_*.?
@@ -330,6 +350,15 @@ for lib in libfabric.so.?; do
330350
fi
331351
patchelf --set-rpath "\$ORIGIN" "$lib"
332352
done
353+
if test -d libfabric; then
354+
patchelf --add-rpath "\$ORIGIN/libfabric" libfabric.so.?
355+
fi
356+
for lib in libfabric/lib*-fi.so; do
357+
if test -f "$lib"; then
358+
patchelf --set-rpath "\$ORIGIN" "$lib"
359+
patchelf --set-rpath "\$ORIGIN/.." "$lib"
360+
fi
361+
done
333362

334363
} # fixup-ofi()
335364

@@ -350,6 +379,7 @@ rm -f lib/lib*mpi.la
350379
rm -f lib/lib*mpich*.*
351380
rm -f lib/lib*mpicxx.*
352381
rm -f lib/lib*mpifort.*
382+
rm -fr lib/cmake
353383
rm -fr lib/pkgconfig
354384
rm -fr share
355385

@@ -398,10 +428,16 @@ if test "$(uname)" = Linux; then
398428
if test -f libfabric.so; then
399429
mkdir -p "$mpiname"
400430
mv libfabric.* "$mpiname"
431+
if test -d libfabric; then
432+
mv libfabric "$mpiname"
433+
fi
401434
fi
402435
if test -f libucp.so; then
403436
mkdir -p "$mpiname"
404-
mv libuc[mpst]*.* ucx "$mpiname"
437+
mv libuc[mpst]*.* "$mpiname"
438+
if test -d ucx; then
439+
mv ucx "$mpiname"
440+
fi
405441
fi
406442
cd "${DESTDIR}${PREFIX}/lib"
407443
for lib in lib*.so; do
@@ -415,6 +451,11 @@ if test "$(uname)" = Linux; then
415451
if test -d "$mpiname"; then
416452
patchelf --add-rpath "\$ORIGIN/$mpiname" libmpi.so.*
417453
fi
454+
for dep in psm2 rdmacm ibverbs efa; do
455+
for v in $(seq 0 2); do
456+
patchelf --remove-needed "lib$dep.so.$v" libmpi.so.*
457+
done
458+
done
418459
cd "${DESTDIR}${PREFIX}/lib"
419460
find . -name '*.so' -type l -delete
420461
ln -s libmpi.so.* libmpi.so
@@ -561,10 +602,16 @@ if test "$(uname)" = Linux; then
561602
if test -f libfabric.so; then
562603
mkdir -p "$mpiname"
563604
mv libfabric.* "$mpiname"
605+
if test -d libfabric; then
606+
mv libfabric "$mpiname"
607+
fi
564608
fi
565609
if test -f libucp.so; then
566610
mkdir -p "$mpiname"
567-
mv libuc[mpst]*.* ucx "$mpiname"
611+
mv libuc[mpst]*.* "$mpiname"
612+
if test -d ucx; then
613+
mv ucx "$mpiname"
614+
fi
568615
fi
569616
for lib in lib*.so; do
570617
patchelf --set-rpath "\$ORIGIN" "$lib"

‎patches/auditwheel.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,18 @@ def walk(topdir):
3131

3232
auditwheel.tools.walk = walk
3333

34+
exclude = (
35+
"psm2",
36+
"rdmacm",
37+
"ibverbs",
38+
"mlx5",
39+
"efa",
40+
)
41+
3442
if "repair" in sys.argv:
3543
sys.argv.append("--only-plat")
44+
for name in exclude:
45+
sys.argv.append("--exclude")
46+
sys.argv.append(f"lib{name}.so.*")
3647

3748
sys.exit(main())

‎wheel-build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ fi
3131
if test "$(uname)" = Darwin; then
3232
export MACOSX_DEPLOYMENT_TARGET=11.0
3333
export CIBW_BUILD_FRONTEND='build'
34-
export CIBW_BUILD='pp310-*'
34+
export CIBW_BUILD='pp311-*'
3535
export SOURCE="$PWD/$SOURCE"
3636
export WORKDIR="$PWD/$WORKDIR"
3737
export DESTDIR="$PWD/$DESTDIR"

‎wheel-check.sh‎

Lines changed: 30 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -73,19 +73,22 @@ if test "$mpiname" = "mpich"; then
7373
"$data"/bin/hydra_*
7474
)
7575
libraries=(
76-
"$data"/lib/lib*mpi.*
76+
"$data"/lib/lib*mpi.*.*
7777
)
78-
if ls "$data"/lib/*/libfabric.* > /dev/null 2>&1; then
78+
if ls "$data"/lib/*/libucp.* > /dev/null 2>&1; then
7979
libraries+=(
80-
"$data"/lib/*/libfabric.*
80+
"$data"/lib/*/libuc[mpst]*.*
8181
)
8282
fi
83-
if ls "$data"/lib/*/libucp.* > /dev/null 2>&1; then
83+
if ls "$data"/lib/*/libfabric.* > /dev/null 2>&1; then
8484
libraries+=(
85-
"$data"/lib/*/libuc[mpst]*.*
86-
"$data"/lib/*/ucx/libuc[mpst]*.*
85+
"$data"/lib/*/libfabric.*
8786
)
8887
fi
88+
plugins=(
89+
"$data"/lib/*/ucx/libuc[mpst]_*.*
90+
"$data"/lib/*/libfabric/lib*-fi.*
91+
)
8992
fi
9093

9194
if test "$mpiname" = "openmpi"; then
@@ -106,7 +109,7 @@ if test "$mpiname" = "openmpi"; then
106109
"$data"/bin/*_wrapper
107110
)
108111
libraries=(
109-
"$data"/lib/libmpi.*
112+
"$data"/lib/libmpi.*.*
110113
"$data"/lib/libopen-*.*
111114
)
112115
if test "${version%%.*}" -ge 5; then
@@ -117,28 +120,38 @@ if test "$mpiname" = "openmpi"; then
117120
"$data"/lib/openmpi/libprrte.*
118121
)
119122
fi
123+
runlibs+='|lib(z|util|event.*|hwloc)'$soregex
124+
runlibs+='|lib(open-(pal|rte)|pmix|prrte)'$soregex
120125
if ls "$data"/lib/*/libucp.* > /dev/null 2>&1; then
121126
libraries+=(
122127
"$data"/lib/*/libuc[mpst]*.*
123-
"$data"/lib/*/ucx/libuc[mpst]*.*
124128
)
125129
fi
126130
if ls "$data"/lib/*/libfabric.* > /dev/null 2>&1; then
127131
libraries+=(
128132
"$data"/lib/*/libfabric.*
129133
)
130134
fi
131-
runlibs+='|lib(z|util|event.*|hwloc)'$soregex
132-
runlibs+='|lib(open-(pal|rte)|pmix|prrte)'$soregex
135+
plugins=(
136+
"$data"/lib/*/ucx/libuc[mpst]_*.*
137+
"$data"/lib/*/libfabric/lib*-fi.*
138+
)
139+
fi
140+
141+
deplibs=$runlibs
142+
if test "$(uname)" = Linux; then
143+
deplibs+='|lib(psm2|uct_ib)'$soregex
144+
deplibs+='|lib(rdmacm|ibverbs|mlx5|efa)'$soregex
133145
fi
134146

135147
check-binary() {
136-
local dso=$1 out1="" out2=""
148+
local dso="$1" deplibs="${2:-$runlibs}"
149+
local out1="" out2=""
137150
echo checking "$dso"...
138151
test -f "$dso" || (printf "ERROR: file not found\n"; exit 1)
139152
out1="$(print-runpath "$dso" | grep -vE "$runpath" || true)"
140153
test -z "$out1" || printf "ERROR: RUNPATH\n%s\n" "$out1"
141-
out2="$(print-needed "$dso" | grep -vE "$runlibs" || true)"
154+
out2="$(print-needed "$dso" | grep -vE "$deplibs" || true)"
142155
test -z "$out2" || printf "ERROR: NEEDED\n%s\n" "$out2"
143156
test -z "$out1"
144157
test -z "$out2"
@@ -172,6 +185,11 @@ for lib in "${libraries[@]-}"; do
172185
test -n "$lib" || break
173186
check-binary "$lib"
174187
done
188+
for lib in "${plugins[@]-}"; do
189+
test -n "$lib" || break
190+
test -f "$lib" || break
191+
check-binary "$lib" "$deplibs"
192+
done
175193
if test -d "$pkgname$libsdir"; then
176194
echo checking "$pkgname$libsdir"...
177195
out=$(ls -A "$pkgname$libsdir")

0 commit comments

Comments
 (0)