Skip to content

Commit 428e105

Browse files
dalcinldalcinl
committed
Build UCX and OFI with psm2 and rdmacm-core on manylinux_2_28
1 parent f589195 commit 428e105

File tree

4 files changed

+98
-28
lines changed

4 files changed

+98
-28
lines changed

‎cibw-build-mpi.sh‎

Lines changed: 56 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,16 @@ case $(uname) in
1919
esac
2020

2121
if test "$(uname)" = Linux; then
22+
# shellcheck disable=SC1091
23+
PLATFORM_ID="$(source /etc/os-release && echo "${PLATFORM_ID:-}")"
2224
if test "${CIBUILDWHEEL}" = 1; then
2325
yum remove -y libatomic
26+
if test "$PLATFORM_ID" = "platform:el8"; then
27+
yum install -y rdma-core-devel
28+
if test "$(uname -m)" = x86_64; then
29+
yum install -y libpsm2-devel
30+
fi
31+
fi
2432
fi
2533
fi
2634
if test "$(uname)" = Darwin; then
@@ -84,6 +92,15 @@ if test -d "$MODSOURCE"/libfabric-*; then
8492
echo running configure for OFI
8593
oficonfigure="$OFISOURCE"/configure
8694
ofioptions=(--prefix="$PREFIX" --disable-static)
95+
if test -f /usr/lib*/librdmacm.so; then
96+
ofioptions+=(--enable-verbs=dl)
97+
fi
98+
if test -f /usr/lib*/libpsm2.so; then
99+
ofioptions+=(--enable-psm2=dl)
100+
fi
101+
if test -f /usr/lib*/libefa.so; then
102+
ofioptions+=(--enable-efa=dl)
103+
fi
87104
"$oficonfigure" "${ofioptions[@]}" \
88105
CFLAGS="${build_cflags[*]}" \
89106
LDFLAGS="${build_ldflags[*]}"
@@ -95,6 +112,9 @@ if test -d "$MODSOURCE"/libfabric-*; then
95112
done
96113
for pkg in "${DESTDIR}${PREFIX}"/lib/pkgconfig/libfabric*.pc; do
97114
sed -i "/prefix\s*=/s|\($PREFIX\)|$DESTDIR\1|g" "$pkg"
115+
for dep in rdmacm ibverbs psm2 efa; do
116+
sed -i "/Libs\.private:\s*/s|-l$dep\s*||g" "$pkg"
117+
done
98118
done
99119
fi
100120

@@ -181,6 +201,7 @@ if test "$mpiname" = "openmpi"; then
181201
--prefix="$PREFIX"
182202
--without-ucx
183203
--without-ofi
204+
--without-psm2
184205
--without-cuda
185206
--without-rocm
186207
--with-hwloc=internal
@@ -284,6 +305,7 @@ rm -f lib/libuc[mpst]*.a
284305
rm -f lib/libuc[mpst]*.la
285306
rm -f lib/ucx/libuc[mt]_*.a
286307
rm -f lib/ucx/libuc[mt]_*.la
308+
rm -f lib/ucx/libucx_perftest_*.*
287309
rm -fr lib/cmake/ucx
288310
rm -f lib/pkgconfig/ucx*.pc
289311
rm -fr share/ucx
@@ -297,15 +319,18 @@ for lib in libuc[mpst]*.so.?; do
297319
fi
298320
patchelf --set-rpath "\$ORIGIN" "$lib"
299321
done
300-
patchelf --add-rpath "\$ORIGIN/ucx" libucm.so.?
301-
patchelf --add-rpath "\$ORIGIN/ucx" libuct.so.?
302-
for lib in ucx/libuc[mt]_*.so.?; do
322+
if test -d ucx; then
323+
patchelf --add-rpath "\$ORIGIN/ucx" libuc[mpst].so.?
324+
fi
325+
for lib in ucx/libuc[mpst]_*.so.?; do
326+
if test -f "$lib"; then
327+
patchelf --set-rpath "\$ORIGIN" "$lib"
328+
patchelf --add-rpath "\$ORIGIN/.." "$lib"
329+
fi
303330
if test -f "$lib".*; then
304331
mv "$(dirname "$lib")/$(readlink "$lib")" "$lib"
305332
ln -srf "$lib" "${lib%.*}"
306333
fi
307-
patchelf --set-rpath "\$ORIGIN" "$lib"
308-
patchelf --add-rpath "\$ORIGIN/.." "$lib"
309334
done
310335

311336
} # fixup-ucx()
@@ -317,6 +342,8 @@ rm -fr include/rdma
317342
rm -f bin/fi_*
318343
rm -f lib/libfabric.a
319344
rm -f lib/libfabric.la
345+
rm -f lib/libfabric/lib*-fi.a
346+
rm -f lib/libfabric/lib*-fi.la
320347
rm -f lib/pkgconfig/libfabric.pc
321348
rm -f share/man/man?/fabric.?
322349
rm -f share/man/man?/fi_*.?
@@ -330,6 +357,15 @@ for lib in libfabric.so.?; do
330357
fi
331358
patchelf --set-rpath "\$ORIGIN" "$lib"
332359
done
360+
if test -d libfabric; then
361+
patchelf --add-rpath "\$ORIGIN/libfabric" libfabric.so.?
362+
fi
363+
for lib in libfabric/lib*-fi.so; do
364+
if test -f "$lib"; then
365+
patchelf --set-rpath "\$ORIGIN" "$lib"
366+
patchelf --set-rpath "\$ORIGIN/.." "$lib"
367+
fi
368+
done
333369

334370
} # fixup-ofi()
335371

@@ -350,6 +386,7 @@ rm -f lib/lib*mpi.la
350386
rm -f lib/lib*mpich*.*
351387
rm -f lib/lib*mpicxx.*
352388
rm -f lib/lib*mpifort.*
389+
rm -fr lib/cmake
353390
rm -fr lib/pkgconfig
354391
rm -fr share
355392

@@ -398,10 +435,16 @@ if test "$(uname)" = Linux; then
398435
if test -f libfabric.so; then
399436
mkdir -p "$mpiname"
400437
mv libfabric.* "$mpiname"
438+
if test -d libfabric; then
439+
mv libfabric "$mpiname"
440+
fi
401441
fi
402442
if test -f libucp.so; then
403443
mkdir -p "$mpiname"
404-
mv libuc[mpst]*.* ucx "$mpiname"
444+
mv libuc[mpst]*.* "$mpiname"
445+
if test -d ucx; then
446+
mv ucx "$mpiname"
447+
fi
405448
fi
406449
cd "${DESTDIR}${PREFIX}/lib"
407450
for lib in lib*.so; do
@@ -536,14 +579,6 @@ for exe in 'mpirun' 'ompi*' 'pmix*' 'prte*' 'opal*' 'orte*'; do
536579
done < <(find . -name "$exe" -type f)
537580
done
538581

539-
cd "${DESTDIR}${PREFIX}/lib"
540-
unset libraries
541-
for lib in 'lib*.so.*' 'lib*.*.dylib'; do
542-
while IFS= read -r filename
543-
do libraries+=("$(basename "$filename")")
544-
done < <(find . -name "$lib" -type f)
545-
done
546-
547582
if test "$(uname)" = Linux; then
548583
cd "${DESTDIR}${PREFIX}/bin"
549584
for exe in "${executables[@]}"; do
@@ -561,10 +596,16 @@ if test "$(uname)" = Linux; then
561596
if test -f libfabric.so; then
562597
mkdir -p "$mpiname"
563598
mv libfabric.* "$mpiname"
599+
if test -d libfabric; then
600+
mv libfabric "$mpiname"
601+
fi
564602
fi
565603
if test -f libucp.so; then
566604
mkdir -p "$mpiname"
567-
mv libuc[mpst]*.* ucx "$mpiname"
605+
mv libuc[mpst]*.* "$mpiname"
606+
if test -d ucx; then
607+
mv ucx "$mpiname"
608+
fi
568609
fi
569610
for lib in lib*.so; do
570611
patchelf --set-rpath "\$ORIGIN" "$lib"

‎patches/auditwheel.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,18 @@ def walk(topdir):
3131

3232
auditwheel.tools.walk = walk
3333

34+
exclude = (
35+
"rdmacm",
36+
"ibverbs",
37+
"mlx5",
38+
"psm2",
39+
"efa",
40+
)
41+
3442
if "repair" in sys.argv:
3543
sys.argv.append("--only-plat")
44+
for name in exclude:
45+
sys.argv.append("--exclude")
46+
sys.argv.append(f"lib{name}.so.*")
3647

3748
sys.exit(main())

‎wheel-build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ fi
3131
if test "$(uname)" = Darwin; then
3232
export MACOSX_DEPLOYMENT_TARGET=11.0
3333
export CIBW_BUILD_FRONTEND='build'
34-
export CIBW_BUILD='pp310-*'
34+
export CIBW_BUILD='pp311-*'
3535
export SOURCE="$PWD/$SOURCE"
3636
export WORKDIR="$PWD/$WORKDIR"
3737
export DESTDIR="$PWD/$DESTDIR"

‎wheel-check.sh‎

Lines changed: 30 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -73,19 +73,22 @@ if test "$mpiname" = "mpich"; then
7373
"$data"/bin/hydra_*
7474
)
7575
libraries=(
76-
"$data"/lib/lib*mpi.*
76+
"$data"/lib/lib*mpi.*.*
7777
)
78-
if ls "$data"/lib/*/libfabric.* > /dev/null 2>&1; then
78+
if ls "$data"/lib/*/libucp.* > /dev/null 2>&1; then
7979
libraries+=(
80-
"$data"/lib/*/libfabric.*
80+
"$data"/lib/*/libuc[mpst]*.*
8181
)
8282
fi
83-
if ls "$data"/lib/*/libucp.* > /dev/null 2>&1; then
83+
if ls "$data"/lib/*/libfabric.* > /dev/null 2>&1; then
8484
libraries+=(
85-
"$data"/lib/*/libuc[mpst]*.*
86-
"$data"/lib/*/ucx/libuc[mpst]*.*
85+
"$data"/lib/*/libfabric.*
8786
)
8887
fi
88+
plugins=(
89+
"$data"/lib/*/ucx/libuc[mpst]_*.*
90+
"$data"/lib/*/libfabric/lib*-fi.*
91+
)
8992
fi
9093

9194
if test "$mpiname" = "openmpi"; then
@@ -106,7 +109,7 @@ if test "$mpiname" = "openmpi"; then
106109
"$data"/bin/*_wrapper
107110
)
108111
libraries=(
109-
"$data"/lib/libmpi.*
112+
"$data"/lib/libmpi.*.*
110113
"$data"/lib/libopen-*.*
111114
)
112115
if test "${version%%.*}" -ge 5; then
@@ -117,28 +120,38 @@ if test "$mpiname" = "openmpi"; then
117120
"$data"/lib/openmpi/libprrte.*
118121
)
119122
fi
123+
runlibs+='|lib(z|util|event.*|hwloc)'$soregex
124+
runlibs+='|lib(open-(pal|rte)|pmix|prrte)'$soregex
120125
if ls "$data"/lib/*/libucp.* > /dev/null 2>&1; then
121126
libraries+=(
122127
"$data"/lib/*/libuc[mpst]*.*
123-
"$data"/lib/*/ucx/libuc[mpst]*.*
124128
)
125129
fi
126130
if ls "$data"/lib/*/libfabric.* > /dev/null 2>&1; then
127131
libraries+=(
128132
"$data"/lib/*/libfabric.*
129133
)
130134
fi
131-
runlibs+='|lib(z|util|event.*|hwloc)'$soregex
132-
runlibs+='|lib(open-(pal|rte)|pmix|prrte)'$soregex
135+
plugins=(
136+
"$data"/lib/*/ucx/libuc[mpst]_*.*
137+
"$data"/lib/*/libfabric/lib*-fi.*
138+
)
139+
fi
140+
141+
deplibs=$runlibs
142+
if test "$(uname)" = Linux; then
143+
deplibs+='|lib(psm2|uct_ib)'$soregex
144+
deplibs+='|lib(rdmacm|ibverbs|mlx5|efa)'$soregex
133145
fi
134146

135147
check-binary() {
136-
local dso=$1 out1="" out2=""
148+
local dso="$1" deplibs="${2:-$runlibs}"
149+
local out1="" out2=""
137150
echo checking "$dso"...
138151
test -f "$dso" || (printf "ERROR: file not found\n"; exit 1)
139152
out1="$(print-runpath "$dso" | grep -vE "$runpath" || true)"
140153
test -z "$out1" || printf "ERROR: RUNPATH\n%s\n" "$out1"
141-
out2="$(print-needed "$dso" | grep -vE "$runlibs" || true)"
154+
out2="$(print-needed "$dso" | grep -vE "$deplibs" || true)"
142155
test -z "$out2" || printf "ERROR: NEEDED\n%s\n" "$out2"
143156
test -z "$out1"
144157
test -z "$out2"
@@ -172,6 +185,11 @@ for lib in "${libraries[@]-}"; do
172185
test -n "$lib" || break
173186
check-binary "$lib"
174187
done
188+
for lib in "${plugins[@]-}"; do
189+
test -n "$lib" || break
190+
test -f "$lib" || break
191+
check-binary "$lib" "$deplibs"
192+
done
175193
if test -d "$pkgname$libsdir"; then
176194
echo checking "$pkgname$libsdir"...
177195
out=$(ls -A "$pkgname$libsdir")

0 commit comments

Comments
 (0)