Skip to content

Commit e56c447

Browse files
dalcinldalcinl
committed
Build UCX and OFI with rdmacm-core and psm2 on manylinux_2_28
1 parent f589195 commit e56c447

File tree

5 files changed

+102
-32
lines changed

5 files changed

+102
-32
lines changed

‎.github/workflows/cd-wheel.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -179,8 +179,6 @@ jobs:
179179
SOURCE="$PWD/package/source"
180180
WORKDIR="$PWD/package/workdir"
181181
DESTDIR="$PWD/package/install"
182-
CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014
183-
CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
184182
185183
- id: sha256sum
186184
run: |

‎cibw-build-mpi.sh‎

Lines changed: 57 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,16 @@ case $(uname) in
1919
esac
2020

2121
if test "$(uname)" = Linux; then
22+
# shellcheck disable=SC1091
23+
PLATFORM_ID="$(source /etc/os-release && echo "${PLATFORM_ID:-}")"
2224
if test "${CIBUILDWHEEL}" = 1; then
2325
yum remove -y libatomic
26+
if test "$PLATFORM_ID" = "platform:el8"; then
27+
yum install -y rdma-core-devel
28+
if test "$(uname -m)" = x86_64; then
29+
yum install -y libpsm2-devel
30+
fi
31+
fi
2432
fi
2533
fi
2634
if test "$(uname)" = Darwin; then
@@ -84,6 +92,15 @@ if test -d "$MODSOURCE"/libfabric-*; then
8492
echo running configure for OFI
8593
oficonfigure="$OFISOURCE"/configure
8694
ofioptions=(--prefix="$PREFIX" --disable-static)
95+
if test -f /usr/lib*/librdmacm.so; then
96+
ofioptions+=(--enable-verbs=dl)
97+
fi
98+
if test -f /usr/lib*/libpsm2.so; then
99+
ofioptions+=(--enable-psm2=dl)
100+
fi
101+
if test -f /usr/lib*/libefa.so; then
102+
ofioptions+=(--enable-efa=dl)
103+
fi
87104
"$oficonfigure" "${ofioptions[@]}" \
88105
CFLAGS="${build_cflags[*]}" \
89106
LDFLAGS="${build_ldflags[*]}"
@@ -95,6 +112,9 @@ if test -d "$MODSOURCE"/libfabric-*; then
95112
done
96113
for pkg in "${DESTDIR}${PREFIX}"/lib/pkgconfig/libfabric*.pc; do
97114
sed -i "/prefix\s*=/s|\($PREFIX\)|$DESTDIR\1|g" "$pkg"
115+
for dep in rdmacm ibverbs psm2 efa; do
116+
sed -i "/Libs\.private:\s*/s|-l$dep\s*||g" "$pkg"
117+
done
98118
done
99119
fi
100120

@@ -181,6 +201,8 @@ if test "$mpiname" = "openmpi"; then
181201
--prefix="$PREFIX"
182202
--without-ucx
183203
--without-ofi
204+
--without-psm2
205+
--without-verbs
184206
--without-cuda
185207
--without-rocm
186208
--with-hwloc=internal
@@ -284,6 +306,7 @@ rm -f lib/libuc[mpst]*.a
284306
rm -f lib/libuc[mpst]*.la
285307
rm -f lib/ucx/libuc[mt]_*.a
286308
rm -f lib/ucx/libuc[mt]_*.la
309+
rm -f lib/ucx/libucx_perftest_*.*
287310
rm -fr lib/cmake/ucx
288311
rm -f lib/pkgconfig/ucx*.pc
289312
rm -fr share/ucx
@@ -297,15 +320,18 @@ for lib in libuc[mpst]*.so.?; do
297320
fi
298321
patchelf --set-rpath "\$ORIGIN" "$lib"
299322
done
300-
patchelf --add-rpath "\$ORIGIN/ucx" libucm.so.?
301-
patchelf --add-rpath "\$ORIGIN/ucx" libuct.so.?
302-
for lib in ucx/libuc[mt]_*.so.?; do
323+
if test -d ucx; then
324+
patchelf --add-rpath "\$ORIGIN/ucx" libuc[mpst].so.?
325+
fi
326+
for lib in ucx/libuc[mpst]_*.so.?; do
327+
if test -f "$lib"; then
328+
patchelf --set-rpath "\$ORIGIN" "$lib"
329+
patchelf --add-rpath "\$ORIGIN/.." "$lib"
330+
fi
303331
if test -f "$lib".*; then
304332
mv "$(dirname "$lib")/$(readlink "$lib")" "$lib"
305333
ln -srf "$lib" "${lib%.*}"
306334
fi
307-
patchelf --set-rpath "\$ORIGIN" "$lib"
308-
patchelf --add-rpath "\$ORIGIN/.." "$lib"
309335
done
310336

311337
} # fixup-ucx()
@@ -317,6 +343,8 @@ rm -fr include/rdma
317343
rm -f bin/fi_*
318344
rm -f lib/libfabric.a
319345
rm -f lib/libfabric.la
346+
rm -f lib/libfabric/lib*-fi.a
347+
rm -f lib/libfabric/lib*-fi.la
320348
rm -f lib/pkgconfig/libfabric.pc
321349
rm -f share/man/man?/fabric.?
322350
rm -f share/man/man?/fi_*.?
@@ -330,6 +358,15 @@ for lib in libfabric.so.?; do
330358
fi
331359
patchelf --set-rpath "\$ORIGIN" "$lib"
332360
done
361+
if test -d libfabric; then
362+
patchelf --add-rpath "\$ORIGIN/libfabric" libfabric.so.?
363+
fi
364+
for lib in libfabric/lib*-fi.so; do
365+
if test -f "$lib"; then
366+
patchelf --set-rpath "\$ORIGIN" "$lib"
367+
patchelf --set-rpath "\$ORIGIN/.." "$lib"
368+
fi
369+
done
333370

334371
} # fixup-ofi()
335372

@@ -350,6 +387,7 @@ rm -f lib/lib*mpi.la
350387
rm -f lib/lib*mpich*.*
351388
rm -f lib/lib*mpicxx.*
352389
rm -f lib/lib*mpifort.*
390+
rm -fr lib/cmake
353391
rm -fr lib/pkgconfig
354392
rm -fr share
355393

@@ -398,10 +436,16 @@ if test "$(uname)" = Linux; then
398436
if test -f libfabric.so; then
399437
mkdir -p "$mpiname"
400438
mv libfabric.* "$mpiname"
439+
if test -d libfabric; then
440+
mv libfabric "$mpiname"
441+
fi
401442
fi
402443
if test -f libucp.so; then
403444
mkdir -p "$mpiname"
404-
mv libuc[mpst]*.* ucx "$mpiname"
445+
mv libuc[mpst]*.* "$mpiname"
446+
if test -d ucx; then
447+
mv ucx "$mpiname"
448+
fi
405449
fi
406450
cd "${DESTDIR}${PREFIX}/lib"
407451
for lib in lib*.so; do
@@ -536,14 +580,6 @@ for exe in 'mpirun' 'ompi*' 'pmix*' 'prte*' 'opal*' 'orte*'; do
536580
done < <(find . -name "$exe" -type f)
537581
done
538582

539-
cd "${DESTDIR}${PREFIX}/lib"
540-
unset libraries
541-
for lib in 'lib*.so.*' 'lib*.*.dylib'; do
542-
while IFS= read -r filename
543-
do libraries+=("$(basename "$filename")")
544-
done < <(find . -name "$lib" -type f)
545-
done
546-
547583
if test "$(uname)" = Linux; then
548584
cd "${DESTDIR}${PREFIX}/bin"
549585
for exe in "${executables[@]}"; do
@@ -561,10 +597,16 @@ if test "$(uname)" = Linux; then
561597
if test -f libfabric.so; then
562598
mkdir -p "$mpiname"
563599
mv libfabric.* "$mpiname"
600+
if test -d libfabric; then
601+
mv libfabric "$mpiname"
602+
fi
564603
fi
565604
if test -f libucp.so; then
566605
mkdir -p "$mpiname"
567-
mv libuc[mpst]*.* ucx "$mpiname"
606+
mv libuc[mpst]*.* "$mpiname"
607+
if test -d ucx; then
608+
mv ucx "$mpiname"
609+
fi
568610
fi
569611
for lib in lib*.so; do
570612
patchelf --set-rpath "\$ORIGIN" "$lib"

‎patches/auditwheel.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,18 @@ def walk(topdir):
3131

3232
auditwheel.tools.walk = walk
3333

34+
exclude = (
35+
"rdmacm",
36+
"ibverbs",
37+
"mlx5",
38+
"psm2",
39+
"efa",
40+
)
41+
3442
if "repair" in sys.argv:
3543
sys.argv.append("--only-plat")
44+
for name in exclude:
45+
sys.argv.append("--exclude")
46+
sys.argv.append(f"lib{name}.so.*")
3647

3748
sys.exit(main())

‎wheel-build.sh‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,11 @@ export CIBW_BEFORE_ALL='bash {project}/cibw-build-mpi.sh'
1717
export CIBW_BEFORE_BUILD='bash {project}/cibw-patch-cmd.sh'
1818
export CIBW_TEST_COMMAND='bash {project}/cibw-check-mpi.sh'
1919
export CIBW_ENVIRONMENT_PASS='SOURCE WORKDIR DESTDIR'
20-
export CIBW_MANYLINUX_AARCH64_IMAGE=manylinux2014
21-
export CIBW_MANYLINUX_X86_64_IMAGE=manylinux2014
2220

2321
if test "$(uname)" = Linux; then
22+
# manylinuximage=manylinux2014
23+
# export CIBW_MANYLINUX_AARCH64_IMAGE=$manylinuximage
24+
# export CIBW_MANYLINUX_X86_64_IMAGE=$manylinuximage
2425
containerengine=$(basename "$(command -v podman || command -v docker)")
2526
export CIBW_CONTAINER_ENGINE=$containerengine
2627
export SOURCE="/project/$SOURCE"
@@ -31,7 +32,7 @@ fi
3132
if test "$(uname)" = Darwin; then
3233
export MACOSX_DEPLOYMENT_TARGET=11.0
3334
export CIBW_BUILD_FRONTEND='build'
34-
export CIBW_BUILD='pp310-*'
35+
export CIBW_BUILD='pp311-*'
3536
export SOURCE="$PWD/$SOURCE"
3637
export WORKDIR="$PWD/$WORKDIR"
3738
export DESTDIR="$PWD/$DESTDIR"

‎wheel-check.sh‎

Lines changed: 30 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -73,19 +73,22 @@ if test "$mpiname" = "mpich"; then
7373
"$data"/bin/hydra_*
7474
)
7575
libraries=(
76-
"$data"/lib/lib*mpi.*
76+
"$data"/lib/lib*mpi.*.*
7777
)
78-
if ls "$data"/lib/*/libfabric.* > /dev/null 2>&1; then
78+
if ls "$data"/lib/*/libucp.* > /dev/null 2>&1; then
7979
libraries+=(
80-
"$data"/lib/*/libfabric.*
80+
"$data"/lib/*/libuc[mpst]*.*
8181
)
8282
fi
83-
if ls "$data"/lib/*/libucp.* > /dev/null 2>&1; then
83+
if ls "$data"/lib/*/libfabric.* > /dev/null 2>&1; then
8484
libraries+=(
85-
"$data"/lib/*/libuc[mpst]*.*
86-
"$data"/lib/*/ucx/libuc[mpst]*.*
85+
"$data"/lib/*/libfabric.*
8786
)
8887
fi
88+
plugins=(
89+
"$data"/lib/*/ucx/libuc[mpst]_*.*
90+
"$data"/lib/*/libfabric/lib*-fi.*
91+
)
8992
fi
9093

9194
if test "$mpiname" = "openmpi"; then
@@ -106,7 +109,7 @@ if test "$mpiname" = "openmpi"; then
106109
"$data"/bin/*_wrapper
107110
)
108111
libraries=(
109-
"$data"/lib/libmpi.*
112+
"$data"/lib/libmpi.*.*
110113
"$data"/lib/libopen-*.*
111114
)
112115
if test "${version%%.*}" -ge 5; then
@@ -117,28 +120,38 @@ if test "$mpiname" = "openmpi"; then
117120
"$data"/lib/openmpi/libprrte.*
118121
)
119122
fi
123+
runlibs+='|lib(z|util|event.*|hwloc)'$soregex
124+
runlibs+='|lib(open-(pal|rte)|pmix|prrte)'$soregex
120125
if ls "$data"/lib/*/libucp.* > /dev/null 2>&1; then
121126
libraries+=(
122127
"$data"/lib/*/libuc[mpst]*.*
123-
"$data"/lib/*/ucx/libuc[mpst]*.*
124128
)
125129
fi
126130
if ls "$data"/lib/*/libfabric.* > /dev/null 2>&1; then
127131
libraries+=(
128132
"$data"/lib/*/libfabric.*
129133
)
130134
fi
131-
runlibs+='|lib(z|util|event.*|hwloc)'$soregex
132-
runlibs+='|lib(open-(pal|rte)|pmix|prrte)'$soregex
135+
plugins=(
136+
"$data"/lib/*/ucx/libuc[mpst]_*.*
137+
"$data"/lib/*/libfabric/lib*-fi.*
138+
)
139+
fi
140+
141+
deplibs=$runlibs
142+
if test "$(uname)" = Linux; then
143+
deplibs+='|lib(psm2|uct_ib)'$soregex
144+
deplibs+='|lib(rdmacm|ibverbs|mlx5|efa)'$soregex
133145
fi
134146

135147
check-binary() {
136-
local dso=$1 out1="" out2=""
148+
local dso="$1" deplibs="${2:-$runlibs}"
149+
local out1="" out2=""
137150
echo checking "$dso"...
138151
test -f "$dso" || (printf "ERROR: file not found\n"; exit 1)
139152
out1="$(print-runpath "$dso" | grep -vE "$runpath" || true)"
140153
test -z "$out1" || printf "ERROR: RUNPATH\n%s\n" "$out1"
141-
out2="$(print-needed "$dso" | grep -vE "$runlibs" || true)"
154+
out2="$(print-needed "$dso" | grep -vE "$deplibs" || true)"
142155
test -z "$out2" || printf "ERROR: NEEDED\n%s\n" "$out2"
143156
test -z "$out1"
144157
test -z "$out2"
@@ -172,6 +185,11 @@ for lib in "${libraries[@]-}"; do
172185
test -n "$lib" || break
173186
check-binary "$lib"
174187
done
188+
for lib in "${plugins[@]-}"; do
189+
test -n "$lib" || break
190+
test -f "$lib" || break
191+
check-binary "$lib" "$deplibs"
192+
done
175193
if test -d "$pkgname$libsdir"; then
176194
echo checking "$pkgname$libsdir"...
177195
out=$(ls -A "$pkgname$libsdir")

0 commit comments

Comments
 (0)