Skip to content

Commit cc3cbd2

Browse files
tbirdsotbirdso
authored
Merge pull request #236 from InsightSoftwareConsortium/amend-arm-builds
COMP: Consolidate Linux ARM build script
2 parents 7d2aace + a0123a5 commit cc3cbd2

9 files changed

+95
-78
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ ITKPythonPackage currently supports building wheels for the following platforms
8888
- MacOS 10.9+ x86_64 platforms
8989
- MacOS 11.0+ arm64 platforms
9090
- Linux glibc 2.17+ (E.g. Ubuntu 18.04+) x86_64 platforms
91-
- _Coming Soon: Linux ARM platforms_
91+
- Linux glibc 2.28+ (E.g. Ubuntu 20.04+) aarch64 (ARMv8) platforms
9292

9393
### What should I do if my target platform/architecture does not appear on the list above?
9494

‎requirements-dev.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1+
ninja==1.11.1
12
scikit-build==0.16.2

‎scripts/dockcross-manylinux-build-module-wheels.sh‎

Lines changed: 33 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,32 +28,36 @@
2828
# See https://github.com/dockcross/dockcross for available versions and tags.
2929
# For instance, `export MANYLINUX_VERSION=2014`
3030
#
31+
# `TARGET_ARCH`: Target architecture for which wheels should be built.
32+
# For instance, `export MANYLINUX_VERSION=aarch64`
33+
#
3134
# `IMAGE_TAG`: Specialized manylinux image tag to use for building.
32-
# For instance, `export IMAGE_TAG=20221205-459c9f0`
35+
# For instance, `export IMAGE_TAG=20221205-459c9f0`.
36+
# Tagged images are available at:
37+
# - https://github.com/dockcross/dockcross (x64 architecture)
38+
# - https://quay.io/organization/pypa (ARM architecture)
3339
#
3440
# `ITK_MODULE_PREQ`: Prerequisite ITK modules that must be built before the requested module.
3541
# See notes in `dockcross-manylinux-build-module-deps.sh`.
3642
#
3743
# `ITK_MODULE_NO_CLEANUP`: Option to skip cleanup steps.
3844
#
45+
# - `NO_SUDO`: Disable the use of superuser permissions for running docker.
46+
#
3947
########################################################################
4048

4149
# Handle case where the script directory is not the working directory
4250
script_dir=$(cd $(dirname $0) || exit 1; pwd)
4351
source "${script_dir}/dockcross-manylinux-set-vars.sh"
4452

45-
echo "ITK_MODULE_PREQ ${ITK_MODULE_PREQ}"
4653
if [[ -n ${ITK_MODULE_PREQ} ]]; then
54+
echo "Building module dependencies ${ITK_MODULE_PREQ}"
4755
source "${script_dir}/dockcross-manylinux-build-module-deps.sh"
4856
fi
4957

50-
# Generate dockcross scripts
51-
docker run --rm dockcross/manylinux${MANYLINUX_VERSION}-x64:${IMAGE_TAG} > /tmp/dockcross-manylinux-x64
52-
chmod u+x /tmp/dockcross-manylinux-x64
53-
58+
# Set up paths and variables for build
5459
mkdir -p $(pwd)/tools
5560
chmod 777 $(pwd)/tools
56-
# Build wheels
5761
mkdir -p dist
5862
DOCKER_ARGS="-v $(pwd)/dist:/work/dist/ -v ${script_dir}/..:/ITKPythonPackage -v $(pwd)/tools:/tools"
5963
DOCKER_ARGS+=" -e MANYLINUX_VERSION"
@@ -64,9 +68,28 @@ if [[ -n ${LD_LIBRARY_PATH} ]]; then
6468
done
6569
fi
6670

67-
/tmp/dockcross-manylinux-x64 \
68-
-a "$DOCKER_ARGS" \
69-
"/ITKPythonPackage/scripts/internal/manylinux-build-module-wheels.sh" "$@"
71+
if [[ "${TARGET_ARCH}" = "aarch64" ]]; then
72+
echo "Install aarch64 architecture emulation tools to perform build for ARM platform"
73+
74+
if [[ ! ${NO_SUDO} ]]; then
75+
docker_prefix="sudo"
76+
fi
77+
78+
${docker_prefix} docker run --privileged --rm tonistiigi/binfmt --install all
79+
80+
# Build wheels
81+
DOCKER_ARGS+=" -v $(pwd):/work/ --rm"
82+
${docker_prefix} docker run $DOCKER_ARGS ${CONTAINER_SOURCE} "/ITKPythonPackage/scripts/internal/manylinux-aarch64-build-module-wheels.sh" "$@"
83+
else
84+
# Generate dockcross scripts
85+
docker run --rm ${CONTAINER_SOURCE} > /tmp/dockcross-manylinux-x64
86+
chmod u+x /tmp/dockcross-manylinux-x64
87+
88+
# Build wheels
89+
/tmp/dockcross-manylinux-x64 \
90+
-a "$DOCKER_ARGS" \
91+
"/ITKPythonPackage/scripts/internal/manylinux-build-module-wheels.sh" "$@"
92+
fi
7093

7194
if [[ -z ${ITK_MODULE_NO_CLEANUP} ]]; then
7295
source "${script_dir}/dockcross-manylinux-cleanup.sh"

‎scripts/dockcross-manylinux-cleanup.sh‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,15 +15,32 @@
1515
# Format is `<org_name>/<module_name>@<module_tag>:<org_name>/<module_name>@<module_tag>:...`.
1616
# For instance, `export ITK_MODULE_PREQ=InsightSoftwareConsortium/[email protected]`
1717
#
18+
# - `NO_SUDO`: Disable the use of superuser permissions for removing directories.
19+
# `sudo` is required by default for cleanup on Github Actions runners.
20+
#
1821
########################################################################
1922

20-
echo "Cleaning up module dependencies"
23+
echo "Cleaning up artifacts from module build"
24+
25+
# ARM platform observed to require sudo for removing ITKPythonPackage sources
26+
rm_prefix=""
27+
if [[ ! ${NO_SUDO} ]]; then
28+
rm_prefix="sudo "
29+
fi
30+
2131
unlink oneTBB-prefix
22-
rm -rf ITKPythonPackage/
32+
${rm_prefix} rm -rf ITKPythonPackage/
33+
${rm_prefix} rm -rf tools/
34+
${rm_prefix} rm -rf _skbuild/
35+
${rm_prefix} rm -rf ./*.egg-info/
36+
${rm_prefix} rm -rf ./ITK-*-manylinux${MANYLINUX_VERSION}_${TARGET_ARCH}/
37+
${rm_prefix} rm -rf ./ITKPythonBuilds-linux-manylinux*${MANYLINUX_VERSION}*.tar.zst
2338

2439
if [[ -n ${ITK_MODULE_PREQ} ]]; then
2540
for MODULE_INFO in ${ITK_MODULE_PREQ//:/ }; do
2641
MODULE_NAME=`(echo ${MODULE_INFO} | cut -d'@' -f 1 | cut -d'/' -f 2)`
27-
sudo rm -rf ${MODULE_NAME}/
42+
${rm_prefix} rm -rf ${MODULE_NAME}/
2843
done
2944
fi
45+
46+
# Leave dist/ and download scripts intact

‎scripts/dockcross-manylinux-download-cache-and-build-module-wheels.sh‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,4 @@ chmod u+x dockcross-manylinux-download-cache.sh
7373

7474
echo "Building module wheels"
7575
set -- "${FORWARD_ARGS[@]}"; # Restore initial argument list
76-
if [[ "${MANYLINUX_VERSION}" = "_2_28_aarch64" ]]; then
77-
./ITKPythonPackage/scripts/manylinux_2_28_aarch64-build-module-wheels.sh "$@"
78-
else
79-
./ITKPythonPackage/scripts/dockcross-manylinux-build-module-wheels.sh "$@"
80-
fi
76+
./ITKPythonPackage/scripts/dockcross-manylinux-build-module-wheels.sh "$@"

‎scripts/dockcross-manylinux-download-cache.sh‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,17 @@ ${unzstd_exe} --version
7474
# -----------------------------------------------------------------------
7575
# Fetch build archive
7676

77-
TARBALL_SPECIALIZATION="-manylinux${MANYLINUX_VERSION:=_2_28}"
77+
MANYLINUX_VERSION=${MANYLINUX_VERSION:=_2_28}
78+
TARGET_ARCH=${TARGET_ARCH:=x64}
79+
80+
case ${TARGET_ARCH} in
81+
x64)
82+
TARBALL_SPECIALIZATION="-manylinux${MANYLINUX_VERSION}"
83+
;;
84+
*)
85+
TARBALL_SPECIALIZATION="-manylinux${MANYLINUX_VERSION}_${TARGET_ARCH}"
86+
;;
87+
esac
7888
TARBALL_NAME="ITKPythonBuilds-linux${TARBALL_SPECIALIZATION}.tar"
7989

8090
if [[ ! -f ${TARBALL_NAME}.zst ]]; then

‎scripts/dockcross-manylinux-set-vars.sh‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,29 @@ ITKPYTHONPACKAGE_TAG=${ITKPYTHONPACKAGE_TAG:=master}
3434
# See https://github.com/dockcross/dockcross for available versions and tags.
3535
MANYLINUX_VERSION=${MANYLINUX_VERSION:=_2_28}
3636

37+
# Target platform architecture (x64, aarch64)
38+
TARGET_ARCH=${TARGET_ARCH:=x64}
39+
3740
# Specialized manylinux image tag to use for building.
38-
if [[ ${MANYLINUX_VERSION} == _2_28 ]]; then
41+
if [[ ${MANYLINUX_VERSION} == _2_28 && ${TARGET_ARCH} == x64 ]]; then
3942
IMAGE_TAG=${IMAGE_TAG:=20221205-459c9f0}
43+
elif [[ ${MANYLINUX_VERSION} == _2_28 && ${TARGET_ARCH} == aarch64 ]]; then
44+
IMAGE_TAG=${IMAGE_TAG:=2022-11-19-1b19e81}
4045
elif [[ ${MANYLINUX_VERSION} == 2014 ]]; then
4146
IMAGE_TAG=${IMAGE_TAG:=20221201-fd49c08}
4247
else
4348
echo "Unknown manylinux version ${MANYLINUX_VERSION}"
4449
exit 1;
4550
fi
51+
52+
# Set container for requested version/arch/tag.
53+
if [[ ${TARGET_ARCH} == x64 ]]; then
54+
MANYLINUX_IMAGE_NAME=${MANYLINUX_IMAGE_NAME:="manylinux${MANYLINUX_VERSION}-${TARGET_ARCH}:${IMAGE_TAG}"}
55+
CONTAINER_SOURCE="dockcross/${MANYLINUX_IMAGE_NAME}"
56+
elif [[ ${TARGET_ARCH} == aarch64 ]]; then
57+
MANYLINUX_IMAGE_NAME=${MANYLINUX_IMAGE_NAME:="manylinux${MANYLINUX_VERSION}_${TARGET_ARCH}:${IMAGE_TAG}"}
58+
CONTAINER_SOURCE="quay.io/pypa/${MANYLINUX_IMAGE_NAME}"
59+
else
60+
echo "Unknown target architecture ${TARGET_ARCH}"
61+
exit 1;
62+
fi

‎scripts/internal/manylinux-build-module-wheels.sh‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -117,17 +117,16 @@ for PYBIN in "${PYBINARIES[@]}"; do
117117
|| exit 1
118118
done
119119

120-
if test "${ARCH}" == "x64"; then
121-
# Make sure auditwheel is installed for this python exe before importing
122-
# it in auditwheel_whitelist_monkeypatch.py
123-
sudo ${Python3_EXECUTABLE} -m pip install auditwheel
124-
for whl in dist/*linux_$(uname -m).whl; do
125-
# Repair wheel using monkey patch to exclude shared libraries provided in whitelist
126-
${Python3_EXECUTABLE} "${script_dir}/auditwheel_whitelist_monkeypatch.py" \
127-
repair ${whl} -w /work/dist/ --whitelist "${EXCLUDE_LIBS}"
128-
rm ${whl}
129-
done
130-
fi
120+
# Make sure auditwheel is installed for this python exe before importing
121+
# it in auditwheel_whitelist_monkeypatch.py
122+
sudo ${Python3_EXECUTABLE} -m pip install auditwheel
123+
for whl in dist/*linux*$(uname -m).whl; do
124+
# Repair wheel using monkey patch to exclude shared libraries provided in whitelist
125+
${Python3_EXECUTABLE} "${script_dir}/auditwheel_whitelist_monkeypatch.py" \
126+
repair ${whl} -w /work/dist/ --whitelist "${EXCLUDE_LIBS}"
127+
rm ${whl}
128+
done
129+
131130
if compgen -G "dist/itk*-linux*.whl" > /dev/null; then
132131
for itk_wheel in dist/itk*-linux*.whl; do
133132
mv ${itk_wheel} ${itk_wheel/linux/manylinux${MANYLINUX_VERSION}}

‎scripts/manylinux_2_28_aarch64-build-module-wheels.sh‎

Lines changed: 0 additions & 46 deletions
This file was deleted.

0 commit comments

Comments
 (0)