Skip to content

Commit 49f7e81

Browse files
tbirdsotbirdso
committed
ENH: Support iterative ITK module dependencies
Formalizes build steps on respective platforms for building modules that depend on other modules. Adapted from ITKBSplineGradient, ITKUltrasound, and other ITK modules whose build procedures include building at least one other ITK module before the target module itself.
1 parent b3f6d0e commit 49f7e81

9 files changed

+338
-27
lines changed

‎scripts/dockcross-manylinux-build-module-deps.sh‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
#!/bin/bash
2+
3+
########################################################################
4+
# Run this script in an ITK external module directory to generate
5+
# build artifacts for prerequisite ITK external modules.
6+
#
7+
# Module dependencies are built in a flat directory structure regardless
8+
# of recursive dependencies. Prerequisite sources are required to be passed
9+
# in the order in which they should be built.
10+
# For example, if ITKTargetModule depends on ITKTargetModuleDep2 which
11+
# depends on ITKTargetModuleDep1, the output directory structure
12+
# will look like this:
13+
#
14+
# / ITKTargetModule
15+
# -- / ITKTargetModuleDep1
16+
# -- / ITKTargetModuleDep2
17+
# ..
18+
#
19+
# ===========================================
20+
# ENVIRONMENT VARIABLES
21+
#
22+
# - `ITK_MODULE_PREQ`: Prerequisite ITK modules that must be built before the requested module.
23+
# Format is `<org_name>/<module_name>@<module_tag>:<org_name>/<module_name>@<module_tag>:...`.
24+
# For instance, `export ITK_MODULE_PREQ=InsightSoftwareConsortium/[email protected]`
25+
#
26+
########################################################################
27+
28+
# Initialize variables
29+
30+
script_dir=$(cd $(dirname $0) || exit 1; pwd)
31+
if [[ ! -f "${script_dir}/dockcross-manylinux-download-cache-and-build-module-wheels.sh" ]]; then
32+
echo "Could not find download script to use for building module dependencies!"
33+
exit 1
34+
fi
35+
36+
source "${script_dir}/dockcross-manylinux-set-vars.sh"
37+
38+
# Temporarily update prerequisite environment variable to prevent infinite recursion.
39+
ITK_MODULE_PREQ_TOPLEVEL=${ITK_MODULE_PREQ}
40+
ITK_MODULE_NO_CLEANUP_TOPLEVEL=${ITK_MODULE_NO_CLEANUP}
41+
export ITK_MODULE_PREQ=""
42+
export ITK_MODULE_NO_CLEANUP="ON"
43+
44+
########################################################################
45+
# Build ITK module dependencies
46+
47+
for MODULE_INFO in ${ITK_MODULE_PREQ_TOPLEVEL//:/ }; do
48+
MODULE_ORG=`(echo ${MODULE_INFO} | cut -d'/' -f 1)`
49+
MODULE_NAME=`(echo ${MODULE_INFO} | cut -d'@' -f 1 | cut -d'/' -f 2)`
50+
MODULE_TAG=`(echo ${MODULE_INFO} | cut -d'@' -f 2)`
51+
52+
MODULE_UPSTREAM=https://github.com/${MODULE_ORG}/${MODULE_NAME}.git
53+
echo "Cloning from ${MODULE_UPSTREAM}"
54+
git clone ${MODULE_UPSTREAM}
55+
56+
# Reuse cached build archive instead of redownloading.
57+
# Build archives are usually ~2GB so it is reasonable to move
58+
# instead of redownloading.
59+
if [[ `(compgen -G ./ITKPythonBuilds-linux*.tar.zst)` ]]; then
60+
mv ITKPythonBuilds-linux*.tar.zst ${MODULE_NAME}/
61+
fi
62+
63+
pushd ${MODULE_NAME}
64+
git checkout ${MODULE_TAG}
65+
cp ../dockcross-manylinux-download-cache-and-build-module-wheels.sh .
66+
if [[ -d ../ITKPythonPackage ]]; then
67+
ln -s ../ITKPythonPackage
68+
ln -s ./ITKPythonPackage/oneTBB-prefix
69+
fi
70+
71+
echo "Building module dependency ${MODULE_NAME}"
72+
./dockcross-manylinux-download-cache-and-build-module-wheels.sh $@
73+
popd
74+
75+
echo "Cleaning up module dependency"
76+
cp ./${MODULE_NAME}/include/* include/
77+
78+
# Cache build archive
79+
if [[ `(compgen -G ./ITKPythonBuilds-linux*.tar.zst)` ]]; then
80+
rm -f ./${MODULE_NAME}/ITKPythonBuilds-linux*.tar.zst
81+
else
82+
mv ./${MODULE_NAME}/ITKPythonBuilds-linux*.tar.zst .
83+
fi
84+
85+
# Cache ITKPythonPackage build scripts
86+
if [[ ! -d ./ITKPythonPackage ]]; then
87+
mv ./${MODULE_NAME}/ITKPythonPackage .
88+
ln -s ./ITKPythonPackage/oneTBB-prefix .
89+
fi
90+
91+
done
92+
93+
# Restore environment variable
94+
export ITK_MODULE_PREQ=${ITK_MODULE_PREQ_TOPLEVEL}
95+
ITK_MODULE_PREQ_TOPLEVEL=""
96+
export ITK_MODULE_NO_CLEANUP=${ITK_MODULE_NO_CLEANUP_TOPLEVEL}
97+
ITK_MODULE_NO_CLEANUP_TOPLEVEL=""
98+
99+
# Summarize disk usage for debugging
100+
du -sh ./* | sort -hr | head -n 20
101+
102+
echo "Done building ITK external module dependencies"

‎scripts/dockcross-manylinux-build-module-wheels.sh‎

Lines changed: 40 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,52 +3,66 @@
33
# Run this script to build the Python wheel packages for Linux for an ITK
44
# external module.
55
#
6+
# ========================================================================
7+
# PARAMETERS
8+
#
69
# Versions can be restricted by passing them in as arguments to the script
710
# For example,
811
#
912
# scripts/dockcross-manylinux-build-module-wheels.sh cp39
1013
#
11-
# Shared libraries can be included in the wheel by exporting them to LD_LIBRARY_PATH before
12-
# running this script.
13-
#
14+
# ===========================================
15+
# ENVIRONMENT VARIABLES
16+
#
17+
# These variables are set with the `export` bash command before calling the script.#
1418
# For example,
1519
#
16-
# export LD_LIBRARY_PATH="/path/to/OpenCL.so:/path/to/OpenCL.so.1.2"
20+
# export ITK_PACKAGE_VERSION="v5.3.0"
1721
# scripts/dockcross-manylinux-build-module-wheels.sh cp39
1822
#
19-
# A specialized manylinux image and tag can be used by exporting
20-
# MANYLINUX_VERSION and IMAGE_TAG before running this script.
21-
# See https://github.com/dockcross/dockcross for available versions and tags.
23+
# `ITK_PACKAGE_VERSION`: ITKPythonBuilds archive tag to use for ITK build artifacts.
24+
# See https://github.com/InsightSoftwareConsortium/ITKPythonBuilds for available tags.
25+
# For instance, `export ITK_PACKAGE_VERSION=v5.3.0`.
26+
#
27+
# `LD_LIBRARY_PATH`: Shared libraries to be included in the resulting wheel.
28+
# For instance, `export LD_LIBRARY_PATH="/path/to/OpenCL.so:/path/to/OpenCL.so.1.2"`
29+
#
30+
# `MANYLINUX_VERSION`: Specialized manylinux image to use for building. Default is _2_28.
31+
# See https://github.com/dockcross/dockcross for available versions and tags.
32+
# For instance, `export MANYLINUX_VERSION=2014`
2233
#
23-
# For example,
34+
# `IMAGE_TAG`: Specialized manylinux image tag to use for building.
35+
# For instance, `export IMAGE_TAG=20221205-459c9f0`
2436
#
25-
# export MANYLINUX_VERSION=2014
26-
# export IMAGE_TAG=20221205-459c9f0
27-
# scripts/dockcross-manylinux-build-module-wheels.sh cp39
37+
# `ITK_MODULE_PREQ`: Prerequisite ITK modules that must be built before the requested module.
38+
# Format is `<org_name>/<module_name>@<module_tag>:<org_name>/<module_name>@<module_tag>:...`.
39+
# For instance, `export ITK_MODULE_PREQ=InsightSoftwareConsortium/[email protected]`
40+
#
41+
# `ITKPYTHONPACKAGE_ORG`: Github organization for fetching ITKPythonPackage build scripts.
42+
#
43+
# `ITKPYTHONPACKAGE_TAG`: ITKPythonPackage tag for fetching build scripts.
44+
#
45+
# `ITK_MODULE_NO_CLEANUP`: Option to skip cleanup steps.
2846
#
2947

30-
MANYLINUX_VERSION=${MANYLINUX_VERSION:=_2_28}
48+
# Handle case where the script directory is not the working directory
49+
script_dir=$(cd $(dirname $0) || exit 1; pwd)
50+
source "${script_dir}/dockcross-manylinux-set-vars.sh"
3151

32-
if [[ ${MANYLINUX_VERSION} == _2_28 ]]; then
33-
IMAGE_TAG=${IMAGE_TAG:=20221205-459c9f0}
34-
elif [[ ${MANYLINUX_VERSION} == 2014 ]]; then
35-
IMAGE_TAG=${IMAGE_TAG:=20221201-fd49c08}
36-
else
37-
echo "Unknown manylinux version ${MANYLINUX_VERSION}"
38-
exit 1;
52+
echo "ITK_MODULE_PREQ ${ITK_MODULE_PREQ}"
53+
if [[ -n ${ITK_MODULE_PREQ} ]]; then
54+
source "${script_dir}/dockcross-manylinux-build-module-deps.sh"
3955
fi
4056

4157
# Generate dockcross scripts
4258
docker run --rm dockcross/manylinux${MANYLINUX_VERSION}-x64:${IMAGE_TAG} > /tmp/dockcross-manylinux-x64
4359
chmod u+x /tmp/dockcross-manylinux-x64
4460

45-
script_dir=$(cd $(dirname $0) || exit 1; pwd)
46-
4761
mkdir -p $(pwd)/tools
4862
chmod 777 $(pwd)/tools
4963
# Build wheels
5064
mkdir -p dist
51-
DOCKER_ARGS="-v $(pwd)/dist:/work/dist/ -v $script_dir/..:/ITKPythonPackage -v $(pwd)/tools:/tools"
65+
DOCKER_ARGS="-v $(pwd)/dist:/work/dist/ -v ${script_dir}/..:/ITKPythonPackage -v $(pwd)/tools:/tools"
5266
DOCKER_ARGS+=" -e MANYLINUX_VERSION"
5367
# Mount any shared libraries
5468
if [[ -n ${LD_LIBRARY_PATH} ]]; then
@@ -60,3 +74,7 @@ fi
6074
/tmp/dockcross-manylinux-x64 \
6175
-a "$DOCKER_ARGS" \
6276
"/ITKPythonPackage/scripts/internal/manylinux-build-module-wheels.sh" "$@"
77+
78+
if [[ -z ${ITK_MODULE_NO_CLEANUP} ]]; then
79+
source "${script_dir}/dockcross-manylinux-cleanup.sh"
80+
fi

‎scripts/dockcross-manylinux-cleanup.sh‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/bash
2+
3+
########################################################################
4+
# Run this script in an ITK external module directory to clean up
5+
# build artifacts for prerequisite ITK external modules.
6+
# Typically required for building multiple module wheels in the same
7+
# directory, such as using different toolsets or targeting different
8+
# architectures.
9+
#
10+
# ===========================================
11+
# ENVIRONMENT VARIABLES
12+
#
13+
# - `ITK_MODULE_PREQ`: Prerequisite ITK modules that must be built before the requested module.
14+
# Format is `<org_name>/<module_name>@<module_tag>:<org_name>/<module_name>@<module_tag>:...`.
15+
# For instance, `export ITK_MODULE_PREQ=InsightSoftwareConsortium/[email protected]`
16+
#
17+
########################################################################
18+
19+
echo "Cleaning up module dependencies"
20+
unlink oneTBB-prefix
21+
rm -rf ITKPythonPackage/
22+
23+
if [[ -n ${ITK_MODULE_PREQ} ]]; then
24+
for MODULE_INFO in ${ITK_MODULE_PREQ//:/ }; do
25+
MODULE_NAME=`(echo ${MODULE_INFO} | cut -d'@' -f 1 | cut -d'/' -f 2)`
26+
sudo rm -rf ${MODULE_NAME}/
27+
done
28+
fi

‎scripts/dockcross-manylinux-download-cache.sh‎

100644100755
File mode changed.

‎scripts/dockcross-manylinux-set-vars.sh‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/bin/bash
2+
3+
# Run this script to set enviroment variables used in building the
4+
# ITK Python wheel packages for Linux.
5+
#
6+
# ENVIRONMENT VARIABLES
7+
# These environment variables will be populated by the script when invoked with `source`
8+
# if their value is not set with `export` before invocation.
9+
# For example,
10+
#
11+
# export ITK_PACKAGE_VERSION=v5.3.0
12+
# scripts/dockcross-manylinux-set-vars.sh cp39
13+
#
14+
15+
########################################################################
16+
# ITKPythonBuilds parameters
17+
18+
# ITKPythonBuilds archive tag to use for ITK build artifacts.
19+
# See https://github.com/insightSoftwareConsortium/ITKpythonbuilds for available tags.
20+
ITK_PACKAGE_VERSION=${ITK_PACKAGE_VERSION:=v5.3.0}
21+
22+
# Github organization for fetching ITKPythonPackage build scripts
23+
ITKPYTHONPACKAGE_ORG=${ITKPYTHONPACKAGE_ORG:=InsightSoftwareConsortium}
24+
25+
# ITKPythonPackage tag for fetching build scripts
26+
ITKPYTHONPACKAGE_TAG=${ITKPYTHONPACKAGE_TAG:=master}
27+
28+
########################################################################
29+
# Docker image parameters
30+
31+
# Specialized manylinux image to use for building. Default is _2_28.
32+
# See https://github.com/dockcross/dockcross for available versions and tags.
33+
MANYLINUX_VERSION=${MANYLINUX_VERSION:=_2_28}
34+
35+
# Specialized manylinux image tag to use for building.
36+
if [[ ${MANYLINUX_VERSION} == _2_28 ]]; then
37+
IMAGE_TAG=${IMAGE_TAG:=20221205-459c9f0}
38+
elif [[ ${MANYLINUX_VERSION} == 2014 ]]; then
39+
IMAGE_TAG=${IMAGE_TAG:=20221201-fd49c08}
40+
else
41+
echo "Unknown manylinux version ${MANYLINUX_VERSION}"
42+
exit 1;
43+
fi

‎scripts/macpython-build-module-deps.sh‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
#!/bin/bash
2+
3+
########################################################################
4+
# Run this script in an ITK external module directory to generate
5+
# build artifacts for prerequisite ITK MacOS modules.
6+
#
7+
# Module dependencies are built in a flat directory structure regardless
8+
# of recursive dependencies. Prerequisite sources are required to be passed
9+
# in the order in which they should be built.
10+
# For example, if ITKTargetModule depends on ITKTargetModuleDep2 which
11+
# depends on ITKTargetModuleDep1, the output directory structure
12+
# will look like this:
13+
#
14+
# / ITKTargetModule
15+
# -- / ITKTargetModuleDep1
16+
# -- / ITKTargetModuleDep2
17+
# ..
18+
#
19+
# ===========================================
20+
# ENVIRONMENT VARIABLES
21+
#
22+
# - `ITK_MODULE_PREQ`: Prerequisite ITK modules that must be built before the requested module.
23+
# Format is `<org_name>/<module_name>@<module_tag>:<org_name>/<module_name>@<module_tag>:...`.
24+
# For instance, `export ITK_MODULE_PREQ=InsightSoftwareConsortium/[email protected]`
25+
#
26+
########################################################################
27+
28+
script_dir=$(cd $(dirname $0) || exit 1; pwd)
29+
if [[ ! -f "${script_dir}/macpython-download-cache-and-build-module-wheels.sh" ]]; then
30+
echo "Could not find download script to use for building module dependencies!"
31+
exit 1
32+
fi
33+
34+
# Temporarily update prerequisite environment variable to prevent infinite recursion.
35+
ITK_MODULE_PREQ_TOPLEVEL=${ITK_MODULE_PREQ}
36+
ITK_USE_LOCAL_PYTHON_TOPLEVEL=${ITK_USE_LOCAL_PYTHON}
37+
export ITK_MODULE_PREQ=""
38+
export ITK_USE_LOCAL_PYTHON="ON"
39+
40+
########################################################################
41+
echo "Building ITK module dependencies"
42+
43+
for MODULE_INFO in ${ITK_MODULE_PREQ_TOPLEVEL//:/ }; do
44+
MODULE_ORG=`(echo ${MODULE_INFO} | cut -d'/' -f 1)`
45+
MODULE_NAME=`(echo ${MODULE_INFO} | cut -d'@' -f 1 | cut -d'/' -f 2)`
46+
MODULE_TAG=`(echo ${MODULE_INFO} | cut -d'@' -f 2)`
47+
48+
MODULE_UPSTREAM=https://github.com/${MODULE_ORG}/${MODULE_NAME}.git
49+
echo "Cloning from ${MODULE_UPSTREAM}"
50+
git clone ${MODULE_UPSTREAM}
51+
52+
pushd ${MODULE_NAME}
53+
git checkout ${MODULE_TAG}
54+
cp ${script_dir}/macpython-download-cache-and-build-module-wheels.sh .
55+
echo "Building dependency ${MODULE_NAME}"
56+
./macpython-download-cache-and-build-module-wheels.sh $@
57+
popd
58+
59+
cp ./${MODULE_NAME}/include/* include/
60+
rm -f ./${MODULE_NAME}/ITKPythonBuilds-macosx.tar.zst
61+
done
62+
63+
# Restore environment variable
64+
export ITK_MODULE_PREQ=${ITK_MODULE_PREQ_TOPLEVEL}
65+
export ITK_USE_LOCAL_PYTHON=${ITK_USE_LOCAL_PYTHON_TOPLEVEL}
66+
ITK_MODULE_PREQ_TOPLEVEL=""
67+
ITK_USE_LOCAL_PYTHON_TOPLEVEL=""
68+
69+
# Summarize disk usage for debugging
70+
du -sh ./* | sort -hr | head -n 20
71+
72+
echo "Done building ITK external module dependencies"

‎scripts/macpython-build-module-wheels.sh‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,17 @@
1414
# scripts/macpython-build-module-wheels.sh 3.7 3.9
1515
#
1616

17+
18+
# -----------------------------------------------------------------------
19+
# (Optional) Build ITK module dependencies
20+
21+
script_dir=$(cd $(dirname $0) || exit 1; pwd)
22+
23+
echo "ITK_MODULE_PREQ ${ITK_MODULE_PREQ}"
24+
if [[ -n ${ITK_MODULE_PREQ} ]]; then
25+
source "${script_dir}/macpython-build-module-deps.sh"
26+
fi
27+
1728
# -----------------------------------------------------------------------
1829
# These variables are set in common script:
1930
#
@@ -22,7 +33,6 @@ MACPYTHON_PY_PREFIX=""
2233
SCRIPT_DIR=""
2334
VENVS=()
2435

25-
script_dir=$(cd $(dirname $0) || exit 1; pwd)
2636
source "${script_dir}/macpython-build-common.sh"
2737
# -----------------------------------------------------------------------
2838

0 commit comments

Comments
 (0)