Merge pull request #185 from LucasGandel/linux-exclude-libs

Add argument parsing for Linux module wheels scripts

Author: thewtex

scripts/dockcross-manylinux-download-cache-and-build-module-wheels.sh

@@ -3,6 +3,38 @@
 # This module should be pulled and run from an ITKModule root directory to generate the Linux python wheels of this module,
 # it is used by the azure-pipeline.yml file contained in ITKModuleTemplate: https://github.com/InsightSoftwareConsortium/ITKModuleTemplate
 
+# -----------------------------------------------------------------------
+# Script argument parsing
+#
+usage()
+{
+  echo "Usage:
+  dockcross-manylinux-download-cache-and-build-module-wheels
+    [ -h | --help ]           show usage
+    [ -c | --cmake_options ]  space-delimited string containing CMake options to forward to the module (e.g. \"-DBUILD_TESTING=OFF\")
+    [ -x | --exclude_libs ]   semicolon-delimited library names to exclude when repairing wheel (e.g. \"libcuda.so\")
+    [ python_version ]        build wheel for a specific python version. (e.g. cp39)"
+  exit 2
+}
+
+FORWARD_ARGS=("$@") # Store arguments to forward them later
+PARSED_ARGS=$(getopt -a -n dockcross-manylinux-download-cache-and-build-module-wheels \
+  -o hc:x: --long help,cmake_options:,exclude_libs: -- "$@")
+eval set -- "$PARSED_ARGS"
+
+while :
+do
+  case "$1" in
+    -h | --help) usage; break ;;
+    -c | --cmake_options) CMAKE_OPTIONS="$2" ; shift 2 ;;
+    -x | --exclude_libs) EXCLUDE_LIBS="$2" ; shift 2 ;;
+    --) shift; break ;;
+    *) echo "Unexpected option: $1.";
+       usage; break ;;
+  esac
+done
+# -----------------------------------------------------------------------
+
 # Packages distributed by github are in zstd format, so we need to download that binary to uncompress
 if [[ ! -f zstd-1.2.0-linux.tar.gz ]]; then
   curl https://data.kitware.com/api/v1/file/592dd8068d777f16d01e1a92/download -o zstd-1.2.0-linux.tar.gz
@@ -22,7 +54,7 @@ if [[ ! -f ./ITKPythonBuilds-linux.tar.zst ]]; then
   exit 255
 fi
 ./zstd-1.2.0-linux/bin/unzstd ./ITKPythonBuilds-linux.tar.zst -o ITKPythonBuilds-linux.tar
-if [ "$#" -le 1 ]; then
+if [ "$#" -lt 1 ]; then
   echo "Extracting all files";
   tar xf ITKPythonBuilds-linux.tar
 else
@@ -39,4 +71,5 @@ if [[ ! -f ./ITKPythonPackage/scripts/dockcross-manylinux-build-module-wheels.sh
 fi
 cp -a ITKPythonPackage/oneTBB-prefix ./
 
+set -- "${FORWARD_ARGS[@]}"; # Restore initial argument list
 ./ITKPythonPackage/scripts/dockcross-manylinux-build-module-wheels.sh "$@"

scripts/internal/auditwheel_whitelist_monkeypatch.py

@@ -0,0 +1,32 @@
+"""Patch auditwheel to skip actions on libraries specified in the whitelist.
+Other arguments are forwarded to auditwheel."""
+
+import argparse
+import sys
+
+from auditwheel.main import main
+from auditwheel.policy import _POLICIES as POLICIES
+
+
+def exclude_libs(whitelist):
+    # Do not include the following libraries when repairing wheels.
+    for lib in whitelist.split(';'):
+        for p in POLICIES:
+            p['lib_whitelist'].append(lib)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Driver script to build ITK Python module wheels.')
+    parser.add_argument('command', nargs=1, default='', help='auditwheel command (e.g. repair).')
+    parser.add_argument('wheel_file', nargs=1, default='', help='auditwheel wheel file.')
+    parser.add_argument('--wheel-dir', '-w', nargs=1, default='', type=str, help='Directory to store delocated wheels.')
+    parser.add_argument('--whitelist', nargs=1, default=None, type=str, help='Semicolon-delimited libraries to exclude from repaired wheel (e.g. libcuda.so)')
+    args = parser.parse_args()
+
+    if args.whitelist is not None:
+        whitelist = ';'.join(args.whitelist)
+        exclude_libs(whitelist)
+        # Do not forward whitelist args to auditwheel
+        sys.argv.remove('--whitelist')
+        sys.argv.remove(whitelist)
+
+    sys.exit(main())

scripts/internal/manylinux-build-module-wheels.sh

@@ -16,6 +16,39 @@
 #   /tmp/dockcross-manylinux-x64 -a "$DOCKER_ARGS" manylinux-build-module-wheels.sh
 #
 
+# -----------------------------------------------------------------------
+# Script argument parsing
+#
+usage()
+{
+  echo "Usage:
+  manylinux-build-module-wheels
+    [ -h | --help ]           show usage
+    [ -c | --cmake_options ]  space-separated string of CMake options to forward to the module (e.g. \"-DBUILD_TESTING=OFF\")
+    [ -x | --exclude_libs ]   semicolon-separated library names to exclude when repairing wheel (e.g. \"libcuda.so\")
+    [ python_version ]        build wheel for a specific python version. (e.g. cp39)"
+  exit 2
+}
+
+PARSED_ARGS=$(getopt -a -n dockcross-manylinux-download-cache-and-build-module-wheels \
+  -o hc:x: --long help,cmake_options:,exclude_libs: -- "$@")
+eval set -- "$PARSED_ARGS"
+
+while :
+do
+  case "$1" in
+    -h | --help) usage; break ;;
+    -c | --cmake_options) CMAKE_OPTIONS="$2" ; shift 2 ;;
+    -x | --exclude_libs) EXCLUDE_LIBS="$2" ; shift 2 ;;
+    --) shift; break ;;
+    *) echo "Unexpected option: $1.";
+       usage; break ;;
+  esac
+done
+
+PYTHON_VERSION="$@"
+# -----------------------------------------------------------------------
+
 # -----------------------------------------------------------------------
 # These variables are set in common script:
 #
@@ -66,13 +99,19 @@ for PYBIN in "${PYBINARIES[@]}"; do
       -DBUILD_TESTING:BOOL=OFF \
       -DPython3_EXECUTABLE:FILEPATH=${Python3_EXECUTABLE} \
       -DPython3_INCLUDE_DIR:PATH=${Python3_INCLUDE_DIR} \
+      ${CMAKE_OPTIONS} \
     || exit 1
     ${PYBIN}/python setup.py clean
 done
 
 if test "${ARCH}" == "x64"; then
+  # Make sure auditwheel is installed for this python exe before importing
+  # it in auditwheel_whitelist_monkeypatch.py
+  sudo ${Python3_EXECUTABLE} -m pip install auditwheel
   for whl in dist/*linux_$(uname -p).whl; do
-    auditwheel repair ${whl} -w /work/dist/
+    # Repair wheel using monkey patch to exclude shared libraries provided in whitelist
+    ${Python3_EXECUTABLE} "${script_dir}/auditwheel_whitelist_monkeypatch.py" \
+      repair ${whl} -w /work/dist/ --whitelist "${EXCLUDE_LIBS}"
     rm ${whl}
   done
 fi