Merge pull request #169 from tbirdso/expose-library-paths

ENH: Add cross platform options for wheel repair with shared libraries

Author: thewtex

scripts/dockcross-manylinux-build-module-wheels.sh

@@ -7,6 +7,15 @@
 # For example,
 #
 #   scripts/dockcross-manylinux-build-module-wheels.sh cp39
+#
+# Shared libraries can be included in the wheel by exporting them to LD_LIBRARY_PATH before
+# running this script.
+# 
+# For example,
+#
+#   export LD_LIBRARY_PATH="/path/to/OpenCL.so:/path/to/OpenCL.so.1.2"
+#   scripts/dockcross-manylinux-build-module-wheels.sh cp39
+#
 
 # Generate dockcross scripts
 docker run --rm dockcross/manylinux2014-x64:20210823-d7b98b4 > /tmp/dockcross-manylinux-x64
@@ -19,6 +28,13 @@ chmod 777 $(pwd)/tools
 # Build wheels
 mkdir -p dist
 DOCKER_ARGS="-v $(pwd)/dist:/work/dist/ -v $script_dir/..:/ITKPythonPackage -v $(pwd)/tools:/tools"
+# Mount any shared libraries
+if [[ -n ${LD_LIBRARY_PATH} ]]; then
+  for libpath in ${LD_LIBRARY_PATH//:/ }; do
+	  DOCKER_ARGS+=" -v ${libpath}:/usr/lib64/$(basename -- ${libpath})"
+  done
+fi
+
 /tmp/dockcross-manylinux-x64 \
   -a "$DOCKER_ARGS" \
   "/ITKPythonPackage/scripts/internal/manylinux-build-module-wheels.sh" "$@"

scripts/internal/manylinux-build-module-wheels.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+# Run this script inside a dockcross container to build Python wheels for an ITK module
+
 # -----------------------------------------------------------------------
 # These variables are set in common script:
 #
@@ -10,8 +12,9 @@ script_dir=$(cd $(dirname $0) || exit 1; pwd)
 source "${script_dir}/manylinux-build-common.sh"
 # -----------------------------------------------------------------------
 
-# So auditwheel can find the libs
-export LD_LIBRARY_PATH=/work/oneTBB-prefix/lib64
+# Set up library paths in container so that shared libraries can be added to wheels
+sudo ldconfig
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/work/oneTBB-prefix/lib64:/usr/lib:/usr/lib64:/usr/local/lib:/usr/local/lib64
 
 # Compile wheels re-using standalone project and archive cache
 for PYBIN in "${PYBINARIES[@]}"; do

scripts/internal/manylinux-build-wheels.sh

@@ -20,7 +20,8 @@ pushd /work/ITK-source > /dev/null 2>&1
 popd > /dev/null 2>&1
 tbb_dir=/work/oneTBB-prefix/lib64/cmake/TBB
 # So auditwheel can find the libs
-export LD_LIBRARY_PATH=/work/oneTBB-prefix/lib64
+sudo ldconfig
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/work/oneTBB-prefix/lib64:/usr/lib:/usr/lib64
 
 SINGLE_WHEEL=0
 

scripts/macpython-build-module-wheels.sh

@@ -26,7 +26,7 @@ ${Python3_EXECUTABLE} -m pip install --no-cache delocate
 DELOCATE_LISTDEPS=${VENV}/bin/delocate-listdeps
 DELOCATE_WHEEL=${VENV}/bin/delocate-wheel
 # So delocate can find the libs
-export DYLD_LIBRARY_PATH=${script_dir}/../oneTBB-prefix/lib
+export DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}:${script_dir}/../oneTBB-prefix/lib
 
 # Compile wheels re-using standalone project and archive cache
 for VENV in "${VENVS[@]}"; do
@@ -66,7 +66,7 @@ for VENV in "${VENVS[@]}"; do
       -DPython3_EXECUTABLE:FILEPATH=${Python3_EXECUTABLE} \
       -DPython3_INCLUDE_DIR:PATH=${Python3_INCLUDE_DIR} \
     || exit 1
-    ${Python3_EXECUTABLE} setup.py clean
+    # ${Python3_EXECUTABLE} setup.py clean    # Permission denied
 done
 
 for wheel in $PWD/dist/*.whl; do

scripts/macpython-build-wheels.sh

@@ -44,7 +44,7 @@ DELOCATE_PATCH=${VENV}/bin/delocate-patch
 # Build standalone project and populate archive cache
 tbb_dir=$PWD/oneTBB-prefix/lib/cmake/TBB
 # So delocate can find the libs
-export DYLD_LIBRARY_PATH=$PWD/oneTBB-prefix/lib
+export DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}:$PWD/oneTBB-prefix/lib
 mkdir -p ITK-source
 pushd ITK-source > /dev/null 2>&1
   ${CMAKE_EXECUTABLE} -DITKPythonPackage_BUILD_PYTHON:PATH=0 \

scripts/windows_build_module_wheels.py

@@ -1,12 +1,12 @@
 from subprocess import check_call
 import os
+import glob
 import sys
 import argparse
 
 SCRIPT_DIR = os.path.dirname(__file__)
 ROOT_DIR = os.path.abspath(os.getcwd())
 
-
 print("SCRIPT_DIR: %s" % SCRIPT_DIR)
 print("ROOT_DIR: %s" % ROOT_DIR)
 
@@ -33,6 +33,7 @@ def build_wheels(py_envs=DEFAULT_PY_ENVS, cleanup=True, cmake_options=[]):
             check_call([pip, "install", "cmake"])
             check_call([pip, "install", "scikit_build"])
             check_call([pip, "install", "ninja"])
+            check_call([pip, "install", "delvewheel"])
 
             build_type = "Release"
             source_path = ROOT_DIR
@@ -59,12 +60,33 @@ def build_wheels(py_envs=DEFAULT_PY_ENVS, cleanup=True, cmake_options=[]):
             if cleanup:
                 check_call([python_executable, "setup.py", "clean"])
 
+
+def fixup_wheel(py_envs, filepath, lib_paths:str=''):
+    lib_paths = lib_paths.strip() if lib_paths.isspace() else lib_paths.strip() + ";"
+    lib_paths += "C:/P/IPP/oneTBB-prefix/bin"
+    print(f'Library paths for fixup: {lib_paths}')
+
+    py_env = py_envs[0]
+    
+    delve_wheel = os.path.join(ROOT_DIR, "venv-" + py_env, "Scripts", "delvewheel.exe")
+    check_call([delve_wheel, "repair", "--no-mangle-all", "--add-path",
+        lib_paths, "--ignore-in-wheel", "-w",
+        os.path.join(ROOT_DIR, "dist"), filepath])
+
+
+def fixup_wheels(py_envs, lib_paths:str=''):
+    # shared library fix-up
+    for wheel in glob.glob(os.path.join(ROOT_DIR, "dist", "*.whl")):
+        fixup_wheel(py_envs, wheel, ';'.join(lib_paths))
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Driver script to build ITK Python module wheels.')
     parser.add_argument('--py-envs', nargs='+', default=DEFAULT_PY_ENVS,
             help='Target Python environment versions, e.g. "37-x64".')
     parser.add_argument('--no-cleanup', dest='cleanup', action='store_false', help='Do not clean up temporary build files.')
+    parser.add_argument('--lib-paths', nargs=1, default='', help='Add semicolon-delimited library directories for delvewheel to include in the module wheel')
     parser.add_argument('cmake_options', nargs='*', help='Extra options to pass to CMake, e.g. -DBUILD_SHARED_LIBS:BOOL=OFF')
     args = parser.parse_args()
 
     build_wheels(cleanup=args.cleanup, py_envs=args.py_envs, cmake_options=args.cmake_options)
+    fixup_wheels(args.py_envs, ';'.join(args.lib_paths))

scripts/windows_build_wheels.py

@@ -219,21 +219,26 @@ def build_wheel(python_version, single_wheel=False,
             shutil.rmtree(
                 os.path.join(build_path, "Wrapping", "Generators", "CastXML"))
 
-def fixup_wheel(py_envs, filepath):
+def fixup_wheel(py_envs, filepath, lib_paths:str=''):
+    lib_paths = lib_paths.strip() if lib_paths.isspace() else lib_paths.strip() + ";"
+    lib_paths += "C:/P/IPP/oneTBB-prefix/bin"
+    print(f'Library paths for fixup: {lib_paths}')
+
     py_env = py_envs[0]
+    
     delve_wheel = os.path.join(ROOT_DIR, "venv-" + py_env, "Scripts", "delvewheel.exe")
     check_call([delve_wheel, "repair", "--no-mangle-all", "--add-path",
-        "C:/P/IPP/oneTBB-prefix/bin", "--ignore-in-wheel", "-w",
+        lib_paths, "--ignore-in-wheel", "-w",
         os.path.join(ROOT_DIR, "dist"), filepath])
 
 
-def fixup_wheels(single_wheel, py_envs):
+def fixup_wheels(single_wheel, py_envs, lib_paths:str=''):
     # TBB library fix-up
     tbb_wheel = "itk_core"
     if single_wheel:
         tbb_wheel = "itk"
     for wheel in glob.glob(os.path.join(ROOT_DIR, "dist", tbb_wheel + "*.whl")):
-        fixup_wheel(py_envs, wheel)
+        fixup_wheel(py_envs, wheel, lib_paths)
 
 
 def test_wheels(python_env):
@@ -299,13 +304,14 @@ def main(wheel_names=None):
     parser.add_argument('--py-envs', nargs='+', default=DEFAULT_PY_ENVS,
             help='Target Python environment versions, e.g. "37-x64".')
     parser.add_argument('--no-cleanup', dest='cleanup', action='store_false', help='Do not clean up temporary build files.')
+    parser.add_argument('--lib-paths', nargs=1, default='', help='Add semicolon-delimited library directories for delvewheel to include in the module wheel')
     parser.add_argument('cmake_options', nargs='*', help='Extra options to pass to CMake, e.g. -DBUILD_SHARED_LIBS:BOOL=OFF')
     args = parser.parse_args()
 
     build_wheels(single_wheel=args.single_wheel, cleanup=args.cleanup,
         py_envs=args.py_envs, wheel_names=wheel_names,
         cmake_options=args.cmake_options)
-    fixup_wheels(args.single_wheel, args.py_envs)
+    fixup_wheels(args.single_wheel, args.py_envs, ';'.join(args.lib_paths))
     for py_env in args.py_envs:
         test_wheels(py_env)