Merge branch 'master' into release

Author: thewtex

CMakeLists.txt

@@ -271,6 +271,7 @@ if(ITKPythonPackage_SUPERBUILD)
         -DITK_WRAP_PYTHON:BOOL=ON
         -DITK_WRAP_unsigned_short:BOOL=ON
         -DITK_WRAP_double:BOOL=ON
+        -DITK_WRAP_complex_double:BOOL=ON
         -DITK_WRAP_IMAGE_DIMS:STRING=2;3;4
         -DITK_WRAP_DOC:BOOL=ON
         -DDOXYGEN_EXECUTABLE:FILEPATH=${DOXYGEN_EXECUTABLE}

scripts/dockcross-manylinux-download-cache-and-build-module-wheels.sh

@@ -3,6 +3,38 @@
 # This module should be pulled and run from an ITKModule root directory to generate the Linux python wheels of this module,
 # it is used by the azure-pipeline.yml file contained in ITKModuleTemplate: https://github.com/InsightSoftwareConsortium/ITKModuleTemplate
 
+# -----------------------------------------------------------------------
+# Script argument parsing
+#
+usage()
+{
+  echo "Usage:
+  dockcross-manylinux-download-cache-and-build-module-wheels
+    [ -h | --help ]           show usage
+    [ -c | --cmake_options ]  space-delimited string containing CMake options to forward to the module (e.g. \"-DBUILD_TESTING=OFF\")
+    [ -x | --exclude_libs ]   semicolon-delimited library names to exclude when repairing wheel (e.g. \"libcuda.so\")
+    [ python_version ]        build wheel for a specific python version. (e.g. cp39)"
+  exit 2
+}
+
+FORWARD_ARGS=("$@") # Store arguments to forward them later
+PARSED_ARGS=$(getopt -a -n dockcross-manylinux-download-cache-and-build-module-wheels \
+  -o hc:x: --long help,cmake_options:,exclude_libs: -- "$@")
+eval set -- "$PARSED_ARGS"
+
+while :
+do
+  case "$1" in
+    -h | --help) usage; break ;;
+    -c | --cmake_options) CMAKE_OPTIONS="$2" ; shift 2 ;;
+    -x | --exclude_libs) EXCLUDE_LIBS="$2" ; shift 2 ;;
+    --) shift; break ;;
+    *) echo "Unexpected option: $1.";
+       usage; break ;;
+  esac
+done
+# -----------------------------------------------------------------------
+
 # Packages distributed by github are in zstd format, so we need to download that binary to uncompress
 if [[ ! -f zstd-1.2.0-linux.tar.gz ]]; then
   curl https://data.kitware.com/api/v1/file/592dd8068d777f16d01e1a92/download -o zstd-1.2.0-linux.tar.gz
@@ -22,7 +54,7 @@ if [[ ! -f ./ITKPythonBuilds-linux.tar.zst ]]; then
   exit 255
 fi
 ./zstd-1.2.0-linux/bin/unzstd ./ITKPythonBuilds-linux.tar.zst -o ITKPythonBuilds-linux.tar
-if [ "$#" -le 1 ]; then
+if [ "$#" -lt 1 ]; then
   echo "Extracting all files";
   tar xf ITKPythonBuilds-linux.tar
 else
@@ -39,4 +71,5 @@ if [[ ! -f ./ITKPythonPackage/scripts/dockcross-manylinux-build-module-wheels.sh
 fi
 cp -a ITKPythonPackage/oneTBB-prefix ./
 
+set -- "${FORWARD_ARGS[@]}"; # Restore initial argument list
 ./ITKPythonPackage/scripts/dockcross-manylinux-build-module-wheels.sh "$@"

scripts/internal/auditwheel_whitelist_monkeypatch.py

@@ -0,0 +1,32 @@
+"""Patch auditwheel to skip actions on libraries specified in the whitelist.
+Other arguments are forwarded to auditwheel."""
+
+import argparse
+import sys
+
+from auditwheel.main import main
+from auditwheel.policy import _POLICIES as POLICIES
+
+
+def exclude_libs(whitelist):
+    # Do not include the following libraries when repairing wheels.
+    for lib in whitelist.split(';'):
+        for p in POLICIES:
+            p['lib_whitelist'].append(lib)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Driver script to build ITK Python module wheels.')
+    parser.add_argument('command', nargs=1, default='', help='auditwheel command (e.g. repair).')
+    parser.add_argument('wheel_file', nargs=1, default='', help='auditwheel wheel file.')
+    parser.add_argument('--wheel-dir', '-w', nargs=1, default='', type=str, help='Directory to store delocated wheels.')
+    parser.add_argument('--whitelist', nargs=1, default=None, type=str, help='Semicolon-delimited libraries to exclude from repaired wheel (e.g. libcuda.so)')
+    args = parser.parse_args()
+
+    if args.whitelist is not None:
+        whitelist = ';'.join(args.whitelist)
+        exclude_libs(whitelist)
+        # Do not forward whitelist args to auditwheel
+        sys.argv.remove('--whitelist')
+        sys.argv.remove(whitelist)
+
+    sys.exit(main())

scripts/internal/manylinux-build-module-wheels.sh

@@ -16,6 +16,39 @@
 #   /tmp/dockcross-manylinux-x64 -a "$DOCKER_ARGS" manylinux-build-module-wheels.sh
 #
 
+# -----------------------------------------------------------------------
+# Script argument parsing
+#
+usage()
+{
+  echo "Usage:
+  manylinux-build-module-wheels
+    [ -h | --help ]           show usage
+    [ -c | --cmake_options ]  space-separated string of CMake options to forward to the module (e.g. \"-DBUILD_TESTING=OFF\")
+    [ -x | --exclude_libs ]   semicolon-separated library names to exclude when repairing wheel (e.g. \"libcuda.so\")
+    [ python_version ]        build wheel for a specific python version. (e.g. cp39)"
+  exit 2
+}
+
+PARSED_ARGS=$(getopt -a -n dockcross-manylinux-download-cache-and-build-module-wheels \
+  -o hc:x: --long help,cmake_options:,exclude_libs: -- "$@")
+eval set -- "$PARSED_ARGS"
+
+while :
+do
+  case "$1" in
+    -h | --help) usage; break ;;
+    -c | --cmake_options) CMAKE_OPTIONS="$2" ; shift 2 ;;
+    -x | --exclude_libs) EXCLUDE_LIBS="$2" ; shift 2 ;;
+    --) shift; break ;;
+    *) echo "Unexpected option: $1.";
+       usage; break ;;
+  esac
+done
+
+PYTHON_VERSION="$@"
+# -----------------------------------------------------------------------
+
 # -----------------------------------------------------------------------
 # These variables are set in common script:
 #
@@ -66,13 +99,19 @@ for PYBIN in "${PYBINARIES[@]}"; do
       -DBUILD_TESTING:BOOL=OFF \
       -DPython3_EXECUTABLE:FILEPATH=${Python3_EXECUTABLE} \
       -DPython3_INCLUDE_DIR:PATH=${Python3_INCLUDE_DIR} \
+      ${CMAKE_OPTIONS} \
     || exit 1
     ${PYBIN}/python setup.py clean
 done
 
 if test "${ARCH}" == "x64"; then
+  # Make sure auditwheel is installed for this python exe before importing
+  # it in auditwheel_whitelist_monkeypatch.py
+  sudo ${Python3_EXECUTABLE} -m pip install auditwheel
   for whl in dist/*linux_$(uname -p).whl; do
-    auditwheel repair ${whl} -w /work/dist/
+    # Repair wheel using monkey patch to exclude shared libraries provided in whitelist
+    ${Python3_EXECUTABLE} "${script_dir}/auditwheel_whitelist_monkeypatch.py" \
+      repair ${whl} -w /work/dist/ --whitelist "${EXCLUDE_LIBS}"
     rm ${whl}
   done
 fi

scripts/windows_build_module_wheels.py

@@ -7,6 +7,7 @@
 import glob
 import sys
 import argparse
+import shutil
 
 SCRIPT_DIR = os.path.dirname(__file__)
 ROOT_DIR = os.path.abspath(os.getcwd())
@@ -19,6 +20,19 @@
 from wheel_builder_utils import push_dir, push_env
 from windows_build_common import DEFAULT_PY_ENVS, venv_paths
 
+def install_and_import(package):
+    """
+    Install package with pip and import in current script.
+    """
+    import importlib
+    try:
+        importlib.import_module(package)
+    except ImportError:
+        import pip
+        pip.main(['install', package])
+    finally:
+        globals()[package] = importlib.import_module(package)
+
 def build_wheels(py_envs=DEFAULT_PY_ENVS, cleanup=True, cmake_options=[]):
     for py_env in py_envs:
         python_executable, \
@@ -64,33 +78,65 @@ def build_wheels(py_envs=DEFAULT_PY_ENVS, cleanup=True, cmake_options=[]):
             if cleanup:
                 check_call([python_executable, "setup.py", "clean"])
 
-
-def fixup_wheel(py_envs, filepath, lib_paths:str=''):
-    lib_paths = lib_paths.strip() if lib_paths.isspace() else lib_paths.strip() + ";"
-    lib_paths += "C:/P/IPP/oneTBB-prefix/bin"
+def rename_wheel_init(py_env, filepath):
+    """
+    Rename module __init__ file in wheel.
+    This is required to prevent modules to override ITK's __init__ file on install.
+    If the module ships its own __init__ file, it is automatically renamed to
+    __init_{module_name}__ by this function. The renamed __init__ file will be executed
+    by ITK's __init__ file when loading ITK.
+    """
+    python_executable, python_include_dir, python_library, pip, ninja_executable, path = venv_paths(py_env)
+
+    # Get module info
+    install_and_import("pkginfo")
+    w = pkginfo.Wheel(filepath)
+    module_name = w.name.split("itk-")[-1]
+    module_version = w.version
+
+    dist_dir = os.path.dirname(filepath)
+    wheel_dir = os.path.join(dist_dir, "itk_" + module_name + "-" + module_version)
+    init_dir = os.path.join(wheel_dir, "itk")
+    init_file = os.path.join(init_dir, "__init__.py")
+
+    # Unpack wheel and rename __init__ file if it exists.
+    check_call([python_executable, "-m", "wheel", "unpack", filepath, "-d", dist_dir])
+    if os.path.isfile(init_file):
+        shutil.move(init_file, os.path.join(init_dir, "__init_" + module_name + "__.py"))
+    # Pack wheel and clean wheel folder
+    check_call([python_executable, "-m", "wheel", "pack", wheel_dir, "-d", dist_dir])
+    shutil.rmtree(wheel_dir)
+
+def fixup_wheel(py_envs, filepath, lib_paths:str='', exclude_libs:str=''):
+    lib_paths = ';'.join(["C:/P/IPP/oneTBB-prefix/bin",lib_paths.strip()]).strip(';')
     print(f'Library paths for fixup: {lib_paths}')
 
     py_env = py_envs[0]
 
     delve_wheel = os.path.join(ROOT_DIR, "venv-" + py_env, "Scripts", "delvewheel.exe")
     check_call([delve_wheel, "repair", "--no-mangle-all", "--add-path",
-        lib_paths, "--ignore-in-wheel", "-w",
+        lib_paths, "--no-dll", exclude_libs, "--ignore-in-wheel", "-w",
         os.path.join(ROOT_DIR, "dist"), filepath])
 
+    # The delve_wheel patch loading shared libraries is added to the module
+    # __init__ file. Rename this file here to prevent conflicts on installation.
+    # The renamed __init__ file will be executed when loading ITK.
+    rename_wheel_init(py_env, filepath)
 
-def fixup_wheels(py_envs, lib_paths:str=''):
+def fixup_wheels(py_envs, lib_paths:str='', exclude_libs:str=''):
     # shared library fix-up
     for wheel in glob.glob(os.path.join(ROOT_DIR, "dist", "*.whl")):
-        fixup_wheel(py_envs, wheel, ';'.join(lib_paths))
+        fixup_wheel(py_envs, wheel, lib_paths, exclude_libs)
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Driver script to build ITK Python module wheels.')
     parser.add_argument('--py-envs', nargs='+', default=DEFAULT_PY_ENVS,
             help='Target Python environment versions, e.g. "37-x64".')
     parser.add_argument('--no-cleanup', dest='cleanup', action='store_false', help='Do not clean up temporary build files.')
     parser.add_argument('--lib-paths', nargs=1, default='', help='Add semicolon-delimited library directories for delvewheel to include in the module wheel')
+    parser.add_argument('--exclude-libs', nargs=1, default='', help='Add semicolon-delimited library names that must not be included in the module wheel, e.g. "nvcuda.dll"')
     parser.add_argument('cmake_options', nargs='*', help='Extra options to pass to CMake, e.g. -DBUILD_SHARED_LIBS:BOOL=OFF')
     args = parser.parse_args()
 
     build_wheels(cleanup=args.cleanup, py_envs=args.py_envs, cmake_options=args.cmake_options)
-    fixup_wheels(args.py_envs, ';'.join(args.lib_paths))
+    fixup_wheels(args.py_envs, ';'.join(args.lib_paths), ';'.join(args.exclude_libs))