Add option to bundle external libraries

Signed-off-by: Cristian Le <[email protected]>

Author: LecrisUT

README.md

@@ -268,6 +268,12 @@ wheel.repair.in-wheel = true
 # to each other and strict manylinux compliance is not required.
 wheel.repair.cross-wheel = false
 
+# A list of external library files that will be bundled in the wheel. Each entry
+# is treated as a regex pattern, and only the filenames are considered for the
+# match. The libraries are taken from the CMake dependency. The bundled
+# libraries are under `site-packages/${name}.libs`
+wheel.repair.bundle-external = []
+
 # If CMake is less than this value, backport a copy of FindPython. Set to 0
 # disable this, or the empty string.
 backport.find-python = "3.26.1"

src/scikit_build_core/repair_wheel/__init__.py

@@ -8,6 +8,8 @@
 import functools
 import os
 import platform
+import re
+import shutil
 import sysconfig
 from abc import ABC, abstractmethod
 from importlib import import_module
@@ -94,6 +96,28 @@ def __init_subclass__(cls) -> None:
         if cls._platform:
             WheelRepairer._platform_repairers[cls._platform] = cls
 
+    @functools.cached_property
+    def bundled_libs_path(self) -> Path:
+        """Staging path for the bundled library directory."""
+        return Path(self.wheel_dirs["platlib"]) / f"{self.name}.libs"
+
+    @functools.cached_property
+    def bundle_external(self) -> list[re.Pattern[str]]:
+        """List of compiled regex patterns of the library files to bundle."""
+        patterns = []
+        for pattern_str in self.settings.wheel.repair.bundle_external:
+            try:
+                pattern = re.compile(pattern_str)
+            except re.error as exc:
+                logger.warning(
+                    'Skipping "{pattern}" as an invalid pattern',
+                    pattern=pattern_str,
+                )
+                logger.debug(str(exc))
+                continue
+            patterns.append(pattern)
+        return patterns
+
     @functools.cached_property
     def configuration(self) -> Configuration:
         """Current file-api configuration."""
@@ -199,8 +223,91 @@ def get_library_dependencies(self, target: Target) -> list[Target]:
             dependencies.append(dep_target)
         return dependencies
 
+    def try_bundle(self, external_lib: Path) -> Path | None:
+        """
+        Try to bundle an external library file.
+
+        :param external_lib: path to actual external library to bundle
+        :returns: ``None`` if the library is not bundled, otherwise the path
+          to the bundled file
+        """
+        assert external_lib.is_absolute()
+        if not external_lib.exists():
+            logger.warning(
+                "External library file does not exist: {external_lib}",
+                external_lib=external_lib,
+            )
+            return None
+        if external_lib.is_dir():
+            logger.debug(
+                "Skip bundling directory: {external_lib}",
+                external_lib=external_lib,
+            )
+            return None
+        libname = external_lib.name
+        bundled_lib = self.bundled_libs_path / libname
+        if bundled_lib.exists():
+            # If we have already bundled the library no need to do it again
+            return bundled_lib
+        for pattern in self.bundle_external:
+            if pattern.match(libname):
+                logger.debug(
+                    'Bundling library matching "{pattern}": {external_lib}',
+                    external_lib=external_lib,
+                    pattern=pattern.pattern,
+                )
+                shutil.copy(external_lib, bundled_lib)
+                return bundled_lib
+        logger.debug(
+            "Skip bundling: {external_lib}",
+            external_lib=external_lib,
+        )
+        return None
+
+    def get_package_lib_path(
+        self, original_lib: Path, relative_to: Path | None = None
+    ) -> Path | None:
+        """
+        Get the file path of a library to be used.
+
+        This checks for the settings in ``settings.wheel.repair`` returning either:
+         - If the dependency should be skipped: ``None``
+         - If ``original_lib`` is a library in another wheel: a relative path to the original library file
+         - If ``original_lib`` is a library to be bundled: a relative path to the bundled library file
+
+        The relative paths are relative to ``relative_to`` or the ``platlib`` wheel path if not passed.
+        """
+        if not original_lib.is_absolute() or not original_lib.exists():
+            logger.debug(
+                "Could not handle {original_lib} because it is either relative or does not exist.",
+                original_lib=original_lib,
+            )
+            return None
+        if self.path_is_in_site_packages(original_lib):
+            # The other library is in another wheel
+            if not self.settings.wheel.repair.cross_wheel:
+                logger.debug(
+                    "Skipping {original_lib} because it is in another wheel.",
+                    original_lib=original_lib,
+                )
+                return None
+            final_lib = original_lib
+        # Otherwise, check if we need to bundle the external library
+        elif not self.bundle_external or not (
+            final_lib := self.try_bundle(original_lib)  # type: ignore[assignment]
+        ):
+            logger.debug(
+                "Skipping {original_lib} because it is not being bundled.",
+                original_lib=original_lib,
+            )
+            return None
+        return self.path_relative_site_packages(final_lib, relative_to=relative_to)
+
     def repair_wheel(self) -> None:
         """Repair the current wheel."""
+        if self.bundle_external:
+            self.bundled_libs_path.mkdir(exist_ok=True)
+
         for target in self.targets:
             if self._filter_targets:
                 if target.type == "STATIC_LIBRARY":

src/scikit_build_core/repair_wheel/rpath.py

@@ -91,31 +91,40 @@ def get_package_rpaths(self, target: Target, install_path: Path) -> list[str]:
                             # Skip empty rpaths. Most likely will have on at the end
                             continue
                         rpath = Path(rpath_str)
-                        if not self.path_is_in_site_packages(rpath):
-                            # Skip any paths that cannot be handled. We do not check for paths in
-                            # the build directory, it should be covered by `get_dependency_rpaths`
+                        # Relative paths should be covered by `get_dependency_rpaths` so we skip them.
+                        if not rpath.is_absolute():
+                            continue
+                        # Get the relative rpath to either the cross-wheel or bundled file
+                        if not (
+                            rpath := self.get_package_lib_path(  # type: ignore[assignment]
+                                rpath, relative_to=install_path
+                            )
+                        ):
                             continue
-                        rpath = self.path_relative_site_packages(rpath, install_path)
                         new_rpath_str = f"{self._origin_symbol}/{rpath}"
                         rpaths.append(new_rpath_str)
                     continue
                 # The remaining case should be a path
                 try:
                     # TODO: how to best catch if a string is a valid path?
                     rpath = Path(link_part)
-                    if not rpath.is_absolute():
-                        # Relative paths should be handled by `get_dependency_rpaths`
-                        continue
-                    rpath = self.path_relative_site_packages(rpath, install_path)
-                    new_rpath_str = f"{self._origin_symbol}/{rpath.parent}"
-                    rpaths.append(new_rpath_str)
                 except Exception as exc:
                     logger.warning(
                         "Could not parse link-library as a path: {fragment}\nexc = {exc}",
                         fragment=link_command.fragment,
                         exc=exc,
                     )
                     continue
+                if not rpath.is_absolute():
+                    # Relative paths should be covered by `get_dependency_rpaths` so we skip them.
+                    continue
+                # Get the relative rpath to either the cross-wheel or bundled file
+                if not (
+                    rpath := self.get_package_lib_path(rpath, relative_to=install_path)  # type: ignore[assignment]
+                ):
+                    continue
+                new_rpath_str = f"{self._origin_symbol}/{rpath.parent}"
+                rpaths.append(new_rpath_str)
         return rpaths
 
     def get_existing_rpaths(self, artifact: Path) -> list[str]:
@@ -168,10 +177,7 @@ def patch_target(self, target: Target) -> None:
                 dependency_rpaths = self.get_dependency_rpaths(target, install_path)
             else:
                 dependency_rpaths = []
-            if self.settings.wheel.repair.cross_wheel:
-                package_rpaths = self.get_package_rpaths(target, install_path)
-            else:
-                package_rpaths = []
+            package_rpaths = self.get_package_rpaths(target, install_path)
             existing_rpaths = self.get_existing_rpaths(artifact_path)
             logger.debug(
                 "Patching rpaths for artifact {artifact}\n"

src/scikit_build_core/repair_wheel/windows.py

@@ -5,6 +5,7 @@
 from __future__ import annotations
 
 import dataclasses
+import functools
 import os.path
 import textwrap
 from pathlib import Path
@@ -14,6 +15,8 @@
 from . import WheelRepairer, _get_buildenv_platlib
 
 if TYPE_CHECKING:
+    import re
+
     from ..file_api.model.codemodel import Target
 
 __all__ = ["WindowsWheelRepairer"]
@@ -52,8 +55,20 @@ def _skbuild_patch_dll_dir():
     dll_dirs: set[Path] = dataclasses.field(default_factory=set, init=False)
     """All dll paths used relative to ``platlib``."""
 
+    @functools.cached_property
+    def bundle_external(self) -> list[re.Pattern[str]]:
+        if self.settings.wheel.repair.bundle_external:
+            logger.warning("Bundling Windows dll files is not supported yet.")
+        return []
+
+    def try_bundle(self, external_lib: Path) -> Path | None:
+        # Everything should be gated by `bundle_external` so this should not be called
+        # TODO: figure out a better way to find the corresponding dll file of the linked lib file
+        raise NotImplementedError
+
     def get_dll_path_from_lib(self, lib_path: Path) -> Path | None:
         """Guess the dll path from lib path."""
+        # TODO: rework the logic of this to work with `try_bundle`
         dll_path = None
         platlib = Path(_get_buildenv_platlib())
         lib_path = lib_path.relative_to(platlib)
@@ -180,32 +195,27 @@ def get_package_dll(self, target: Target) -> list[Path]:
             try:
                 # TODO: how to best catch if a string is a valid path?
                 lib_path = Path(link_command.fragment)
-                if not lib_path.is_absolute():
-                    # If the link_command is a space-separated list of libraries, this should be skipped
-                    logger.debug(
-                        "Skipping non-absolute-path library: {fragment}",
-                        fragment=link_command.fragment,
-                    )
-                    continue
-                try:
-                    self.path_relative_site_packages(lib_path)
-                except ValueError:
-                    logger.debug(
-                        "Skipping library outside site-package path: {lib_path}",
-                        lib_path=lib_path,
-                    )
-                    continue
-                dll_path = self.get_dll_path_from_lib(lib_path)
-                if not dll_path:
-                    continue
-                dll_paths.append(dll_path.parent)
             except Exception as exc:
                 logger.warning(
                     "Could not parse link-library as a path: {fragment}\nexc = {exc}",
                     fragment=link_command.fragment,
                     exc=exc,
                 )
                 continue
+            if not lib_path.is_absolute():
+                # If the link_command is a space-separated list of libraries, this should be skipped
+                logger.debug(
+                    "Skipping non-absolute-path library: {fragment}",
+                    fragment=link_command.fragment,
+                )
+                continue
+            # TODO: Handle this better when revisiting `try_bundle`
+            if not self.get_package_lib_path(lib_path):
+                continue
+            dll_path = self.get_dll_path_from_lib(lib_path)
+            if not dll_path:
+                continue
+            dll_paths.append(dll_path.parent)
         return dll_paths
 
     def patch_target(self, target: Target) -> None:
@@ -214,10 +224,7 @@ def patch_target(self, target: Target) -> None:
             dependency_dlls = self.get_dependency_dll(target)
         else:
             dependency_dlls = []
-        if self.settings.wheel.repair.cross_wheel:
-            package_dlls = self.get_package_dll(target)
-        else:
-            package_dlls = []
+        package_dlls = self.get_package_dll(target)
 
         if not package_dlls and not dependency_dlls:
             logger.warning(

src/scikit_build_core/resources/scikit-build.schema.json

@@ -258,6 +258,13 @@
               "type": "boolean",
               "default": false,
               "description": "Patch the dynamic links to libraries in other wheels. BEWARE that this may result in incompatible wheels. Use this only if the wheels are strongly linked to each other and strict manylinux compliance is not required."
+            },
+            "bundle-external": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "A list of external library files that will be bundled in the wheel. Each entry is treated as a regex pattern, and only the filenames are considered for the match. The libraries are taken from the CMake dependency. The bundled libraries are under `site-packages/${name}.libs`"
             }
           },
           "description": "Wheel repair options"

src/scikit_build_core/settings/skbuild_model.py

@@ -192,6 +192,14 @@ class WheelRepair:
     not required.
     """
 
+    bundle_external: List[str] = dataclasses.field(default_factory=list)
+    """
+    A list of external library files that will be bundled in the wheel. Each entry
+    is treated as a regex pattern, and only the filenames are considered for the match.
+    The libraries are taken from the CMake dependency. The bundled libraries are under
+    `site-packages/${name}.libs`
+    """
+
 
 @dataclasses.dataclass
 class WheelSettings: