From 91949d436970eda049f985b29221cd72d165a22b Mon Sep 17 00:00:00 2001 From: Cristian Le Date: Mon, 17 Mar 2025 19:18:31 +0100 Subject: [PATCH 1/6] Document dynamic linking Signed-off-by: Cristian Le --- docs/conf.py | 3 +- docs/guide/dynamic_link.md | 70 ++++++++++++++++++++++++++++++++++++++ docs/index.md | 1 + 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 docs/guide/dynamic_link.md diff --git a/docs/conf.py b/docs/conf.py index 06452df3f..23b35cef8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -90,6 +90,7 @@ intersphinx_mapping = { + "cmake": ("https://cmake.org/cmake/help/latest/", None), "python": ("https://docs.python.org/3", None), "packaging": ("https://packaging.pypa.io/en/stable", None), "setuptools": ("https://setuptools.pypa.io/en/latest", None), @@ -99,7 +100,7 @@ val[0] for key, val in intersphinx_mapping.items() # Only works with RTD hosted intersphinx - if key not in ("hatchling", "python") + if key not in ("hatchling", "python", "cmake") ] nitpick_ignore = [ diff --git a/docs/guide/dynamic_link.md b/docs/guide/dynamic_link.md new file mode 100644 index 000000000..e1eeb3f79 --- /dev/null +++ b/docs/guide/dynamic_link.md @@ -0,0 +1,70 @@ +# Dynamic linking + +If you want to support dynamic linkages between python projects or system +libraries, you will likely encounter some issues in making sure the compiled +libraries/python bindings work after the wheel is created and the python project +is installed on the system. The most common issue are the missing hints pointing +to where the runtime libraries are located, specifically `RPATH` on Linux and +MacOS systems, and `PATH`/`os.add_dll_directory` on Windows systems. Here are +some recommendations on how to address them. + +## Link to the static libraries + +The easiest solution is to make sure you link to the static libraries +counterparts during the CMake build. How to achieve this depends on the specific +dependency, how it is imported in the CMake project and how the dependency is +packaged in each ecosystem. + +For example for [Boost][FindBoost] this is controlled via the variable +`Boost_USE_STATIC_LIBS`. + +[FindBoost]: inv:cmake:cmake:module#module:FindBoost + +## Wheel repair tools + +There are wheel repair tools for each operating system that bundle any dynamic +libraries used and patch the libraries/python bindings to point to prioritize +those libraries. The most common tools for these are [auditwheel] for Linux, +[delocate] for MacOS and [delvewheel] for Windows. [cibuildwheel] incorporates +these tools in its [repair wheel] feature. + +One downside of these tools is that all dependencies are duplicated and bundled +in each project. + +[auditwheel]: https://pypi.org/project/auditwheel/ +[delocate]: https://pypi.org/project/delocate/ +[delvewheel]: https://pypi.org/project/delvewheel/ +[cibuildwheel]: https://cibuildwheel.pypa.io/en/stable/ +[repair wheel]: + https://cibuildwheel.pypa.io/en/stable/options/#repair-wheel-command + +## Manual patching + +For maximum control, you can do the equivalent patching inside CMake manually. +This is not recommended because it does not have access to the python +environment metadata. + +The `RPATH` patching can be done as + +```cmake +if(APPLE) + set(origin_token "@loader_path") +else() + set(origin_token "$ORIGIN") +endif() +set_property(TARGET PROPERTY INSTALL_RPATH + "${origin_token}/install_path/to/dynamic_library" +) +``` + +For Windows patching, this has to be done at the python files using +`os.add_dll_directory` at the top-most package `__init__.py` file or top-level +python module files. + +```python +import os +from pathlib import Path + +dependency_dll_path = Path(__file__).parent / "install_path/to/dynamic_library" +os.add_dll_directory(str(dependency_dll_path)) +``` diff --git a/docs/index.md b/docs/index.md index 913a6d349..1c3924708 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,6 +29,7 @@ first Friday of every month at the same time. Our past meeting minutes are guide/getting_started guide/cmakelists +guide/dynamic_link guide/crosscompile guide/migration_guide guide/build From 8f6f4aac901d4971536448f0c36ed23df160a1d4 Mon Sep 17 00:00:00 2001 From: Cristian Le Date: Tue, 25 Mar 2025 14:53:19 +0100 Subject: [PATCH 2/6] Mention PEP600 for manylinux requirement Signed-off-by: Cristian Le --- docs/guide/dynamic_link.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/guide/dynamic_link.md b/docs/guide/dynamic_link.md index e1eeb3f79..b341b4f59 100644 --- a/docs/guide/dynamic_link.md +++ b/docs/guide/dynamic_link.md @@ -28,8 +28,10 @@ those libraries. The most common tools for these are [auditwheel] for Linux, [delocate] for MacOS and [delvewheel] for Windows. [cibuildwheel] incorporates these tools in its [repair wheel] feature. -One downside of these tools is that all dependencies are duplicated and bundled -in each project. +These tools also rename the library with a unique hash to avoid any potential +name collision if the same library is being bundled by a different package, and +check if the packages confirm to standards like [PEP600] (`manylinux_X_Y`). +These tools do not allow to have cross wheel library dependency. [auditwheel]: https://pypi.org/project/auditwheel/ [delocate]: https://pypi.org/project/delocate/ @@ -37,6 +39,7 @@ in each project. [cibuildwheel]: https://cibuildwheel.pypa.io/en/stable/ [repair wheel]: https://cibuildwheel.pypa.io/en/stable/options/#repair-wheel-command +[PEP600]: https://peps.python.org/pep-0600 ## Manual patching From 42bb84c877005a3d3b9c664588bf54fbc85b290d Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 14 Apr 2025 18:05:19 -0400 Subject: [PATCH 3/6] Update dynamic_link.md --- docs/guide/dynamic_link.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/guide/dynamic_link.md b/docs/guide/dynamic_link.md index b341b4f59..d489da1c8 100644 --- a/docs/guide/dynamic_link.md +++ b/docs/guide/dynamic_link.md @@ -43,9 +43,8 @@ These tools do not allow to have cross wheel library dependency. ## Manual patching -For maximum control, you can do the equivalent patching inside CMake manually. -This is not recommended because it does not have access to the python -environment metadata. +You can manually make a relative RPath. This has the benefit of working +when not running scikit-build-core, as well. The `RPATH` patching can be done as From 3a3b8d14dc2faafbf929f1df86ba3b1e27f0493c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 14 Apr 2025 22:05:55 +0000 Subject: [PATCH 4/6] style: pre-commit fixes --- docs/guide/dynamic_link.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guide/dynamic_link.md b/docs/guide/dynamic_link.md index d489da1c8..9b13c1579 100644 --- a/docs/guide/dynamic_link.md +++ b/docs/guide/dynamic_link.md @@ -43,8 +43,8 @@ These tools do not allow to have cross wheel library dependency. ## Manual patching -You can manually make a relative RPath. This has the benefit of working -when not running scikit-build-core, as well. +You can manually make a relative RPath. This has the benefit of working when not +running scikit-build-core, as well. The `RPATH` patching can be done as From f16a6154514064fe3e4df327b0a97bbe7d525cac Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 14 Apr 2025 18:07:45 -0400 Subject: [PATCH 5/6] Update dynamic_link.md --- docs/guide/dynamic_link.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/guide/dynamic_link.md b/docs/guide/dynamic_link.md index 9b13c1579..96162a152 100644 --- a/docs/guide/dynamic_link.md +++ b/docs/guide/dynamic_link.md @@ -33,18 +33,10 @@ name collision if the same library is being bundled by a different package, and check if the packages confirm to standards like [PEP600] (`manylinux_X_Y`). These tools do not allow to have cross wheel library dependency. -[auditwheel]: https://pypi.org/project/auditwheel/ -[delocate]: https://pypi.org/project/delocate/ -[delvewheel]: https://pypi.org/project/delvewheel/ -[cibuildwheel]: https://cibuildwheel.pypa.io/en/stable/ -[repair wheel]: - https://cibuildwheel.pypa.io/en/stable/options/#repair-wheel-command -[PEP600]: https://peps.python.org/pep-0600 - ## Manual patching -You can manually make a relative RPath. This has the benefit of working when not -running scikit-build-core, as well. +You can manually make a relative RPath. This has the benefit of working +when not running scikit-build-core, as well. The `RPATH` patching can be done as @@ -70,3 +62,14 @@ from pathlib import Path dependency_dll_path = Path(__file__).parent / "install_path/to/dynamic_library" os.add_dll_directory(str(dependency_dll_path)) ``` + + + +[auditwheel]: https://pypi.org/project/auditwheel/ +[delocate]: https://pypi.org/project/delocate/ +[delvewheel]: https://pypi.org/project/delvewheel/ +[cibuildwheel]: https://cibuildwheel.pypa.io/en/stable/ +[repair wheel]: https://cibuildwheel.pypa.io/en/stable/options/#repair-wheel-command +[PEP600]: https://peps.python.org/pep-0600 + + From f8ff31e34801d117f19639431c2995d67298b891 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 14 Apr 2025 22:08:13 +0000 Subject: [PATCH 6/6] style: pre-commit fixes --- docs/guide/dynamic_link.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guide/dynamic_link.md b/docs/guide/dynamic_link.md index 96162a152..2cff72976 100644 --- a/docs/guide/dynamic_link.md +++ b/docs/guide/dynamic_link.md @@ -35,8 +35,8 @@ These tools do not allow to have cross wheel library dependency. ## Manual patching -You can manually make a relative RPath. This has the benefit of working -when not running scikit-build-core, as well. +You can manually make a relative RPath. This has the benefit of working when not +running scikit-build-core, as well. The `RPATH` patching can be done as