Add --platform option to facilitate installing wheels for platforms other than the host system (#123)

Author: BrandonLWhite

README.md

@@ -66,3 +66,62 @@ You can also ensure that the virtual environment is recreated by using the `--cl
 ```bash
 poetry bundle venv /path/to/environment --clear
 ```
+
+#### --platform option (Experimental)
+This option allows you to specify a target platform for binary wheel selection, allowing you to install wheels for
+architectures/platforms other than the host system.
+
+The primary use case is in CI/CD operations to produce a deployable asset, such as a ZIP file for AWS Lambda and other
+such cloud providers. It is common for the runtimes of these target environments to be different enough from the CI/CD's
+runner host such that the binary wheels selected using the host's criteria are not compatible with the target system's.
+
+#### Supported platform values
+The `--platform` option requires a value that conforms to the [Python Packaging Platform Tag format](
+https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#platform-tag). Only the following
+"families" are supported at this time:
+- `manylinux`
+- `musllinux`
+- `macosx`
+
+#### Examples of valid platform tags
+This is not a comprehensive list, but illustrates typical examples.
+
+`manylinux_2_28_x86_64`, `manylinux1_x86_64`, `manylinux2010_x86_64`, `manylinux2014_x86_64`
+
+`musllinux_1_2_x86_64`
+
+`macosx_10_9_x86_64`, `macosx_10_9_intel`, `macosx_11_1_universal2`, `macosx_11_0_arm64`
+
+#### Example use case for AWS Lambda
+As an example of one motivating use case for this option, consider the AWS Lambda "serverless" execution environment.
+Depending upon which Python version you configure for your runtime, you may get different versions of the Linux system
+runtime.  When dealing with pre-compiled binary wheels, these runtime differences can matter.  If a shared library from
+a wheel is packaged in your deployment artifact that is incompatible with the runtime provided environment, then your
+Python "function" will error at execution time in the serverless environment.
+The issue arises when the build system that is producing the deployment artifact has a materially different platform
+from the selected serverless Lambda runtime.
+
+For example, the Python 3.11 Lambda runtime is Amazon Linux 2, which includes the Glibc 2.26 library.  If a Python
+application is packaged and deployed in this environment that contains wheels built for a more recent version of Glibc,
+then a runtime error will result.  This is likely to occur even if the build system is the same CPU architecture
+(e.g. x86_64) and core platform (e.g. Linux) and there is a package dependency that provides multiple precompiled
+wheels for various Glibc (or other system library) versions.  The "best" wheel in the context of the build system can
+differ from that of the target execution environment.
+
+
+#### Limitations
+**This is not an actual cross-compiler**.  Nor is it a containerized compilation/build environment. It simply allows
+controlling which **prebuilt** binaries are selected.  It is not a replacement for cross-compilation or containerized
+builds for use cases requiring that.
+
+If there is not a binary wheel distribution compatible with the specified platform, then the package's source
+distribution is selected.  If there are compile/build steps for "extensions" that need to run for the source
+distribution, then these operations will execute in the context of the host CI/build system.
+**This means that the `--platform` option
+has no impact on any extension compile/build operations that must occur during package installation.**
+This feature is only for
+**selecting** prebuilt wheels, and **not for compiling** them from source.
+
+Arguably, in a vast number of use cases, prebuilt wheel binaries are available for your packages and simply selecting
+them based on a platform other than the host CI/build system is much faster and simpler than heavier build-from-source
+alternatives.

src/poetry_plugin_bundle/bundlers/venv_bundler.py

@@ -12,6 +12,7 @@
     from cleo.io.outputs.section_output import SectionOutput
     from poetry.poetry import Poetry
     from poetry.repositories.lockfile_repository import LockfileRepository
+    from poetry.utils.env import Env
 
 
 class VenvBundler(Bundler):
@@ -23,6 +24,7 @@ def __init__(self) -> None:
         self._remove: bool = False
         self._activated_groups: set[str] | None = None
         self._compile: bool = False
+        self._platform: str | None = None
 
     def set_path(self, path: Path) -> VenvBundler:
         self._path = path
@@ -49,6 +51,11 @@ def set_compile(self, compile: bool = False) -> VenvBundler:
 
         return self
 
+    def set_platform(self, platform: str | None) -> VenvBundler:
+        self._platform = platform
+
+        return self
+
     def bundle(self, poetry: Poetry, io: IO) -> bool:
         from pathlib import Path
         from tempfile import TemporaryDirectory
@@ -60,7 +67,6 @@ def bundle(self, poetry: Poetry, io: IO) -> bool:
         from poetry.installation.installer import Installer
         from poetry.installation.operations.install import Install
         from poetry.packages.locker import Locker
-        from poetry.utils.env import Env
         from poetry.utils.env import EnvManager
         from poetry.utils.env.python import Python
         from poetry.utils.env.python.exceptions import InvalidCurrentPythonVersionError
@@ -134,6 +140,9 @@ def create_venv_at_path(
             )
             env = manager.create_venv_at_path(self._path, python=python, force=True)
 
+        if self._platform:
+            self._constrain_env_platform(env, self._platform)
+
         self._write(io, f"{message}: <info>Installing dependencies</info>")
 
         class CustomLocker(Locker):
@@ -239,3 +248,12 @@ def _write(self, io: IO | SectionOutput, message: str) -> None:
             return
 
         io.overwrite(message)
+
+    def _constrain_env_platform(self, env: Env, platform: str) -> None:
+        """
+        Set the argument environment's supported tags
+        based on the configured platform override.
+        """
+        from poetry_plugin_bundle.utils.platforms import create_supported_tags
+
+        env._supported_tags = create_supported_tags(platform, env)

src/poetry_plugin_bundle/console/commands/bundle/venv.py

@@ -45,6 +45,17 @@ class BundleVenvCommand(BundleCommand):
             " because the old installer always compiles.)",
             flag=True,
         ),
+        option(
+            "platform",
+            None,
+            (
+                "Only use wheels compatible with the specified platform."
+                " Otherwise the default behavior uses the platform"
+                " of the running system. (<comment>Experimental</comment>)"
+            ),
+            flag=False,
+            value_required=True,
+        ),
     ]
 
     bundler_name = "venv"
@@ -54,4 +65,5 @@ def configure_bundler(self, bundler: VenvBundler) -> None:  # type: ignore[overr
         bundler.set_executable(self.option("python"))
         bundler.set_remove(self.option("clear"))
         bundler.set_compile(self.option("compile"))
+        bundler.set_platform(self.option("platform"))
         bundler.set_activated_groups(self.activated_groups)

src/poetry_plugin_bundle/utils/__init__.py

@@ -0,0 +1,158 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+
+if TYPE_CHECKING:
+    from packaging.tags import Tag
+    from poetry.utils.env import Env
+
+
+@dataclass
+class PlatformTagParseResult:
+    platform: str
+    version_major: int
+    version_minor: int
+    arch: str
+
+    @staticmethod
+    def parse(tag: str) -> PlatformTagParseResult:
+        import re
+
+        match = re.match("([a-z]+)_([0-9]+)_([0-9]+)_(.*)", tag)
+        if not match:
+            raise ValueError(f"Invalid platform tag: {tag}")
+        platform, version_major_str, version_minor_str, arch = match.groups()
+        return PlatformTagParseResult(
+            platform=platform,
+            version_major=int(version_major_str),
+            version_minor=int(version_minor_str),
+            arch=arch,
+        )
+
+    def to_tag(self) -> str:
+        return "_".join(
+            [self.platform, str(self.version_major), str(self.version_minor), self.arch]
+        )
+
+
+def create_supported_tags(platform: str, env: Env) -> list[Tag]:
+    """
+    Given a platform specifier string, generate a list of compatible tags
+    for the argument environment's interpreter.
+
+    Refer to:
+        https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#platform-tag
+        https://pip.pypa.io/en/stable/cli/pip_install/#cmdoption-platform
+    """
+    from packaging.tags import INTERPRETER_SHORT_NAMES
+    from packaging.tags import compatible_tags
+    from packaging.tags import cpython_tags
+    from packaging.tags import generic_tags
+
+    if platform.startswith("manylinux"):
+        supported_platforms = create_supported_manylinux_platforms(platform)
+    elif platform.startswith("musllinux"):
+        supported_platforms = create_supported_musllinux_platforms(platform)
+    elif platform.startswith("macosx"):
+        supported_platforms = create_supported_macosx_platforms(platform)
+    else:
+        raise NotImplementedError(f"Platform {platform} not supported")
+
+    python_implementation = env.python_implementation.lower()
+    python_version = env.version_info[:2]
+    interpreter_name = INTERPRETER_SHORT_NAMES.get(
+        python_implementation, python_implementation
+    )
+    interpreter = None
+
+    if interpreter_name == "cp":
+        tags = list(
+            cpython_tags(python_version=python_version, platforms=supported_platforms)
+        )
+        interpreter = f"{interpreter_name}{python_version[0]}{python_version[1]}"
+    else:
+        tags = list(
+            generic_tags(
+                interpreter=interpreter, abis=[], platforms=supported_platforms
+            )
+        )
+
+    tags.extend(
+        compatible_tags(
+            interpreter=interpreter,
+            python_version=python_version,
+            platforms=supported_platforms,
+        )
+    )
+
+    return tags
+
+
+def create_supported_manylinux_platforms(platform: str) -> list[str]:
+    """
+    https://peps.python.org/pep-0600/
+    manylinux_${GLIBCMAJOR}_${GLIBCMINOR}_${ARCH}
+
+    For now, only GLIBCMAJOR "2" is supported.  It is unclear if there will be a need to support a future major
+    version like "3" and if specified, how generate the compatible 2.x version tags.
+    """
+    # Implementation based on https://peps.python.org/pep-0600/#package-installers
+
+    tag = normalize_legacy_manylinux_alias(platform)
+
+    parsed = PlatformTagParseResult.parse(tag)
+    return [
+        f"{parsed.platform}_{parsed.version_major}_{tag_minor}_{parsed.arch}"
+        for tag_minor in range(parsed.version_minor, -1, -1)
+    ]
+
+
+LEGACY_MANYLINUX_ALIASES = {
+    "manylinux1": "manylinux_2_5",
+    "manylinux2010": "manylinux_2_12",
+    "manylinux2014": "manylinux_2_17",
+}
+
+
+def normalize_legacy_manylinux_alias(tag: str) -> str:
+    tag_os_index_end = tag.index("_")
+    tag_os = tag[:tag_os_index_end]
+    tag_arch_suffix = tag[tag_os_index_end:]
+    os_replacement = LEGACY_MANYLINUX_ALIASES.get(tag_os)
+    if not os_replacement:
+        return tag
+
+    return os_replacement + tag_arch_suffix
+
+
+def create_supported_macosx_platforms(platform: str) -> list[str]:
+    import re
+
+    from packaging.tags import mac_platforms
+
+    match = re.match("macosx_([0-9]+)_([0-9]+)_(.*)", platform)
+    if not match:
+        raise ValueError(f"Invalid macosx tag: {platform}")
+    tag_major_str, tag_minor_str, tag_arch = match.groups()
+    tag_major_max = int(tag_major_str)
+    tag_minor_max = int(tag_minor_str)
+
+    return list(mac_platforms(version=(tag_major_max, tag_minor_max), arch=tag_arch))
+
+
+def create_supported_musllinux_platforms(platform: str) -> list[str]:
+    import re
+
+    match = re.match("musllinux_([0-9]+)_([0-9]+)_(.*)", platform)
+    if not match:
+        raise ValueError(f"Invalid musllinux tag: {platform}")
+    tag_major_str, tag_minor_str, tag_arch = match.groups()
+    tag_major_max = int(tag_major_str)
+    tag_minor_max = int(tag_minor_str)
+
+    return [
+        f"musllinux_{tag_major_max}_{minor}_{tag_arch}"
+        for minor in range(tag_minor_max, -1, -1)
+    ]

src/poetry_plugin_bundle/utils/platforms.py

@@ -11,6 +11,7 @@
 from cleo.formatters.style import Style
 from cleo.io.buffered_io import BufferedIO
 from poetry.core.packages.package import Package
+from poetry.core.packages.utils.link import Link
 from poetry.factory import Factory
 from poetry.installation.operations.install import Install
 from poetry.puzzle.exceptions import SolverProblemError
@@ -422,3 +423,79 @@ def test_bundler_non_package_mode(
   • Bundled simple-project-non-package-mode (1.2.3) into {path}
 """
     assert expected == io.fetch_output()
+
+
+def test_bundler_platform_override(
+    io: BufferedIO, tmpdir: str, mocker: MockerFixture, config: Config
+) -> None:
+    poetry = Factory().create_poetry(
+        Path(__file__).parent.parent / "fixtures" / "project_with_binary_wheel"
+    )
+    poetry.set_config(config)
+
+    def get_links_fake(package: Package) -> list[Link]:
+        return [Link(f"https://example.com/{file['file']}") for file in package.files]
+
+    mocker.patch(
+        "poetry.installation.chooser.Chooser._get_links", side_effect=get_links_fake
+    )
+    mocker.patch("poetry.installation.executor.Executor._execute_uninstall")
+    mocker.patch("poetry.installation.executor.Executor._execute_update")
+    mock_download_link = mocker.patch(
+        "poetry.installation.executor.Executor._download_link"
+    )
+    mocker.patch("poetry.installation.wheel_installer.WheelInstaller.install")
+
+    def get_installed_links() -> dict[str, str]:
+        return {
+            call[0][0].package.name: call[0][1].filename
+            for call in mock_download_link.call_args_list
+        }
+
+    bundler = VenvBundler()
+    bundler.set_path(Path(tmpdir))
+    bundler.set_remove(True)
+
+    bundler.set_platform("manylinux_2_28_x86_64")
+    bundler.bundle(poetry, io)
+    installed_link_by_package = get_installed_links()
+    assert "manylinux_2_28_x86_64" in installed_link_by_package["cryptography"]
+    assert "manylinux_2_17_x86_64" in installed_link_by_package["cffi"]
+    assert "py3-none-any.whl" in installed_link_by_package["pycparser"]
+
+    bundler.set_platform("manylinux2014_x86_64")
+    bundler.bundle(poetry, io)
+    installed_link_by_package = get_installed_links()
+    assert "manylinux2014_x86_64" in installed_link_by_package["cryptography"]
+    assert "manylinux_2_17_x86_64" in installed_link_by_package["cffi"]
+    assert "py3-none-any.whl" in installed_link_by_package["pycparser"]
+
+    bundler.set_platform("macosx_10_9_x86_64")
+    bundler.bundle(poetry, io)
+    installed_link_by_package = get_installed_links()
+    assert "macosx_10_9_universal2" in installed_link_by_package["cryptography"]
+    expected_cffi_platform = (
+        "macosx_10_9_x86_64" if sys.version_info < (3, 13) else "cffi-1.17.1.tar.gz"
+    )
+    assert expected_cffi_platform in installed_link_by_package["cffi"]
+    assert "py3-none-any.whl" in installed_link_by_package["pycparser"]
+
+    bundler.set_platform("macosx_11_0_arm64")
+    bundler.bundle(poetry, io)
+    installed_link_by_package = get_installed_links()
+    assert "macosx_10_9_universal2" in installed_link_by_package["cryptography"]
+    expected_cffi_platform = (
+        "macosx_11_0_arm64" if sys.version_info >= (3, 9) else "cffi-1.17.1.tar.gz"
+    )
+    assert expected_cffi_platform in installed_link_by_package["cffi"]
+    assert "py3-none-any.whl" in installed_link_by_package["pycparser"]
+
+    bundler.set_platform("musllinux_1_2_aarch64")
+    bundler.bundle(poetry, io)
+    installed_link_by_package = get_installed_links()
+    assert "musllinux_1_2_aarch64" in installed_link_by_package["cryptography"]
+    expected_cffi_platform = (
+        "musllinux_1_1_aarch64" if sys.version_info >= (3, 9) else "cffi-1.17.1.tar.gz"
+    )
+    assert expected_cffi_platform in installed_link_by_package["cffi"]
+    assert "py3-none-any.whl" in installed_link_by_package["pycparser"]