Add support for cross

Author: messense

.github/workflows/ci.yml

@@ -95,7 +95,7 @@ jobs:
           PYTHON: ${{ matrix.python-version }}
         run: |
           for example_dir in examples/*; do
-              tox -c $example_dir -e py
+              tox -c $example_dir -e py -vvvv
           done
 
       - name: Test macOS universal2
@@ -224,3 +224,48 @@ jobs:
             pip3 install rust_with_cffi/dist/rust_with_cffi*.whl
             python3 -c "from rust_with_cffi import rust; assert rust.rust_func() == 14"
             python3 -c "from rust_with_cffi.cffi import lib; assert lib.cffi_func() == 15"
+
+  test-cross:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - name: Install cross
+        run: cargo install cross
+      - name: Build package
+        run: pip install -e .
+      - name: Build wheel using cross
+        shell: bash
+        env:
+          CARGO: cross
+          CARGO_BUILD_TARGET: aarch64-unknown-linux-gnu
+          PYO3_CROSS_LIB_DIR: /opt/python/cp38-cp38/lib
+        run: |
+          cd examples/namespace_package
+          docker build -t cross-pyo3:aarch64-unknown-linux-gnu .
+          python -m pip install wheel
+          python setup.py bdist_wheel --plat-name manylinux2014_aarch64
+          ls -la dist/
+      - uses: uraimo/[email protected]
+        name: Install built wheel
+        with:
+          arch: aarch64
+          distro: ubuntu20.04
+          dockerRunArgs: |
+            --volume "${PWD}/examples/namespace_package:/io"
+          install: |
+            apt-get update
+            apt-get install -y --no-install-recommends python3 python3-pip
+            pip3 install -U pip
+          run: |
+            pip3 install namespace_package --no-index --find-links /io/dist/ --force-reinstall
+            python3 -c "from namespace_package import rust; assert rust.rust_func() == 14"
+            python3 -c "from namespace_package import python; assert python.python_func() == 15"

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+### Added
+- Add support for cross-compiling using [`cross`](https://github.com/rust-embedded/cross). [#185](https://github.com/PyO3/setuptools-rust/pull/185)
+
 ### Fixed
 - Fix incompatibility with Python 3.6.0 using default values for NamedTuple classes. [#184](https://github.com/PyO3/setuptools-rust/pull/184)
 

examples/namespace_package/Cross.toml

@@ -0,0 +1,9 @@
+[target.aarch64-unknown-linux-gnu]
+image = "cross-pyo3:aarch64-unknown-linux-gnu"
+
+[build.env]
+passthrough = [
+    "RUST_BACKTRACE",
+		"RUST_LOG",
+		"PYO3_CROSS_LIB_DIR",
+]

examples/namespace_package/Dockerfile

@@ -0,0 +1,13 @@
+FROM quay.io/pypa/manylinux2014_aarch64 AS manylinux
+
+FROM rustembedded/cross:aarch64-unknown-linux-gnu
+
+RUN apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa && \
+    apt-get update && \
+    apt-get install -y python3.8 && \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
+
+COPY --from=manylinux /opt/_internal /opt/_internal
+COPY --from=manylinux /opt/python /opt/python

setuptools_rust/build.py

@@ -27,6 +27,7 @@
     binding_features,
     get_rust_target_info,
     get_rust_target_list,
+    split_platform_and_extension,
 )
 
 
@@ -65,6 +66,7 @@ def initialize_options(self) -> None:
         self.build_temp = None
         self.plat_name = None
         self.target = os.getenv("CARGO_BUILD_TARGET")
+        self.cargo = os.getenv("CARGO", "cargo")
 
     def finalize_options(self) -> None:
         super().finalize_options()
@@ -246,7 +248,7 @@ def build_extension(
 
         if executable:
             args = (
-                ["cargo", "build", "--manifest-path", ext.path]
+                [self.cargo, "build", "--manifest-path", ext.path]
                 + feature_args
                 + target_args
                 + list(ext.args or [])
@@ -262,7 +264,7 @@ def build_extension(
 
         else:
             args = (
-                ["cargo", "rustc", "--lib", "--manifest-path", ext.path]
+                [self.cargo, "rustc", "--lib", "--manifest-path", ext.path]
                 + feature_args
                 + target_args
                 + list(ext.args or [])
@@ -396,13 +398,13 @@ def install_extension(
 
             if executable:
                 ext_path = build_ext.get_ext_fullpath(module_name)
-                # remove .so extension
-                ext_path, _ = os.path.splitext(ext_path)
-                # remove python3 extension (i.e. cpython-36m)
-                ext_path, _ = os.path.splitext(ext_path)
+                # remove extensions
+                ext_path, _, _ = split_platform_and_extension(ext_path)
 
                 # Add expected extension
-                ext_path += sysconfig.get_config_var("EXE")
+                exe = sysconfig.get_config_var("EXE")
+                if exe is not None:
+                    ext_path += exe
 
                 os.makedirs(os.path.dirname(ext_path), exist_ok=True)
                 ext.install_script(module_name.split(".")[-1], ext_path)
@@ -434,9 +436,10 @@ def install_extension(
             os.chmod(ext_path, mode)
 
     def get_dylib_ext_path(self, ext: RustExtension, target_fname: str) -> str:
+        assert self.plat_name is not None
         build_ext = cast(CommandBuildExt, self.get_finalized_command("build_ext"))
 
-        filename: str = build_ext.get_ext_fullpath(target_fname)
+        ext_path: str = build_ext.get_ext_fullpath(target_fname)
 
         if (ext.py_limited_api == "auto" and self._py_limited_api()) or (
             ext.py_limited_api
@@ -445,9 +448,34 @@ def get_dylib_ext_path(self, ext: RustExtension, target_fname: str) -> str:
             if abi3_suffix is not None:
                 so_ext = get_config_var("EXT_SUFFIX")
                 assert isinstance(so_ext, str)
-                filename = filename[: -len(so_ext)] + get_abi3_suffix()
-
-        return filename
+                ext_path = ext_path[: -len(so_ext)] + get_abi3_suffix()
+
+        if ".abi3." in ext_path:
+            return ext_path
+        # Examples: linux_x86_64, linux_i686, manylinux2014_aarch64, manylinux_2_24_armv7l
+        plat_name = self.plat_name.lower().replace("-", "_").replace(".", "_")
+        if not plat_name.startswith(("linux", "manylinux")):
+            return ext_path
+
+        arch_parts = []
+        arch_found = False
+        for item in plat_name.split("_"):
+            if item.startswith(("linux", "manylinux")):
+                continue
+            if item.isdigit() and not arch_found:
+                # manylinux_2_24_armv7l arch should be armv7l
+                continue
+            arch_found = True
+            arch_parts.append(item)
+        target_arch = "_".join(arch_parts)
+        host_platform = sysconfig.get_platform()
+        host_arch = host_platform.rsplit("-", 1)[1]
+        # Remove incorrect platform tag if we are cross compiling
+        if target_arch and host_arch != target_arch:
+            ext_path, _, extension = split_platform_and_extension(ext_path)
+            # rust.so, removed platform tag
+            ext_path += extension
+        return ext_path
 
     def _py_limited_api(self) -> PyLimitedApi:
         bdist_wheel = self.distribution.get_command_obj("bdist_wheel", create=False)

setuptools_rust/setuptools_ext.py

@@ -124,9 +124,11 @@ def run(self) -> None:
                 log.info("running build_rust")
                 build_rust = self.get_finalized_command("build_rust")
                 build_rust.inplace = self.inplace
-                build_rust.plat_name = self.plat_name
                 build_rust.target = self.target
                 build_rust.verbose = self.verbose
+                options = self.distribution.get_cmdline_options().get("bdist_wheel", {})
+                plat_name = options.get("plat-name") or self.plat_name
+                build_rust.plat_name = plat_name
                 build_rust.run()
 
             build_ext_base_class.run(self)

setuptools_rust/utils.py

@@ -1,6 +1,7 @@
+import os
 import subprocess
 from distutils.errors import DistutilsPlatformError
-from typing import List, Optional, Set, Union
+from typing import List, Optional, Set, Tuple, Union
 
 from semantic_version import Version
 from typing_extensions import Literal
@@ -58,3 +59,17 @@ def get_rust_target_list() -> List[str]:
         )
         _rust_target_list = output.splitlines()
     return _rust_target_list
+
+
+def split_platform_and_extension(ext_path: str) -> Tuple[str, str, str]:
+    """Splits an extension path into a tuple (ext_path, plat_tag, extension).
+
+    >>> split_platform_and_extension("foo/bar.platform.so")
+    ('foo/bar', '.platform', '.so')
+    """
+
+    # rust.cpython-38-x86_64-linux-gnu.so to (rust.cpython-38-x86_64-linux-gnu, .so)
+    ext_path, extension = os.path.splitext(ext_path)
+    # rust.cpython-38-x86_64-linux-gnu to (rust, .cpython-38-x86_64-linux-gnu)
+    ext_path, platform_tag = os.path.splitext(ext_path)
+    return (ext_path, platform_tag, extension)