Add macOS universal2 wheel building support (#115)

messense · davidhewitt · web-flow · commit 7b7ab01d6e70 · 2021-02-22T07:34:34.000Z
* Add macOS universal2 wheel building support

* Add universal2 examples and test on CI

* Apply suggestions from code review

Co-authored-by: David Hewitt &lt;1939362+davidhewitt@users.noreply.github.com&gt;

* Use ARCHFLAGS to set universal2 build

Co-authored-by: David Hewitt &lt;1939362+davidhewitt@users.noreply.github.com&gt;
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -14,7 +14,7 @@ jobs:
       matrix:
         python-version: [3.6, 3.7, 3.8, 3.9, pypy3]
         platform: [
-          { os: "macOS-latest",   python-architecture: "x64", rust-target: "x86_64-apple-darwin" },
+          { os: "macos-latest",   python-architecture: "x64", rust-target: "x86_64-apple-darwin" },
           { os: "ubuntu-latest", python-architecture: "x64", rust-target: "x86_64-unknown-linux-gnu" },
           { os: "windows-latest", python-architecture: "x64", rust-target: "x86_64-pc-windows-msvc" },
           { os: "windows-latest", python-architecture: "x86", rust-target: "i686-pc-windows-msvc" },
@@ -44,6 +44,10 @@ jobs:
           profile: minimal
           default: true
 
+      - name: Install Rust aarch64-apple-darwin target
+        if: matrix.platform.os == 'macos-latest'
+        run: rustup target add aarch64-apple-darwin
+
       - name: Install test dependencies
         run: pip install --upgrade tox setuptools
 
@@ -74,6 +78,24 @@ jobs:
               tox -c $example_dir -e py
           done
 
+      - name: Test macOS universal2
+        if: matrix.platform.os == 'macos-latest'
+        shell: bash
+        env:
+          DEVELOPER_DIR: /Applications/Xcode.app/Contents/Developer
+          MACOSX_DEPLOYMENT_TARGET: '10.9'
+          ARCHFLAGS: -arch x86_64 -arch arm64
+          PYO3_CROSS_LIB_DIR: /Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.8/lib
+        run: |
+          cd examples/namespace_package
+          pip install wheel
+          python setup.py bdist_wheel
+          ls -l dist/
+          pip install --force-reinstall dist/namespace_package*_universal2.whl
+          cd -
+          python -c "from namespace_package import rust; assert rust.rust_func() == 14"
+          python -c "from namespace_package import python; assert python.python_func() == 15"
+
   test-abi3:
     runs-on: ${{ matrix.os }}
     strategy:
@@ -92,6 +114,10 @@ jobs:
           toolchain: stable
           override: true
 
+      - name: Install Rust aarch64-apple-darwin target
+        if: matrix.os == 'macos-latest'
+        run: rustup target add aarch64-apple-darwin
+
       - name: Build package
         run: pip install -e .
 
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,7 @@
 
 ### Added
 - Support building x86-64 wheel on arm64 macOS machine. [#114](https://github.com/PyO3/setuptools-rust/pull/114)
+- Add macOS universal2 wheel building support. [#115](https://github.com/PyO3/setuptools-rust/pull/115)
 
 ### Changed
 - Respect `PYO3_PYTHON` and `PYTHON_SYS_EXECUTABLE` environment variables if set. [#96](https://github.com/PyO3/setuptools-rust/pull/96)
diff --git a/README.md b/README.md
@@ -133,12 +133,29 @@ It is possible to use any of the `manylinux` docker images: `manylinux1`, `manyl
 
 You can define rust extension with RustExtension class:
 
-RustExtension(name, path, args=None, features=None,
-rust\_version=None, quiet=False, debug=False)
+```python
+RustExtension(
+     name,
+     path="Cargo.toml",
+     args=None,
+     features=None,
+     rustc_flags=None,
+     rust_version=None,
+     quiet=False,
+     debug=None,
+     binding=Binding.PyO3,
+     strip=Strip.No,
+     script=False,
+     native=False,
+     optional=False,
+     py_limited_api=False,
+)
+```
 
 The class for creating rust extensions.
 
-   - param str name
+   - param str `name`
+
      the full name of the extension, including any packages -- ie.
      *not* a filename or pathname, but Python dotted name. It is
      possible to specify multiple binaries, if extension uses
@@ -147,51 +164,71 @@ The class for creating rust extensions.
      binaries and values are full name of the executable inside python
      package.
 
-   - param str path
+   - param str `path`
+
      path to the Cargo.toml manifest file
 
-   - param \[str\] args
+   - param \[str\] `args`
+
      a list of extra argumenents to be passed to cargo.
 
-   - param \[str\] features
+   - param \[str\] `features`
+
      a list of features to also build
 
-   - param \[str\] rustc\_flags
+   - param \[str\] `rustc_flags`
+
      A list of arguments to pass to rustc, e.g. cargo rustc --features
      \<features\> \<args\> -- \<rustc\_flags\>
 
-   - param str rust\_version
+   - param str `rust_version`
+
      sematic version of rust compiler version -- for example
      *\>1.14,\<1.16*, default is None
 
-   - param bool quiet
-     Does not echo cargo's output. default is False
+   - param bool `quiet`
 
-   - param bool debug
-     Controls whether --debug or --release is passed to cargo. If set
+     Does not echo cargo's output. default is `False`
+
+   - param bool `debug`
+
+     Controls whether `--debug` or `--release` is passed to cargo. If set
      to None then build type is auto-detect. Inplace build is debug
-     build otherwise release. Default: None
+     build otherwise release. Default: `None`
+
+   - param int `binding`
+
+     Controls which python binding is in use.
+     * `Binding.PyO3` uses PyO3
+     * `Binding.RustCPython` uses rust-cpython
+     * `Binding.NoBinding` uses no binding.
+     * `Binding.Exec` build executable.
 
-   - param int binding
-     Controls which python binding is in use. Binding.PyO3 uses PyO3
-     Binding.RustCPython uses rust-cpython Binding.NoBinding uses no
-     binding. Binding.Exec build executable.
+   - param int `strip`
 
-   - param int strip
      Strip symbols from final file. Does nothing for debug build.
-     Strip.No - do not strip symbols (default) Strip.Debug - strip
-     debug symbols Strip.All - strip all symbols
+     * `Strip.No` - do not strip symbols (default)
+     * `Strip.Debug` - strip debug symbols
+     * `Strip.All` - strip all symbols
 
-   - param bool script
-     Generate console script for executable if Binding.Exec is used.
+   - param bool `script`
+
+     Generate console script for executable if `Binding.Exec` is used.
+
+   - param bool `native`
 
-   - param bool native
      Build extension or executable with "-C target-cpu=native"
 
-   - param bool optional
+   - param bool `optional`
+
      if it is true, a build failure in the extension will not abort the
      build process, but instead simply not install the failing
      extension.
+   - param bool `py_limited_api`
+
+        Same as `py_limited_api` on `setuptools.Extension`. Note that if you
+        set this to True, your extension must pass the appropriate feature
+        flags to pyo3 (ensuring that `abi3` feature is enabled).
 
 ## Commands
 
diff --git a/examples/namespace_package/tox.ini b/examples/namespace_package/tox.ini
@@ -8,3 +8,4 @@ deps =
     setuptools-rust @ file://{toxinidir}/../../
     pytest
 commands = pytest {posargs}
+passenv = *
diff --git a/setuptools_rust/build.py b/setuptools_rust/build.py
@@ -62,7 +62,39 @@ def finalize_options(self):
             ("inplace", "inplace"),
         )
 
+    def get_target_triple(self):
+        # If we are on a 64-bit machine, but running a 32-bit Python, then
+        # we'll target a 32-bit Rust build.
+        # Automatic target detection can be overridden via the CARGO_BUILD_TARGET
+        # environment variable.
+        if os.getenv("CARGO_BUILD_TARGET"):
+            return os.environ["CARGO_BUILD_TARGET"]
+        elif self.plat_name == "win32":
+            return "i686-pc-windows-msvc"
+        elif self.plat_name == "win-amd64":
+            return "x86_64-pc-windows-msvc"
+        elif self.plat_name.startswith("macosx-") and platform.machine() == "x86_64":
+            # x86_64 or arm64 macOS targeting x86_64
+            return "x86_64-apple-darwin"
+
     def run_for_extension(self, ext: RustExtension):
+        arch_flags = os.getenv("ARCHFLAGS")
+        universal2 = False
+        if self.plat_name.startswith("macosx-") and arch_flags:
+            universal2 = "x86_64" in arch_flags and "arm64" in arch_flags
+        if universal2:
+            arm64_dylib_paths = self.build_extension(ext, "aarch64-apple-darwin")
+            x86_64_dylib_paths = self.build_extension(ext, "x86_64-apple-darwin")
+            dylib_paths = []
+            for (target_fname, arm64_dylib), (_, x86_64_dylib) in zip(arm64_dylib_paths, x86_64_dylib_paths):
+                fat_dylib_path = arm64_dylib.replace("aarch64-apple-darwin/", "")
+                self.create_universal2_binary(fat_dylib_path, [arm64_dylib, x86_64_dylib])
+                dylib_paths.append((target_fname, fat_dylib_path))
+        else:
+            dylib_paths = self.build_extension(ext)
+        self.install_extension(ext, dylib_paths)
+
+    def build_extension(self, ext: RustExtension, target_triple=None):
         executable = ext.binding == Binding.Exec
 
         rust_target_info = get_rust_target_info()
@@ -84,22 +116,8 @@ def run_for_extension(self, ext: RustExtension):
         )
         rustflags = ""
 
-        # If we are on a 64-bit machine, but running a 32-bit Python, then
-        # we'll target a 32-bit Rust build.
-        # Automatic target detection can be overridden via the CARGO_BUILD_TARGET
-        # environment variable.
-        target_triple = None
+        target_triple = target_triple or self.get_target_triple()
         target_args = []
-        if os.getenv("CARGO_BUILD_TARGET"):
-            target_triple = os.environ["CARGO_BUILD_TARGET"]
-        elif self.plat_name == "win32":
-            target_triple = "i686-pc-windows-msvc"
-        elif self.plat_name == "win-amd64":
-            target_triple = "x86_64-pc-windows-msvc"
-        elif self.plat_name.startswith("macosx-") and platform.machine() == "x86_64":
-            # x86_64 or arm64 macOS targeting x86_64
-            target_triple = "x86_64-apple-darwin"
-
         if target_triple is not None:
             target_args = ["--target", target_triple]
 
@@ -264,7 +282,14 @@ def run_for_extension(self, ext: RustExtension):
                 raise DistutilsExecError(
                     f"Rust build failed; unable to find any {wildcard_so} in {artifactsdir}"
                 )
+        return dylib_paths
 
+    def install_extension(self, ext: RustExtension, dylib_paths):
+        executable = ext.binding == Binding.Exec
+        debug_build = ext.debug if ext.debug is not None else self.inplace
+        debug_build = self.debug if self.debug is not None else debug_build
+        if self.release:
+            debug_build = False
         # Ask build_ext where the shared library would go if it had built it,
         # then copy it there.
         build_ext = self.get_finalized_command("build_ext")
@@ -301,7 +326,7 @@ def run_for_extension(self, ext: RustExtension):
                     args.insert(0, "strip")
                     args.append(ext_path)
                     try:
-                        output = subprocess.check_output(args, env=env)
+                        output = subprocess.check_output(args)
                     except subprocess.CalledProcessError:
                         pass
 
@@ -323,3 +348,31 @@ def get_dylib_ext_path(self, ext, target_fname):
             return build_ext.get_ext_fullpath(target_fname)
         finally:
             del build_ext.ext_map[modpath]
+
+    @staticmethod
+    def create_universal2_binary(output_path, input_paths):
+        # Try lipo first
+        command = ["lipo", "-create", "-output", output_path, *input_paths]
+        try:
+            subprocess.check_output(command)
+        except subprocess.CalledProcessError as e:
+            output = e.output
+            if isinstance(output, bytes):
+                output = e.output.decode("latin-1").strip()
+            raise CompileError(
+                "lipo failed with code: %d\n%s" % (e.returncode, output)
+            )
+        except OSError:
+            # lipo not found, try using the fat-macho library
+            try:
+                from fat_macho import FatWriter
+            except ImportError:
+                raise DistutilsExecError(
+                    "failed to locate `lipo` or import `fat_macho.FatWriter`. "
+                    "Try installing with `pip install fat-macho` "
+                )
+            fat = FatWriter()
+            for input_path in input_paths:
+                with open(input_path, "rb") as f:
+                    fat.add(f.read())
+            fat.write_to(output_path)
diff --git a/setuptools_rust/setuptools_ext.py b/setuptools_rust/setuptools_ext.py
@@ -1,20 +1,16 @@
-from abc import ABC, abstractmethod
+import os
 from distutils import log
-from distutils.cmd import Command
 from distutils.command.check import check
 from distutils.command.clean import clean
-from distutils.errors import DistutilsPlatformError
-from setuptools.command.install import install
+
 from setuptools.command.build_ext import build_ext
+from setuptools.command.install import install
 
 try:
     from wheel.bdist_wheel import bdist_wheel
 except ImportError:
     bdist_wheel = None
 
-from .extension import RustExtension
-from .utils import get_rust_version
-
 
 def add_rust_extension(dist):
     build_ext_base_class = dist.cmdclass.get('build_ext', build_ext)
@@ -114,6 +110,25 @@ def finalize_options(self):
                         self.distribution.entry_points["console_scripts"] = ep_scripts
 
                 bdist_wheel_base_class.finalize_options(self)
+
+            def get_tag(self):
+                python, abi, plat = super().get_tag()
+                arch_flags = os.getenv("ARCHFLAGS")
+                universal2 = False
+                if self.plat_name.startswith("macosx-") and arch_flags:
+                    universal2 = "x86_64" in arch_flags and "arm64" in arch_flags
+                if universal2 and plat.startswith("macosx_"):
+                    from wheel.macosx_libfile import calculate_macosx_platform_tag
+
+                    macos_target = os.getenv("MACOSX_DEPLOYMENT_TARGET")
+                    if macos_target is None:
+                        # Example: macosx_11_0_arm64
+                        macos_target = '.'.join(plat.split("_")[1:3])
+                    plat = calculate_macosx_platform_tag(
+                        self.bdist_dir,
+                        "macosx-{}-universal2".format(macos_target)
+                    )
+                return python, abi, plat
         dist.cmdclass["bdist_wheel"] = bdist_wheel_rust_extension
 
 
