Merge pull request #257 from Cadair/macos_arm64_cross

Author: Cadair

.github/workflows/publish.yml

@@ -113,7 +113,7 @@ jobs:
       - run: python -m pip install PyYAML click
       - run: echo $LOAD_BUILD_TARGETS_SCRIPT | base64 --decode > load_build_targets.py
         env:
-          LOAD_BUILD_TARGETS_SCRIPT: aW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCByZQoKaW1wb3J0IGNsaWNrCmltcG9ydCB5YW1sCgpNQUNISU5FX1RZUEUgPSB7CiAgICAibGludXgiOiAidWJ1bnR1LWxhdGVzdCIsCiAgICAibWFjb3MiOiAibWFjb3MtMTIiLAogICAgIndpbmRvd3MiOiAid2luZG93cy1sYXRlc3QiLAp9Ck0xX1JVTk5FUiA9ICJtYWNvcy0xNCIKCkNJQldfQlVJTEQgPSBvcy5lbnZpcm9uLmdldCgiQ0lCV19CVUlMRCIsICIqIikKQ0lCV19BUkNIUyA9IG9zLmVudmlyb24uZ2V0KCJDSUJXX0FSQ0hTIiwgImF1dG8iKQoKCkBjbGljay5jb21tYW5kKCkKQGNsaWNrLm9wdGlvbigiLS10YXJnZXRzIiwgZGVmYXVsdD0iIikKZGVmIGxvYWRfYnVpbGRfdGFyZ2V0cyh0YXJnZXRzKToKICAgICIiIlNjcmlwdCB0byBsb2FkIGNpYnVpbGR3aGVlbCB0YXJnZXRzIGZvciBHaXRIdWIgQWN0aW9ucyB3b3JrZmxvdy4iIiIKICAgICMgTG9hZCBsaXN0IG9mIHRhcmdldHMKICAgIHRhcmdldHMgPSB5YW1sLmxvYWQodGFyZ2V0cywgTG9hZGVyPXlhbWwuQmFzZUxvYWRlcikKICAgIHByaW50KGpzb24uZHVtcHModGFyZ2V0cywgaW5kZW50PTIpKQoKICAgICMgQ3JlYXRlIG1hdHJpeAogICAgbWF0cml4ID0geyJpbmNsdWRlIjogW119CiAgICBmb3IgdGFyZ2V0IGluIHRhcmdldHM6CiAgICAgICAgbWF0cml4WyJpbmNsdWRlIl0uYXBwZW5kKGdldF9tYXRyaXhfaXRlbSh0YXJnZXQpKQoKICAgICMgT3V0cHV0IG1hdHJpeAogICAgcHJpbnQoanNvbi5kdW1wcyhtYXRyaXgsIGluZGVudD0yKSkKICAgIHdpdGggb3Blbihvcy5lbnZpcm9uWyJHSVRIVUJfT1VUUFVUIl0sICJhIikgYXMgZjoKICAgICAgICBmLndyaXRlKGYibWF0cml4PXtqc29uLmR1bXBzKG1hdHJpeCl9XG4iKQoKCmRlZiBnZXRfb3ModGFyZ2V0KToKICAgIGlmICJtYWNvcyIgaW4gdGFyZ2V0OgogICAgICAgIGlmIGdldF9jaWJ3X2FyY2hzKHRhcmdldCkgaW4geyJhcm02NCIsICJ1bml2ZXJzYWwyIn06CiAgICAgICAgICAgIHJldHVybiBNMV9SVU5ORVIKICAgICAgICBlbHNlOgogICAgICAgICAgICByZXR1cm4gTUFDSElORV9UWVBFWyJtYWNvcyJdCiAgICBpZiAid2luIiBpbiB0YXJnZXQ6CiAgICAgICAgcmV0dXJuIE1BQ0hJTkVfVFlQRVsid2luZG93cyJdCiAgICByZXR1cm4gTUFDSElORV9UWVBFWyJsaW51eCJdCgoKZGVmIGdldF9jaWJ3X2J1aWxkKHRhcmdldCk6CiAgICBpZiB0YXJnZXQgaW4geyJsaW51eCIsICJtYWNvcyIsICJ3aW5kb3dzIn06CiAgICAgICAgcmV0dXJuIENJQldfQlVJTEQKICAgIHJldHVybiB0YXJnZXQKCgpkZWYgZ2V0X2NpYndfYXJjaHModGFyZ2V0KToKICAgIGZvciBhcmNoIGluIFsiYWFyY2g2NCIsICJwcGM2NGxlIiwgInMzOTB4IiwgImFybXY3bCIsICJhcm02NCIsICJ1bml2ZXJzYWwyIl06CiAgICAgICAgaWYgdGFyZ2V0LmVuZHN3aXRoKGFyY2gpOgogICAgICAgICAgICByZXR1cm4gYXJjaAogICAgcmV0dXJuIENJQldfQVJDSFMKCgpkZWYgZ2V0X2FydGlmYWN0X25hbWUodGFyZ2V0KToKICAgIGFydGlmYWN0X25hbWUgPSByZS5zdWIociJbXFwgLzo8PnwqP1wiJ10iLCAiLSIsIHRhcmdldCkKICAgIGFydGlmYWN0X25hbWUgPSByZS5zdWIociItKyIsICItIiwgYXJ0aWZhY3RfbmFtZSkKICAgIHJldHVybiBhcnRpZmFjdF9uYW1lCgoKZGVmIGdldF9tYXRyaXhfaXRlbSh0YXJnZXQpOgogICAgZXh0cmFfdGFyZ2V0X2FyZ3MgPSB7fQogICAgaWYgaXNpbnN0YW5jZSh0YXJnZXQsIGRpY3QpOgogICAgICAgIGV4dHJhX3RhcmdldF9hcmdzID0gdGFyZ2V0CiAgICAgICAgdGFyZ2V0ID0gZXh0cmFfdGFyZ2V0X2FyZ3MucG9wKCJ0YXJnZXQiKQogICAgcmV0dXJuIHsKICAgICAgICAidGFyZ2V0IjogdGFyZ2V0LAogICAgICAgICJydW5zLW9uIjogZ2V0X29zKHRhcmdldCksCiAgICAgICAgIkNJQldfQlVJTEQiOiBnZXRfY2lid19idWlsZCh0YXJnZXQpLAogICAgICAgICJDSUJXX0FSQ0hTIjogZ2V0X2NpYndfYXJjaHModGFyZ2V0KSwKICAgICAgICAiYXJ0aWZhY3QtbmFtZSI6IGdldF9hcnRpZmFjdF9uYW1lKHRhcmdldCksCiAgICAgICAgKipleHRyYV90YXJnZXRfYXJncywKICAgIH0KCgppZiBfX25hbWVfXyA9PSAiX19tYWluX18iOgogICAgbG9hZF9idWlsZF90YXJnZXRzKCkK
+          LOAD_BUILD_TARGETS_SCRIPT: aW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCByZQoKaW1wb3J0IGNsaWNrCmltcG9ydCB5YW1sCgpNQUNISU5FX1RZUEUgPSB7CiAgICAibGludXgiOiAidWJ1bnR1LWxhdGVzdCIsCiAgICAibWFjb3MiOiAibWFjb3MtbGF0ZXN0IiwKICAgICJ3aW5kb3dzIjogIndpbmRvd3MtbGF0ZXN0IiwKfQoKQ0lCV19CVUlMRCA9IG9zLmVudmlyb24uZ2V0KCJDSUJXX0JVSUxEIiwgIioiKQpDSUJXX0FSQ0hTID0gb3MuZW52aXJvbi5nZXQoIkNJQldfQVJDSFMiLCAiYXV0byIpCgoKQGNsaWNrLmNvbW1hbmQoKQpAY2xpY2sub3B0aW9uKCItLXRhcmdldHMiLCBkZWZhdWx0PSIiKQpkZWYgbG9hZF9idWlsZF90YXJnZXRzKHRhcmdldHMpOgogICAgIiIiU2NyaXB0IHRvIGxvYWQgY2lidWlsZHdoZWVsIHRhcmdldHMgZm9yIEdpdEh1YiBBY3Rpb25zIHdvcmtmbG93LiIiIgogICAgIyBMb2FkIGxpc3Qgb2YgdGFyZ2V0cwogICAgdGFyZ2V0cyA9IHlhbWwubG9hZCh0YXJnZXRzLCBMb2FkZXI9eWFtbC5CYXNlTG9hZGVyKQogICAgcHJpbnQoanNvbi5kdW1wcyh0YXJnZXRzLCBpbmRlbnQ9MikpCgogICAgIyBDcmVhdGUgbWF0cml4CiAgICBtYXRyaXggPSB7ImluY2x1ZGUiOiBbXX0KICAgIGZvciB0YXJnZXQgaW4gdGFyZ2V0czoKICAgICAgICBtYXRyaXhbImluY2x1ZGUiXS5hcHBlbmQoZ2V0X21hdHJpeF9pdGVtKHRhcmdldCkpCgogICAgIyBPdXRwdXQgbWF0cml4CiAgICBwcmludChqc29uLmR1bXBzKG1hdHJpeCwgaW5kZW50PTIpKQogICAgd2l0aCBvcGVuKG9zLmVudmlyb25bIkdJVEhVQl9PVVRQVVQiXSwgImEiKSBhcyBmOgogICAgICAgIGYud3JpdGUoZiJtYXRyaXg9e2pzb24uZHVtcHMobWF0cml4KX1cbiIpCgoKZGVmIGdldF9vcyh0YXJnZXQpOgogICAgaWYgIm1hY29zIiBpbiB0YXJnZXQ6CiAgICAgICAgcmV0dXJuIE1BQ0hJTkVfVFlQRVsibWFjb3MiXQogICAgaWYgIndpbiIgaW4gdGFyZ2V0OgogICAgICAgIHJldHVybiBNQUNISU5FX1RZUEVbIndpbmRvd3MiXQogICAgcmV0dXJuIE1BQ0hJTkVfVFlQRVsibGludXgiXQoKCmRlZiBnZXRfY2lid19idWlsZCh0YXJnZXQpOgogICAgaWYgdGFyZ2V0IGluIHsibGludXgiLCAibWFjb3MiLCAid2luZG93cyJ9OgogICAgICAgIHJldHVybiBDSUJXX0JVSUxECiAgICByZXR1cm4gdGFyZ2V0CgoKZGVmIGdldF9jaWJ3X2FyY2hzKHRhcmdldCk6CiAgICAiIiIKICAgIEhhbmRsZSBub24tbmF0aXZlIGFyY2hpdGVjdHVyZXMKCiAgICBjaWJ3IGFsbG93cyBydW5uaW5nIG5vbi1uYXRpdmUgYnVpbGRzIG9uIHZhcmlvdXMgcGxhdGZvcm1zOgogICAgaHR0cHM6Ly9jaWJ1aWxkd2hlZWwucHlwYS5pby9lbi9zdGFibGUvb3B0aW9ucy8jYXJjaHMKCiAgICBUaGlzIGxvZ2ljIG92ZXJyaWRlcyB0aGUgImF1dG8iIGZsYWcgYmFzZWQgb24gT1MgYW5kIGEgbGlzdCBvZiBzdXBwb3J0ZWQKICAgIG5vbi1uYXRpdmUgYXJjaCBpZiBhIG5vbi1uYXRpdmUgYXJjaCBpcyBnaXZlbiBmb3IgYSBwYXJ0aWN1bGFyIHBsYXRmb3JtIGluCiAgICB0YXJnZXRzLCByYXRoZXIgdGhhbiB0aGUgdXNlciBoYXZpbmcgdG8gZG8gdGhpcyBtYW51YWxseS4KICAgICIiIgogICAgcGxhdGZvcm1fYXJjaHMgPSB7CiAgICAgICAgIyBXZSBub3cgY3Jvc3MgY29tcGlsZSB4ODZfNjQgb24gYXJtNjQgYnkgZGVmYXVsdAogICAgICAgICJtYWNvcyI6IFsidW5pdmVyc2FsMiIsICJ4ODZfNjQiXSwKICAgICAgICAjIFRoaXMgaXMgYSBsaXN0IG9mIHN1cHBvcnRlZCBldW11bGF0ZWQgYXJjaGVzIG9uIGxpbnV4CiAgICAgICAgImxpbnV4IjogWyJhYXJjaDY0IiwgInBwYzY0bGUiLCAiczM5MHgiLCAiYXJtdjdsIl0sCiAgICB9CiAgICBmb3IgcGxhdGZvcm0sIGFyY2hzIGluIHBsYXRmb3JtX2FyY2hzLml0ZW1zKCk6CiAgICAgICAgaWYgcGxhdGZvcm0gaW4gdGFyZ2V0OgogICAgICAgICAgICBmb3IgYXJjaCBpbiBhcmNoczoKICAgICAgICAgICAgICAgIGlmIHRhcmdldC5lbmRzd2l0aChhcmNoKToKICAgICAgICAgICAgICAgICAgICByZXR1cm4gYXJjaAoKICAgICMgSWYgbm8gZXhwbGljdCBhcmNoIGhhcyBiZWVuIHNwZWNpZmllZCBidWlsZCBib3RoIGFybTY0IGFuZCB4ODZfNjQgb24gbWFjb3MKICAgIGlmICJtYWNvcyIgaW4gdGFyZ2V0OgogICAgICAgIHJldHVybiBvcy5lbnZpcm9uLmdldCgiQ0lCV19BUkNIUyIsICJhcm02NCB4ODZfNjQiKQoKICAgIHJldHVybiBDSUJXX0FSQ0hTCgoKZGVmIGdldF9hcnRpZmFjdF9uYW1lKHRhcmdldCk6CiAgICBhcnRpZmFjdF9uYW1lID0gcmUuc3ViKHIiW1xcIC86PD58Kj9cIiddIiwgIi0iLCB0YXJnZXQpCiAgICBhcnRpZmFjdF9uYW1lID0gcmUuc3ViKHIiLSsiLCAiLSIsIGFydGlmYWN0X25hbWUpCiAgICByZXR1cm4gYXJ0aWZhY3RfbmFtZQoKCmRlZiBnZXRfbWF0cml4X2l0ZW0odGFyZ2V0KToKICAgIGV4dHJhX3RhcmdldF9hcmdzID0ge30KICAgIGlmIGlzaW5zdGFuY2UodGFyZ2V0LCBkaWN0KToKICAgICAgICBleHRyYV90YXJnZXRfYXJncyA9IHRhcmdldAogICAgICAgIHRhcmdldCA9IGV4dHJhX3RhcmdldF9hcmdzLnBvcCgidGFyZ2V0IikKICAgIHJldHVybiB7CiAgICAgICAgInRhcmdldCI6IHRhcmdldCwKICAgICAgICAicnVucy1vbiI6IGdldF9vcyh0YXJnZXQpLAogICAgICAgICJDSUJXX0JVSUxEIjogZ2V0X2NpYndfYnVpbGQodGFyZ2V0KSwKICAgICAgICAiQ0lCV19BUkNIUyI6IGdldF9jaWJ3X2FyY2hzKHRhcmdldCksCiAgICAgICAgImFydGlmYWN0LW5hbWUiOiBnZXRfYXJ0aWZhY3RfbmFtZSh0YXJnZXQpLAogICAgICAgICoqZXh0cmFfdGFyZ2V0X2FyZ3MsCiAgICB9CgoKaWYgX19uYW1lX18gPT0gIl9fbWFpbl9fIjoKICAgIGxvYWRfYnVpbGRfdGFyZ2V0cygpCg==
       - id: set-outputs
         run: python load_build_targets.py --targets "${{ inputs.targets }}"
         shell: sh

.github/workflows/test_publish.yml

@@ -27,8 +27,11 @@ jobs:
       test_command: pytest --pyargs test_package
       targets: |
         - linux
-        - cp311-macosx_x86_64
+        - cp31?-macos*
+        - target: cp311-macosx_x86_64
+          runs-on: macos-13
         - cp312-macosx_arm64
+        - cp313-macosx_universal2
         - cp3?-win_amd64
         - cp312-manylinux_aarch64
         - cp313-manylinux_x86_64

.github/workflows/test_tox.yml

@@ -20,6 +20,7 @@ jobs:
       envs: |
         - linux: py312-inputs-linux
         - macos: py311-inputs-macos
+        - macos: py39-inputs-macos
         - windows: py310-inputs-windows
           toxargs: '-v'
           runs-on: windows-2019

.github/workflows/tox.yml

@@ -35,8 +35,9 @@ Any other target is assumed to be a value for the ``CIBW_BUILD``
 environment variable (e.g. ``cp3?-macosx_x86_64``). In this case the OS
 to run cibuildwheel on is extracted from the target.
 
-Targets that end with ``aarch64``, ``arm64`` and ``universal2`` are also
-supported without any additional configuration of cibuildwheel.
+Targets which end with non-native architectures such as ``aarch64`` on linux or
+``x86_64`` on macos are supported and will be emulated (on linux) or cross
+compiled.
 
 **Note:** ``targets`` is a *string* and must be specified as a
 literal block scalar using the ``|``. (Without the ``|``, it must also
@@ -57,6 +58,15 @@ To not build any wheels:
 
    targets: ''
 
+For additional configuration extra arguments can be passed by making a target a dictionary.
+An example of this is specifying the runner for a target, such as building macos x86_64 wheels on native x86_64 runners:
+
+.. code:: yaml
+
+    targets:
+      - target: cp311-macosx_x86_64
+          runs-on: macos-13
+
 sdist
 ^^^^^
 

docs/source/publish.rst

@@ -7,10 +7,9 @@
 
 MACHINE_TYPE = {
     "linux": "ubuntu-latest",
-    "macos": "macos-12",
+    "macos": "macos-latest",
     "windows": "windows-latest",
 }
-M1_RUNNER = "macos-14"
 
 CIBW_BUILD = os.environ.get("CIBW_BUILD", "*")
 CIBW_ARCHS = os.environ.get("CIBW_ARCHS", "auto")
@@ -37,10 +36,7 @@ def load_build_targets(targets):
 
 def get_os(target):
     if "macos" in target:
-        if get_cibw_archs(target) in {"arm64", "universal2"}:
-            return M1_RUNNER
-        else:
-            return MACHINE_TYPE["macos"]
+        return MACHINE_TYPE["macos"]
     if "win" in target:
         return MACHINE_TYPE["windows"]
     return MACHINE_TYPE["linux"]
@@ -53,9 +49,32 @@ def get_cibw_build(target):
 
 
 def get_cibw_archs(target):
-    for arch in ["aarch64", "ppc64le", "s390x", "armv7l", "arm64", "universal2"]:
-        if target.endswith(arch):
-            return arch
+    """
+    Handle non-native architectures
+
+    cibw allows running non-native builds on various platforms:
+    https://cibuildwheel.pypa.io/en/stable/options/#archs
+
+    This logic overrides the "auto" flag based on OS and a list of supported
+    non-native arch if a non-native arch is given for a particular platform in
+    targets, rather than the user having to do this manually.
+    """
+    platform_archs = {
+        # We now cross compile x86_64 on arm64 by default
+        "macos": ["universal2", "x86_64"],
+        # This is a list of supported eumulated arches on linux
+        "linux": ["aarch64", "ppc64le", "s390x", "armv7l"],
+    }
+    for platform, archs in platform_archs.items():
+        if platform in target:
+            for arch in archs:
+                if target.endswith(arch):
+                    return arch
+
+    # If no explict arch has been specified build both arm64 and x86_64 on macos
+    if "macos" in target:
+        return os.environ.get("CIBW_ARCHS", "arm64 x86_64")
+
     return CIBW_ARCHS
 
 

tools/load_build_targets.py

@@ -139,7 +139,7 @@ def get_matrix_item(env, global_libraries, global_string_parameters,
     # if Python is <3.10 we can't use macos-latest which is arm64
     try:
         if Version(item["python_version"]) < Version('3.10') and item["os"] == "macos-latest":
-            item["os"] = "macos-12"
+            item["os"] = "macos-13"
     except InvalidVersion:
         # python_version might be for example 'pypy-3.10' which won't parse
         pass