ci: fix cross (#542)

Author: davidhewitt

.github/workflows/ci.yml

@@ -250,50 +250,15 @@ jobs:
       - name: Setup python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10" # must match the ubuntu version in the install, and also the STANDALONE_PYTHON_VERSION below
+          python-version: "3.x"
       - uses: dtolnay/rust-toolchain@stable
       - name: Install cross
-        uses: taiki-e/install-action@v2
-        with:
-          tool: cross
-      - name: Build package
-        run: pip install -e .
-      - name: Build wheel using cross
-        shell: bash
-        env:
-          CARGO: cross
-          CARGO_BUILD_TARGET: aarch64-unknown-linux-gnu
-          PYO3_CROSS_LIB_DIR: /opt/python/cp310-cp310/lib
-          DIST_EXTRA_CONFIG: /tmp/build-opts.cfg
-        run: |
-          cd examples/namespace_package
-          docker build \
-            --build-arg STANDALONE_PYTHON_VERSION=3.10.15 \
-            --build-arg STANDALONE_PYTHON_RELEASE=20241016 \
-            -t cross-pyo3:aarch64-unknown-linux-gnu \
-            .
-          python -m pip install build wheel
-          echo -e "[bdist_wheel]\nplat_name=manylinux2014_aarch64" > $DIST_EXTRA_CONFIG
-          python -m build --no-isolation
-          ls -la dist/
-          unzip -l dist/*.whl  # debug all files inside wheel file
-      - uses: uraimo/[email protected]
-        name: Install built wheel
-        with:
-          arch: aarch64
-          distro: ubuntu22.04
-          dockerRunArgs: |
-            --volume "${PWD}/examples/namespace_package:/io"
-          install: |
-            apt-get update
-            apt-get install -y --no-install-recommends python3 python3-venv python3-pip
-            pip3 install -U pip
-          run: |
-            python3 -m venv .venv
-            source .venv/bin/activate
-            pip install namespace_package --no-index --find-links /io/dist/ --force-reinstall
-            python -c "from namespace_package import rust; assert rust.rust_func() == 14"
-            python -c "from namespace_package import python; assert python.python_func() == 15"
+        # need cross HEAD for https://github.com/cross-rs/cross/issues/1645
+        # will probably be ok to use released cross from 0.3
+        run: cargo install cross --git https://github.com/cross-rs/cross
+      - uses: docker/setup-qemu-action@v3
+      - run: pip install nox
+      - run: nox -s test-cross
 
   test-zigbuild:
     runs-on: ubuntu-latest

examples/namespace_package/Cross.toml

@@ -1,7 +1,16 @@
 [target.aarch64-unknown-linux-gnu]
 image = "cross-pyo3:aarch64-unknown-linux-gnu"
 env.passthrough = [
+    "PYO3_PRINT_CONFIG",
     "RUST_BACKTRACE",
     "RUST_LOG",
     "PYO3_CROSS_LIB_DIR",
 ]
+
+# This avoids cross picking up custom linker from host ~/.cargo/config.toml
+# See: https://github.com/cross-rs/cross/issues/621
+[build.env]
+passthrough = [
+    "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc",
+    "RUSTFLAGS",
+]

examples/namespace_package/Dockerfile

@@ -2,13 +2,9 @@ FROM quay.io/pypa/manylinux2014_aarch64 AS manylinux
 
 FROM ghcr.io/cross-rs/aarch64-unknown-linux-gnu:edge
 
-ARG STANDALONE_PYTHON_RELEASE
-ARG STANDALONE_PYTHON_VERSION
-
-RUN test -n "$STANDALONE_PYTHON_RELEASE" || (echo "STANDALONE_PYTHON_RELEASE not set" && false)
-RUN test -n "$STANDALONE_PYTHON_VERSION" || (echo "STANDALONE_PYTHON_VERSION not set" && false)
-
-RUN curl -L https://github.com/indygreg/python-build-standalone/releases/download/${STANDALONE_PYTHON_RELEASE}/cpython-${STANDALONE_PYTHON_VERSION}+${STANDALONE_PYTHON_RELEASE}-x86_64-unknown-linux-gnu-install_only.tar.gz | tar -xz -C /usr/local
+# PyO3 needs a host Python interpreter to parse the sysconfigdata of the
+# cross-compile interpreter found in /opt/python
+RUN apt-get update && apt-get --assume-yes install python3
 
 ENV PATH=/usr/local/python/bin:$PATH
 

noxfile.py

@@ -2,6 +2,8 @@
 from inspect import cleandoc as heredoc
 from glob import glob
 from pathlib import Path
+import shutil
+import sys
 
 import nox
 import nox.command
@@ -135,6 +137,77 @@ def test_crossenv(session: nox.Session):
     )
 
 
+@nox.session(name="test-cross")
+def test_cross(session: nox.Session):
+    session.install(".")
+    session.install("-U", "setuptools", "build")
+
+    shutil.rmtree("examples/namespace_package/dist", ignore_errors=True)
+
+    namespace_package = Path(__file__).parent / "examples" / "namespace_package"
+    os.chdir(namespace_package)
+    session.run(
+        "docker",
+        "build",
+        "-t",
+        "cross-pyo3:aarch64-unknown-linux-gnu",
+        ".",
+        external=True,
+    )
+
+    major_version = sys.version_info[0]
+    minor_version = sys.version_info[1]
+
+    cpXY = f"cp{major_version}{minor_version}"
+
+    tmp = Path(session.create_tmp())
+    extra_config = tmp / "build-opts.cfg"
+    build_config = """
+        [bdist_wheel]
+        plat_name = manylinux2014_aarch64
+        """
+    extra_config.write_text(heredoc(build_config), encoding="utf-8")
+
+    build_env = os.environ.copy()
+    build_env["DIST_EXTRA_CONFIG"] = str(extra_config.absolute())
+    build_env["CARGO"] = "cross"
+    build_env["CARGO_BUILD_TARGET"] = "aarch64-unknown-linux-gnu"
+    build_env["PYO3_CROSS_LIB_DIR"] = f"/opt/python/{cpXY}-{cpXY}/lib"
+
+    # build wheel using cross
+    #
+    # if this step fails, you may need to install cross:
+    # cargo install cross --git https://github.com/cross-rs/cross
+    session.run("python", "-m", "build", "--no-isolation", env=build_env)
+
+    script_check = """
+set -eux
+python3 --version
+python3 -m venv .venv
+source .venv/bin/activate
+pip install namespace_package --no-index --find-links /io/dist/ --force-reinstall
+python -c "from namespace_package import rust; assert rust.rust_func() == 14"
+python -c "from namespace_package import python; assert python.python_func() == 15"
+"""
+
+    session.run(
+        "docker",
+        "run",
+        "--rm",
+        "-v",
+        f"{namespace_package}:/io",
+        "-w",
+        "/io",
+        "--platform",
+        "aarch64",
+        f"python:3.{minor_version}",
+        "bash",
+        "-c",
+        script_check,
+        external=True,
+    )
+
+
 @nox.session()
 def ruff(session: nox.Session):
     session.install("ruff")