From b9f417ef83f68ddd14f3bdaf3b0410d5f538f9c1 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 29 Aug 2025 12:13:49 +0100 Subject: [PATCH 1/3] ci: fix `cross` --- .github/workflows/ci.yml | 48 +++--------------- examples/namespace_package/Cross.toml | 9 ++++ examples/namespace_package/Dockerfile | 10 ++-- noxfile.py | 73 +++++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 49 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dea70690..8e2b9111 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -250,50 +250,14 @@ jobs: - name: Setup python uses: actions/setup-python@v5 with: - python-version: "3.10" # must match the ubuntu version in the install, and also the STANDALONE_PYTHON_VERSION below + python-version: "3.x" - uses: dtolnay/rust-toolchain@stable - name: Install cross - uses: taiki-e/install-action@v2 - with: - tool: cross - - name: Build package - run: pip install -e . - - name: Build wheel using cross - shell: bash - env: - CARGO: cross - CARGO_BUILD_TARGET: aarch64-unknown-linux-gnu - PYO3_CROSS_LIB_DIR: /opt/python/cp310-cp310/lib - DIST_EXTRA_CONFIG: /tmp/build-opts.cfg - run: | - cd examples/namespace_package - docker build \ - --build-arg STANDALONE_PYTHON_VERSION=3.10.15 \ - --build-arg STANDALONE_PYTHON_RELEASE=20241016 \ - -t cross-pyo3:aarch64-unknown-linux-gnu \ - . - python -m pip install build wheel - echo -e "[bdist_wheel]\nplat_name=manylinux2014_aarch64" > $DIST_EXTRA_CONFIG - python -m build --no-isolation - ls -la dist/ - unzip -l dist/*.whl # debug all files inside wheel file - - uses: uraimo/run-on-arch-action@v3.0.1 - name: Install built wheel - with: - arch: aarch64 - distro: ubuntu22.04 - dockerRunArgs: | - --volume "${PWD}/examples/namespace_package:/io" - install: | - apt-get update - apt-get install -y --no-install-recommends python3 python3-venv python3-pip - pip3 install -U pip - run: | - python3 -m venv .venv - source .venv/bin/activate - pip install namespace_package --no-index --find-links /io/dist/ --force-reinstall - python -c "from namespace_package import rust; assert rust.rust_func() == 14" - python -c "from namespace_package import python; assert python.python_func() == 15" + # need cross HEAD for https://github.com/cross-rs/cross/issues/1645 + # will probably be ok to use released cross from 0.3 + run: cargo install cross --git https://github.com/cross-rs/cross + - run: pip instal nox + - run: nox -s test-cross test-zigbuild: runs-on: ubuntu-latest diff --git a/examples/namespace_package/Cross.toml b/examples/namespace_package/Cross.toml index 83d99f2f..dc8ceed9 100644 --- a/examples/namespace_package/Cross.toml +++ b/examples/namespace_package/Cross.toml @@ -1,7 +1,16 @@ [target.aarch64-unknown-linux-gnu] image = "cross-pyo3:aarch64-unknown-linux-gnu" env.passthrough = [ + "PYO3_PRINT_CONFIG", "RUST_BACKTRACE", "RUST_LOG", "PYO3_CROSS_LIB_DIR", ] + +# This avoids cross picking up custom linker from host ~/.cargo/config.toml +# See: https://github.com/cross-rs/cross/issues/621 +[build.env] +passthrough = [ + "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc", + "RUSTFLAGS", +] diff --git a/examples/namespace_package/Dockerfile b/examples/namespace_package/Dockerfile index 8521494d..0bc7676a 100644 --- a/examples/namespace_package/Dockerfile +++ b/examples/namespace_package/Dockerfile @@ -2,13 +2,9 @@ FROM quay.io/pypa/manylinux2014_aarch64 AS manylinux FROM ghcr.io/cross-rs/aarch64-unknown-linux-gnu:edge -ARG STANDALONE_PYTHON_RELEASE -ARG STANDALONE_PYTHON_VERSION - -RUN test -n "$STANDALONE_PYTHON_RELEASE" || (echo "STANDALONE_PYTHON_RELEASE not set" && false) -RUN test -n "$STANDALONE_PYTHON_VERSION" || (echo "STANDALONE_PYTHON_VERSION not set" && false) - -RUN curl -L https://github.com/indygreg/python-build-standalone/releases/download/${STANDALONE_PYTHON_RELEASE}/cpython-${STANDALONE_PYTHON_VERSION}+${STANDALONE_PYTHON_RELEASE}-x86_64-unknown-linux-gnu-install_only.tar.gz | tar -xz -C /usr/local +# PyO3 needs a host Python interpreter to parse the sysconfigdata of the +# cross-compile interpreter found in /opt/python +RUN apt-get update && apt-get --assume-yes install python3 ENV PATH=/usr/local/python/bin:$PATH diff --git a/noxfile.py b/noxfile.py index cc93b46b..85ee7e9d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -2,6 +2,8 @@ from inspect import cleandoc as heredoc from glob import glob from pathlib import Path +import shutil +import sys import nox import nox.command @@ -136,6 +138,77 @@ def test_crossenv(session: nox.Session): ) +@nox.session(name="test-cross") +def test_cross(session: nox.Session): + session.install(".") + session.install("-U", "setuptools", "build") + + shutil.rmtree("examples/namespace_package/dist", ignore_errors=True) + + namespace_package = Path(__file__).parent / "examples" / "namespace_package" + os.chdir(namespace_package) + session.run( + "docker", + "build", + "-t", + "cross-pyo3:aarch64-unknown-linux-gnu", + ".", + external=True, + ) + + major_version = sys.version_info[0] + minor_version = sys.version_info[1] + + cpXY = f"cp{major_version}{minor_version}" + + tmp = Path(session.create_tmp()) + extra_config = tmp / "build-opts.cfg" + build_config = """ + [bdist_wheel] + plat_name = manylinux2014_aarch64 + """ + extra_config.write_text(heredoc(build_config), encoding="utf-8") + + build_env = os.environ.copy() + build_env["DIST_EXTRA_CONFIG"] = str(extra_config.absolute()) + build_env["CARGO"] = "cross" + build_env["CARGO_BUILD_TARGET"] = "aarch64-unknown-linux-gnu" + build_env["PYO3_CROSS_LIB_DIR"] = f"/opt/python/{cpXY}-{cpXY}/lib" + + # build wheel using cross + # + # if this step fails, you may need to install cross: + # cargo install cross --git https://github.com/cross-rs/cross + session.run("python", "-m", "build", "--no-isolation", env=build_env) + + script_check = """ +set -eux +python3 --version +python3 -m venv .venv +source .venv/bin/activate +pip install namespace_package --no-index --find-links /io/dist/ --force-reinstall +python -c "from namespace_package import rust; assert rust.rust_func() == 14" +python -c "from namespace_package import python; assert python.python_func() == 15" +""" + + session.run( + "docker", + "run", + "--rm", + "-v", + f"{namespace_package}:/io", + "-w", + "/io", + "--platform", + "aarch64", + f"python:3.{minor_version}", + "bash", + "-c", + script_check, + external=True, + ) + + @nox.session() def ruff(session: nox.Session): session.install("ruff") From 3089482018cd4a85efb0867bfbbebb7b0cb9ed93 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 29 Aug 2025 12:58:04 +0100 Subject: [PATCH 2/3] fix typo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e2b9111..f83c358b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -256,7 +256,7 @@ jobs: # need cross HEAD for https://github.com/cross-rs/cross/issues/1645 # will probably be ok to use released cross from 0.3 run: cargo install cross --git https://github.com/cross-rs/cross - - run: pip instal nox + - run: pip install nox - run: nox -s test-cross test-zigbuild: From d120f66c3f9a889236bc0a136872accd42c1ce00 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 29 Aug 2025 13:13:00 +0100 Subject: [PATCH 3/3] enable qemu on docker for `test-cross` --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f83c358b..a6076515 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -256,6 +256,7 @@ jobs: # need cross HEAD for https://github.com/cross-rs/cross/issues/1645 # will probably be ok to use released cross from 0.3 run: cargo install cross --git https://github.com/cross-rs/cross + - uses: docker/setup-qemu-action@v3 - run: pip install nox - run: nox -s test-cross