fix: python publishing pipeline (#2898)

1. Stop running pytest an additional time after building the wheel. As
far as I can tell it triggers another build which is the actual code it
runs. Instead we just run `pytest` in an isolated env in CI.
2. Explicitly configure the linux compatibility we want to target.
3. Try and make better use of uv's isolation capabilities to get more
predictable outcomes with steps that don't depend on the results or
state generated by previous steps.

Some of the issues we've had are caused by a surprising behavior by
uv/maturin/pep517 that might be a bug, I've filed an issue with maturin
and hopefully I'll be able to make some progress with them.

Author: AdamGS

.github/workflows/bench-pr.yml

@@ -97,7 +97,7 @@ jobs:
           echo '<details>' >> comment.md
           echo '<summary>Table of Results</summary>' >> comment.md
           echo '' >> comment.md
-          uv run scripts/compare-benchmark-jsons.py base.json ${{ matrix.benchmark.id }}.json \
+          uv run --no-project scripts/compare-benchmark-jsons.py base.json ${{ matrix.benchmark.id }}.json \
             >> comment.md
           echo '</details>' >> comment.md
 

.github/workflows/ci.yml

@@ -48,7 +48,10 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: "recursive"
-      - uses: spiraldb/actions/.github/actions/[email protected]
+      - name: Install uv
+        uses: spiraldb/actions/.github/actions/[email protected]
+        with:
+          sync: false
       - name: Python Lint - Format
         run: uv run ruff format --check .
       - name: Python Lint - Ruff
@@ -57,56 +60,64 @@ jobs:
   python-test:
     name: "Python (test)"
     runs-on: ubuntu-latest
+    env:
+      RUST_LOG: "info,uv=debug"
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: "recursive"
       - uses: ./.github/actions/cleanup
       - uses: rui314/setup-mold@v1
       - uses: ./.github/actions/setup-rust
-      - uses: spiraldb/actions/.github/actions/[email protected]
+      - name: Install uv
+        uses: spiraldb/actions/.github/actions/[email protected]
+        with:
+          sync: false
 
       # Ensure the stub files are up-to-date with the public API of the native module.
       - name: Stubtest - PyVortex
         run: |
-          uv run stubtest vortex._lib --allowlist python/vortex/_lib/mypy.allowlist
+          uv run --isolated --all-packages stubtest vortex._lib --allowlist python/vortex/_lib/mypy.allowlist
         working-directory: pyvortex/
 
       - name: Pytest - PyVortex
         run: |
-          uv run pytest --benchmark-disable test/
+          uv run --isolated --all-packages pytest --benchmark-disable test/
         working-directory: pyvortex/
 
       - name: Doctest - PyVortex
         run: |
-          uv run make clean && uv run make doctest
+          uv run --isolated --all-packages make doctest
         working-directory: docs/
 
       - name: Ensure docs build - PyVortex
         run: |
-          uv run make html
+          uv run --isolated --all-packages make html
         working-directory: docs/
 
+      - uses: mlugg/setup-zig@v1
       - name: Ensure wheel and sdist can be built on Linux - PyVortex
+        shell: bash
         run: |
-          # force linux compatibility to ensure there is exactly one wheel ergo no ambiguity in installation
-          uv run maturin build --interpreter python3.10 --compatibility linux
+          echo "Clearing wheel target directory"
+          rm -rf ../target/wheels/
+
+          uv venv
+          uv pip install "maturin[zig]"
+          uv tool run maturin build --interpreter python3.10 --zig
+          uv tool run maturin build --interpreter python3.10 --zig --sdist
+
+          file_count=$(ls -1 ../target/wheels/ | wc -l)
+
+          if [[ $file_count -ne 2 ]]; then
+            echo "Unexpected number of files detected ${file_count}:"
+            ls ../target/wheels/
+            exit 1
+          else
+            echo "Generated two files"
+          fi
         working-directory: pyvortex/
 
-      - name: Ensure wheel can run tests successfully
-        run: |
-          set -ex
-
-          echo creating a fresh venv which does *not* have the normal dependencies to ensure the wheel dependencies are correct
-
-          rm -rf .venv
-          uv sync --only-dev
-
-          uv run pip install target/wheels/vortex_array-*.whl || { uv run pip debug --verbose ; exit 1 ; }
-          uv run pip install pytest
-          cd pyvortex/test
-          uv run pytest
-
   rust-docs:
     name: "Rust (docs)"
     runs-on:

.github/workflows/docs.yml

@@ -20,11 +20,13 @@ jobs:
           submodules: "recursive"
       - uses: ./.github/actions/cleanup
       - uses: ./.github/actions/setup-rust
-      - uses: spiraldb/actions/.github/actions/[email protected]
-
+      - name: Install uv
+        uses: spiraldb/actions/.github/actions/[email protected]
+        with:
+          sync: false
       - name: build Python and Rust docs
         run: |
-          uv run make -C docs html
+          uv run --all-packages make -C docs html
       - name: Upload static files as artifact
         id: deployment
         uses: actions/upload-pages-artifact@v3

.github/workflows/release-plz.yml

@@ -75,31 +75,20 @@ jobs:
           rust-toolchain: ${{ steps.rust-toolchain.version }}
           working-directory: pyvortex
           target: x86_64
-          args: --release --interpreter python3.10 --sdist
+          command: build
+          args: --release --interpreter python3.10 --auditwheel check --sdist
       - name: Build wheels - aarch64
         uses: PyO3/maturin-action@v1
         with:
           rust-toolchain: ${{ steps.rust-toolchain.version }}
           working-directory: pyvortex
           target: aarch64  # NB: aarch64 becomes arm64 in the wheel's platform tag.
-          args: --release --interpreter python3.10
+          command: build
+          args: --release --interpreter python3.10 --auditwheel check
       - name: Install UV
         uses: spiraldb/actions/.github/actions/[email protected]
         with:
           sync: false
-      - name: test wheel
-        run: |
-          set -ex
-
-          ls -al target/wheels
-
-          echo removing linux_x86_64 if it exists because PyPI will reject the package if it is present
-          rm -f target/wheels/*linux_x86_64.whl
-
-          uv run pip install --no-deps --force-reinstall vortex-array --no-index --find-links target/wheels
-
-          cd pyvortex/test
-          uv run pytest
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:
@@ -128,20 +117,8 @@ jobs:
           working-directory: pyvortex
           target: x86_64
           manylinux: auto
-          args: --release --interpreter python3.10
-      - name: test wheel
-        run: |
-          set -ex
-
-          ls -al target/wheels
-
-          echo removing linux_x86_64 if it exists because PyPI will reject the package if it is present
-          rm -f target/wheels/*linux_x86_64.whl
-
-          uv run pip install --no-deps --force-reinstall vortex-array --no-index --find-links target/wheels
-
-          cd pyvortex/test
-          uv run pytest
+          command: build
+          args: --release --interpreter python3.10 --auditwheel check --zig
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:

README.md

@@ -6,7 +6,7 @@
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/vortex-array)](https://pypi.org/project/vortex-array/)
 
 > \[!TIP\]
-> Check out our [Docs](https://docs.vortex.dev/) and per-commit [Benchmarks](https://bench.vortex.dev) 
+> Check out our [Docs](https://docs.vortex.dev/) and per-commit [Benchmarks](https://bench.vortex.dev)
 
 Vortex is an extensible, state-of-the-art columnar file format, with associated tools for working with compressed Apache
 Arrow arrays in-memory, on-disk, and over-the-wire.
@@ -245,7 +245,7 @@ This repo uses uv to manage the combined Rust/Python monorepo build. After insta
 
 ```bash
 # Install uv from https://docs.astral.sh/uv/getting-started/installation/
-uv sync
+uv sync --all-packages
 ```
 
 ## License

pyvortex/pyproject.toml

@@ -11,6 +11,7 @@ requires-python = ">= 3.10"
 polars = ["polars>=1.24.0"]
 pandas = ["pandas>=2.2.0"]
 numpy = ["numpy>=1.26.0"]
+duckdb = ["duckdb>=1.1.2"]
 
 [project.urls]
 Documentation = "https://docs.vortex.dev"
@@ -29,12 +30,14 @@ managed = true
 python-source = "python"
 module-name = "vortex._lib"
 features = ["pyo3/extension-module"]
+compatibility = "manylinux2014"
 include = [
     { path = "rust-toolchain.toml", format = "sdist" },
     { path = "README.md", format = "sdist" },
     { path = "python/vortex/py.typed", format = "sdist" },
 ]
 
+
 [dependency-groups]
 dev = [
     "mypy>=1.14.1",