build: adjust platforms for release and CI, add Windows for CI (#950)

* build: adjust platforms for release and CI

* fix: adjust workflow

* fix: use `.` instead of `source` for windows compatibility

* fix: another try to make test work on Windows

* refactor: avoid duplicating workflow step

* fix: explicitly shutdown process pool at exit

* fix: register `atexit` for process pools later so it's executed earlier

* fix: add `wait = True` on shutdown `ProcessPoolExecutor`

* fix: avoid unncessary holding of `ProcessPoolExecutor` to allow its GC

* debug: show more messages for debug

* fix: register shutting down process pool for early execution

* Revert "fix: register shutting down process pool for early execution"

This reverts commit 4c716e5.

* fix: explicitly shutdown in pytest fixture

* fix: try to avoid wait on process pool shutdown

* fix: ignore `SIGINT` during shutting down the pool

* fix: directly exit with 0 after tests in Windows

* debug: flush stdout errors, make test run faster

* debug: switch shell in Windows to cmd

* cleanup after pass

Author: georgeh0

.github/workflows/_test.yml

@@ -10,49 +10,56 @@ jobs:
     build-test:
         strategy:
             matrix:
-                python-version: [3.11, 3.13]
-                platform: [ubuntu-latest, macos-latest]
-        runs-on: ${{ matrix.platform }}
+                python-version: [3.11]
+                platform:
+                  - { runner: ubuntu-latest, python_exec: ".venv/bin/python" }
+                  - { runner: ubuntu-24.04-arm, python_exec: ".venv/bin/python" }
+                  - { runner: macos-latest, python_exec: ".venv/bin/python" }
+                  - { runner: macos-13, python_exec: ".venv/bin/python" }
+                  - { runner: windows-latest, python_exec: ".venv\\Scripts\\python" }
+        runs-on: ${{ matrix.platform.runner }}
         steps:
         - uses: actions/checkout@v4
 
+        - uses: actions/setup-python@v5
+          id: setup_python
+          with:
+            python-version: ${{ matrix.python-version }}
+            cache: 'pip'
+
         - run: rustup toolchain install stable --profile minimal
         - name: Rust Cache
           uses: Swatinem/rust-cache@v2
           with:
-            key: ${{ runner.os }}-rust-${{ matrix.python-version }}
-        - name: Rust build
-          run: cargo build --verbose
+            key: rust-${{ matrix.platform.runner }}-${{ matrix.python-version }}
         - name: Rust tests
           run: cargo test --verbose
 
-        - uses: actions/setup-python@v5
-          id: setup_python
-          with:
-            python-version: ${{ matrix.python-version }}
-            cache: 'pip'
         - uses: actions/cache@v4
           with:
             path: .venv
-            key: ${{ runner.os }}-pyenv-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}
+            key: pyenv-${{ matrix.platform.runner }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}
             restore-keys: |
-              ${{ runner.os }}-pyenv-${{ steps.setup_python.outputs.python-version }}-
+              pyenv-${{ matrix.platform.runner }}-${{ steps.setup_python.outputs.python-version }}-
+
         - name: Setup venv
           run: |
             python -m venv .venv
         - name: Install Python toolchains
           run: |
-            source .venv/bin/activate
-            pip install maturin mypy pytest pytest-asyncio
+            ${{ matrix.platform.python_exec }} -m pip install maturin mypy pytest pytest-asyncio
         - name: Python build
           run: |
-            source .venv/bin/activate
-            maturin develop -E all
+            ${{ matrix.platform.python_exec }} -m maturin develop -E all
         - name: Python type check (mypy)
           run: |
-            source .venv/bin/activate
-            mypy python
+            ${{ matrix.platform.python_exec }} -m mypy python
         - name: Python tests
+          if: ${{ !startsWith(matrix.platform.runner, 'windows') }}
+          run: |
+            ${{ matrix.platform.python_exec }} -m pytest --capture=no python/cocoindex/tests
+        - name: Python tests (Windows cmd)
+          if: ${{ startsWith(matrix.platform.runner, 'windows') }}
+          shell: cmd  # Use `cmd` to run test for Windows, as PowerShell doesn't detect exit code by `os._exit(0)` correctly.
           run: |
-            source .venv/bin/activate
-            pytest python/cocoindex/tests
+            ${{ matrix.platform.python_exec }} -m pytest --capture=no python/cocoindex/tests

.github/workflows/release.yml

@@ -30,11 +30,11 @@ jobs:
     strategy:
       matrix:
         platform:
-          - { os: linux, runner: ubuntu-24.04, target: x86_64, container: "ghcr.io/rust-cross/manylinux_2_28-cross:x86_64" }
-          - { os: linux, runner: ubuntu-24.04, target: aarch64, container: "ghcr.io/rust-cross/manylinux_2_28-cross:aarch64" }
-          - { os: windows, runner: windows-latest, target: x64 }
+          - { os: linux, runner: ubuntu-latest, target: x86_64, container: "ghcr.io/rust-cross/manylinux_2_28-cross:x86_64" }
+          - { os: linux, runner: ubuntu-24.04-arm, target: aarch64, container: "ghcr.io/rust-cross/manylinux_2_28-cross:aarch64" }
+          - { os: macos, runner: macos-latest, target: aarch64 }
           - { os: macos, runner: macos-13, target: x86_64 }
-          - { os: macos, runner: macos-14, target: aarch64 }
+          - { os: windows, runner: windows-latest, target: x64 }
     steps:
       - uses: actions/checkout@v4
       - uses: actions/download-artifact@v4
@@ -47,7 +47,7 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist --find-interpreter
+          args: --release --out dist
           sccache: 'true'
           manylinux: auto
           container: ${{ matrix.platform.container }}