perf[ci]: speed up ci MORE (#5576)

Signed-off-by: Joe Isaacs <[email protected]>

Author: joseph-isaacs

.github/actions/setup-rust/action.yml

@@ -19,37 +19,40 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Rust Version
-      id: rust-version
+    - name: Resolve Toolchain Config
+      id: toolchain-config
       shell: bash
-      run: echo "version=$(cat rust-toolchain.toml | grep channel | awk -F'\"' '{print $2}')" >> $GITHUB_OUTPUT
+      run: |
+        DEFAULT_VERSION=$(grep channel rust-toolchain.toml | awk -F'"' '{print $2}')
+        TOOLCHAIN="${TOOLCHAIN_OVERRIDE:-$DEFAULT_VERSION}"
+        echo "toolchain=$TOOLCHAIN" >> $GITHUB_OUTPUT
+        echo "targets=$TARGETS" >> $GITHUB_OUTPUT
+      env:
+        TOOLCHAIN_OVERRIDE: ${{ inputs.toolchain }}
+        TARGETS: ${{ inputs.targets }}
 
     - name: Install Mold
       if: runner.os == 'Linux'
       uses: rui314/setup-mold@v1
 
     - name: Rust Toolchain
       id: rust-toolchain
-      uses: dtolnay/rust-toolchain@master
+      uses: dtolnay/rust-toolchain@stable
       if: steps.rustup-cache.outputs.cache-hit != 'true'
       with:
-        toolchain: "${{ inputs.toolchain || steps.rust-version.outputs.version }}"
-        targets: "${{inputs.targets || ''}}"
+        toolchain: "${{ steps.toolchain-config.outputs.toolchain }}"
+        targets: "${{ steps.toolchain-config.outputs.targets }}"
         components: "${{ inputs.components || 'clippy, rustfmt' }}"
 
     - name: Rust Dependency Cache
       uses: Swatinem/rust-cache@v2
       with:
         save-if: ${{ github.ref_name == 'develop' }}
-        shared-key: "rust-cache-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}"
+        shared-key: "rust-cache-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.toolchain-config.outputs.toolchain }}-${{ steps.toolchain-config.outputs.targets }}"
 
     - name: Rust Compile Cache
       uses: mozilla-actions/[email protected]
 
-    - name: Export Path
-      shell: bash
-      run: echo "PATH=$PATH" >> $GITHUB_ENV
-
     - name: Install Protoc (for lance-encoding build step)
       if: runner.os != 'Windows'
       uses: arduino/setup-protoc@v3

.github/workflows/ci.yml

@@ -17,7 +17,7 @@ permissions:
   issues: write  # audit-check creates issues
 
 env:
-  CARGO_TERM_COLOR: always
+  CARGO_TERM_COLOR: auto
   RUST_BACKTRACE: 1
 
 jobs:
@@ -44,10 +44,22 @@ jobs:
 
   python-lint:
     name: "Python (lint)"
-    runs-on: ubuntu-latest
+    runs-on:
+      - runs-on=${{ github.run_id }}
+      - family=m7i+m7i-flex+m7a
+      - cpu=16
+      - image=ubuntu24-full-x64
+      - extras=s3-cache
+      - tag=python-lint
     timeout-minutes: 120
     steps:
+      - uses: runs-on/action@v2
+        with:
+          sccache: s3
       - uses: actions/checkout@v6
+      - uses: ./.github/actions/setup-rust
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Install uv
         uses: spiraldb/actions/.github/actions/[email protected]
         with:
@@ -64,11 +76,20 @@ jobs:
 
   python-test:
     name: "Python (test)"
-    runs-on: ubuntu-latest
+    runs-on:
+      - runs-on=${{ github.run_id }}
+      - family=m7i+m7i-flex+m7a
+      - cpu=16
+      - image=ubuntu24-full-x64
+      - extras=s3-cache
+      - tag=python-test
     timeout-minutes: 120
     env:
       RUST_LOG: "info,uv=debug"
     steps:
+      - uses: runs-on/action@v2
+        with:
+          sccache: s3
       - uses: actions/checkout@v6
       - uses: ./.github/actions/setup-rust
         with:
@@ -81,7 +102,7 @@ jobs:
 
       - name: Pytest - Vortex
         run: |
-          uv run --all-packages pytest --benchmark-disable test/
+          uv run --all-packages pytest --benchmark-disable -n auto test/
         working-directory: vortex-python/
 
       - name: Pytest Benchmarks - Vortex
@@ -256,7 +277,9 @@ jobs:
       - name: Rust Lint - Format
         run: cargo +nightly fmt --all --check
       - name: Rustc check
-        run: cargo check --locked --all-features --all-targets
+        run: RUSTFLAGS="-D warnings" cargo check --locked --all-features --all-targets
+      - name: Rustc check (release)
+        run: RUSTFLAGS="-D warnings" cargo check --locked --all-features --all-targets --release
       - name: Rust Lint - Clippy All Features
         run: cargo clippy --locked --all-features --all-targets -- -D warnings
       - name: Rust Lint - Clippy Default Features
@@ -302,7 +325,7 @@ jobs:
     runs-on:
       - runs-on=${{ github.run_id }}
       - family=m7i+m7i-flex+m7a
-      - cpu=8
+      - cpu=16
       - image=ubuntu24-full-x64
       - extras=s3-cache
       - tag=rust-coverage
@@ -368,7 +391,7 @@ jobs:
     runs-on:
       - runs-on=${{ github.run_id }}
       - family=m7i+m7i-flex+m7a
-      - cpu=8
+      - cpu=16
       - image=ubuntu24-full-x64
       - extras=s3-cache
       - tag=rust-test-sanitizer
@@ -392,6 +415,7 @@ jobs:
         uses: aminya/setup-cpp@v1
         with:
           compiler: llvm
+          cache-tools: true
       - uses: ./.github/actions/setup-rust
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
@@ -473,9 +497,18 @@ jobs:
 
   build-java:
     name: "Java"
-    runs-on: ubuntu-latest
+    runs-on:
+      - runs-on=${{ github.run_id }}
+      - family=m7i+m7i-flex+m7a
+      - cpu=16
+      - image=ubuntu24-full-x64
+      - extras=s3-cache
+      - tag=java
     timeout-minutes: 120
     steps:
+      - uses: runs-on/action@v2
+        with:
+          sccache: s3
       - uses: actions/checkout@v6
       - uses: actions/setup-java@v5
         with:

vortex-array/src/arrays/dict/array.rs

@@ -7,7 +7,6 @@ use vortex_dtype::PType;
 use vortex_dtype::match_each_integer_ptype;
 use vortex_error::VortexExpect;
 use vortex_error::VortexResult;
-use vortex_error::VortexUnwrap;
 use vortex_error::vortex_bail;
 use vortex_error::vortex_ensure;
 use vortex_mask::AllOr;
@@ -81,6 +80,7 @@ impl DictArray {
 
         #[cfg(debug_assertions)]
         {
+            use vortex_error::VortexUnwrap;
             self.validate_all_values_referenced().vortex_unwrap()
         }
 

vortex-array/src/expr/vtable.rs

@@ -15,11 +15,9 @@ use vortex_dtype::DType;
 use vortex_error::VortexExpect;
 use vortex_error::VortexResult;
 use vortex_error::vortex_bail;
-use vortex_error::vortex_ensure;
 use vortex_error::vortex_err;
 use vortex_vector::Vector;
 use vortex_vector::VectorOps;
-use vortex_vector::vector_matches_dtype;
 
 use crate::ArrayRef;
 use crate::expr::ExprId;
@@ -318,11 +316,15 @@ impl<V: VTable> DynExprVTable for VTableAdapter<V> {
 
         // In debug mode, validate that the output dtype matches the expected return dtype.
         #[cfg(debug_assertions)]
-        vortex_ensure!(
-            vector_matches_dtype(&result, &expected_dtype),
-            "Expression execution invalid for dtype {}",
-            expected_dtype
-        );
+        {
+            use vortex_error::vortex_ensure;
+            use vortex_vector::vector_matches_dtype;
+            vortex_ensure!(
+                vector_matches_dtype(&result, &expected_dtype),
+                "Expression execution invalid for dtype {}",
+                expected_dtype
+            );
+        }
 
         Ok(result)
     }

vortex-python/pyproject.toml

@@ -72,4 +72,5 @@ dev = [
     "pcodec>=0.3.3",
     "pyarrow-stubs>=17.16",
     "pytest-benchmark>=5.1.0",
+    "pytest-xdist>=3.5.0",
 ]