Merge branch 'develop' into bp/vortex-tui

Author: palaska

.github/workflows/bench.yml

@@ -178,20 +178,4 @@ jobs:
             "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
             "scale_factor": "--scale-factor 100"
           },
-          {
-            "id": "gharchive-nvme",
-            "subcommand": "gharchive",
-            "name": "GitHub Archive (NVMe)",
-            "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
-            "scale_factor": "--scale-factor 100"
-          },
-          {
-            "id": "gharchive-s3",
-            "subcommand": "gharchive",
-            "name": "GitHub Archive (S3)",
-            "local_dir": "bench-vortex/data/gharchive",
-            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/gharchive/",
-            "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
-            "scale_factor": "--scale-factor 100"
-          },
         ]

.github/workflows/ci.yml

@@ -493,12 +493,7 @@ jobs:
       matrix:
         include:
           - os: windows-x64
-            runner:
-              - runs-on=${{ github.run_id }}
-              - family=m7i
-              - cpu=8
-              - image=windows22-full-x64
-              - tag=rust-test-windows
+            runner: runs-on=${{ github.run_id }}/pool=windows-x64
           - os: linux-arm64
             runner:
               - runs-on=${{ github.run_id }}
@@ -662,13 +657,10 @@ jobs:
       - uses: ./.github/actions/setup-rust
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-          toolchain: nightly-2025-06-26
           targets: "wasm32-wasip1"
-          components: "rust-src"
       - name: Setup Wasmer
         uses: wasmerio/[email protected]
-        # there is a compiler bug in nightly (but not in nightly-2025-06-26)
-      - run: cargo +nightly-2025-06-26 -Zbuild-std=panic_abort,std build --target wasm32-wasip1
+      - run: cargo build --target wasm32-wasip1
         working-directory: ./wasm-test
       - run: wasmer run ./target/wasm32-wasip1/debug/wasm-test.wasm
         working-directory: ./wasm-test

.github/workflows/fuzz.yml

@@ -56,7 +56,7 @@ jobs:
       - name: Run fuzzing target
         id: fuzz
         run: |
-          RUSTFLAGS="--cfg vortex_nightly" RUST_BACKTRACE=1 cargo +nightly fuzz run --release --debug-assertions file_io -- -max_total_time=7200 2>&1 | tee fuzz_output.log
+          RUSTFLAGS="--cfg vortex_nightly" RUST_BACKTRACE=1 cargo +nightly fuzz run --release --debug-assertions file_io -- -max_total_time=7200 -rss_limit_mb=0 2>&1 | tee fuzz_output.log
         continue-on-error: true
       - name: Check for crashes
         id: check
@@ -189,7 +189,7 @@ jobs:
       - name: Run fuzzing target
         id: fuzz
         run: |
-          RUSTFLAGS="--cfg vortex_nightly" RUST_BACKTRACE=1 cargo +nightly fuzz run --release --debug-assertions array_ops -- -max_total_time=7200 2>&1 | tee fuzz_output.log
+          RUSTFLAGS="--cfg vortex_nightly" RUST_BACKTRACE=1 cargo +nightly fuzz run --release --debug-assertions array_ops -- -max_total_time=7200 -rss_limit_mb=0 2>&1 | tee fuzz_output.log
         continue-on-error: true
       - name: Check for crashes
         id: check

.github/workflows/fuzzer-fix-automation.yml

@@ -187,7 +187,7 @@ jobs:
           echo "Attempting to reproduce crash with fuzzer (debug mode)..."
 
           # Run fuzzer with crash file (debug mode, no sanitizer, full backtrace)
-          RUSTFLAGS="--cfg vortex_nightly" RUST_BACKTRACE=full timeout 30s cargo +nightly fuzz run --dev --sanitizer=none "${{ steps.extract.outputs.target }}" "${{ steps.download.outputs.crash_file_path }}" -- -runs=1 2>&1 | tee crash_reproduction.log
+          RUSTFLAGS="--cfg vortex_nightly" RUST_BACKTRACE=full timeout 30s cargo +nightly fuzz run --dev --sanitizer=none "${{ steps.extract.outputs.target }}" "${{ steps.download.outputs.crash_file_path }}" -- -runs=1 -rss_limit_mb=0 2>&1 | tee crash_reproduction.log
 
           FUZZ_EXIT_CODE=${PIPESTATUS[0]}
 
@@ -216,7 +216,7 @@ jobs:
 
           I ran:
           \`\`\`bash
-          cargo +nightly fuzz run --sanitizer=none ${{ steps.extract.outputs.target }} ${{ steps.download.outputs.crash_file_path }} -- -runs=1
+          cargo +nightly fuzz run --sanitizer=none ${{ steps.extract.outputs.target }} ${{ steps.download.outputs.crash_file_path }} -- -runs=1 -rss_limit_mb=0
           \`\`\`
 
           The fuzzer exited with code 0 (success).
@@ -275,7 +275,7 @@ jobs:
             - This ensures your work is visible and reviewable even if you hit the turn limit
             - Keep fixes minimal - only fix the specific bug
             - Follow CLAUDE.md code style guidelines
-            - **Use `--dev` flag** for faster builds: `cargo +nightly fuzz run --dev --sanitizer=none`
+            - **Use `--dev` flag** for faster builds: `cargo +nightly fuzz run --dev --sanitizer=none <target> <crash_file> -- -rss_limit_mb=0`
 
             ## Fixability Guidelines
 

.github/workflows/report-fuzz-crash.yml

@@ -155,7 +155,7 @@ jobs:
             ### Reproduction
 
             ```bash
-            cargo +nightly fuzz run --sanitizer=none $FUZZ_TARGET $FUZZ_TARGET/$CRASH_FILE
+            cargo +nightly fuzz run -D --sanitizer=none $FUZZ_TARGET $FUZZ_TARGET/$CRASH_FILE -- -rss_limit_mb=0
             ```
 
             ---
@@ -219,12 +219,12 @@ jobs:
             2. Reproduce locally:
             ```bash
             # The artifact contains $FUZZ_TARGET/$CRASH_FILE
-            cargo +nightly fuzz run --sanitizer=none $FUZZ_TARGET $FUZZ_TARGET/$CRASH_FILE
+            cargo +nightly fuzz run -D --sanitizer=none $FUZZ_TARGET $FUZZ_TARGET/$CRASH_FILE -- -rss_limit_mb=0
             ```
 
             3. Get full backtrace:
             ```bash
-            RUST_BACKTRACE=full cargo +nightly fuzz run --sanitizer=none $FUZZ_TARGET $FUZZ_TARGET/$CRASH_FILE
+            RUST_BACKTRACE=full cargo +nightly fuzz run -D --sanitizer=none $FUZZ_TARGET $FUZZ_TARGET/$CRASH_FILE -- -rss_limit_mb=0
             ```
 
             ---

.github/workflows/sql-benchmarks.yml

@@ -85,22 +85,6 @@ on:
               "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
               "scale_factor": "--scale-factor 100"
             },
-            {
-              "id": "gharchive-nvme",
-              "subcommand": "gharchive",
-              "name": "GitHub Archive (NVMe)",
-              "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
-              "scale_factor": "--scale-factor 100"
-            },
-            {
-              "id": "gharchive-s3",
-              "subcommand": "gharchive",
-              "name": "GitHub Archive (S3)",
-              "local_dir": "bench-vortex/data/gharchive",
-              "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/gharchive/",
-              "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
-              "scale_factor": "--scale-factor 100"
-            },
           ]
 
 jobs:

Cargo.lock

@@ -109,7 +109,7 @@ fn compress(
         .map(|f| Target::new(Engine::default(), *f))
         .collect_vec();
 
-    let structlistofints = vec![
+    let structlistofints = [
         StructListOfInts::new(100, 1000, 1),
         StructListOfInts::new(1000, 1000, 1),
         StructListOfInts::new(10000, 1000, 1),