Benchmarks re-org (#5676)

This PR makes two big changes to our benchmarking setup:
1. It splits duckdb, datafusion and lance benchmarks into their own
crates, so they don't depend on each other, minimizing the friction when
bumping dependencies like Arrow or DataFusion.
2. Introduces a new benchmarking tool - `vx-bench` which allows
comparing various runs, persisting results and takes care of building
and running the individual binaries. `vx-bench` is written in Python
using `typer`.

I've tried to not touch the CI setup too much, but it can be simplified
and improved significantly to fit this structure.

## Notes for reviewers
1. DF/Arrow results are worse because the data is chunked differently,
we use an `MemTable` which is just a `Vec<RecordBatch>`, which is now
longer.
2. ~I'm looking into the differences in Vortex results, not sure what's
going on there.~ Turns out the `vortex-compact` results are slower
because they actually run on data compressed with the compact
compressor! Some of the tables are about half the size, but they are
slower to decompress/process.
4. ~Seems like we've been dropping some compression benchmark results,
so they dont have a baseline (or I changed the name) I'm trying to
figure out what's going on there.~ I've changed the name of some
compression benchmarks, ratios have the same name though.

## `vx-bench` demo


![render1765993255912](https://github.com/user-attachments/assets/0e0d3c9d-e7f9-47b6-bdff-e41a34e46348)

---------

Signed-off-by: Adam Gutglick <[email protected]>

Author: AdamGS

.github/workflows/bench-pr.yml

@@ -44,9 +44,9 @@ jobs:
     strategy:
       matrix:
         benchmark:
-          - id: random_access
+          - id: random-access-bench
             name: Random Access
-          - id: compress
+          - id: compress-bench
             name: Compression
     if: ${{ contains(github.event.head_commit.message, '[benchmark]') || github.event.label.name == 'benchmark' && github.event_name == 'pull_request' }}
     steps:
@@ -72,7 +72,7 @@ jobs:
         env:
           RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes"
         run: |
-          cargo build --bin ${{ matrix.benchmark.id }} --package bench-vortex --profile release_debug
+          cargo build --package ${{ matrix.benchmark.id }} --profile release_debug
 
       - name: Setup Polar Signals
         if: github.event.pull_request.head.repo.fork == false
@@ -89,7 +89,7 @@ jobs:
         env:
           RUST_BACKTRACE: full
         run: |
-          target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o ${{ matrix.benchmark.id }}.json
+          target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o results.json
 
       - name: Setup AWS CLI
         if: github.event.pull_request.head.repo.fork == false
@@ -123,7 +123,7 @@ jobs:
 
           echo '# Benchmarks: ${{ matrix.benchmark.name }}' > comment.md
           echo '' >> comment.md
-          uv run --no-project scripts/compare-benchmark-jsons.py base.json ${{ matrix.benchmark.id }}.json "${{ matrix.benchmark.name }}" \
+          uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.benchmark.name }}" \
             >> comment.md
 
       - name: Comment PR

.github/workflows/bench.yml

@@ -68,7 +68,7 @@ jobs:
           RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes"
         # The main difference between this and `bench-pr.yml` is that we add the `lance` feature.
         run: |
-          cargo build --bin ${{ matrix.benchmark.id }} --package bench-vortex --profile release_debug --features lance
+          cargo build --bin ${{ matrix.benchmark.id }} --package vortex-bench --profile release_debug --features lance
 
       - name: Setup Polar Signals
         uses: polarsignals/[email protected]
@@ -84,7 +84,7 @@ jobs:
         env:
           RUST_BACKTRACE: full
         run: |
-          target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o ${{ matrix.benchmark.id }}.json --formats parquet,lance,vortex
+          target/release_debug/${{ matrix.benchmark.id }} --formats parquet,lance,vortex -o results.json
 
       - name: Setup AWS CLI
         uses: aws-actions/configure-aws-credentials@v5
@@ -95,7 +95,8 @@ jobs:
       - name: Upload Benchmark Results
         shell: bash
         run: |
-          bash scripts/cat-s3.sh vortex-benchmark-results-database data.json.gz ${{ matrix.benchmark.id }}.json
+          bash scripts/cat-s3.sh vortex-benchmark-results-database data.json.gz results.json
+
   sql:
     uses: ./.github/workflows/sql-benchmarks.yml
     secrets: inherit
@@ -108,73 +109,71 @@ jobs:
             "subcommand": "clickbench",
             "name": "Clickbench on NVME",
             "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb",
-            "build_args": "--features lance"
+            "build_lance": true
           },
           {
             "id": "tpch-nvme",
             "subcommand": "tpch",
             "name": "TPC-H SF=1 on NVME",
             "targets": "datafusion:arrow,datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb",
-            "scale_factor": "--scale-factor 1.0",
-            "build_args": "--features lance"
+            "scale_factor": "1.0",
+            "build_lance": true
           },
           {
             "id": "tpch-s3",
             "subcommand": "tpch",
             "name": "TPC-H SF=1 on S3",
-            "local_dir": "bench-vortex/data/tpch/1.0",
+            "local_dir": "vortex-bench/data/tpch/1.0",
             "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/",
             "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
-            "scale_factor": "--scale-factor 1.0",
-            "build_args": "--features lance"
+            "scale_factor": "1.0",
+            "build_lance": true
           },
           {
             "id": "tpch-nvme-10",
             "subcommand": "tpch",
             "name": "TPC-H SF=10 on NVME",
             "targets": "datafusion:arrow,datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb",
-            "scale_factor": "--scale-factor 10.0",
-            "build_args": "--features lance"
+            "scale_factor": "10.0",
+            "build_lance": true
           },
           {
             "id": "tpch-s3-10",
             "subcommand": "tpch",
             "name": "TPC-H SF=10 on S3",
-            "local_dir": "bench-vortex/data/tpch/10.0",
+            "local_dir": "vortex-bench/data/tpch/10.0",
             "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/",
             "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
-            "scale_factor": "--scale-factor 10.0",
-            "build_args": "--features lance"
+            "scale_factor": "10.0",
+            "build_lance": true
           },
           {
             "id": "tpcds-nvme",
             "subcommand": "tpcds",
             "name": "TPC-DS SF=1 on NVME",
             "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb",
-            "scale_factor": "--scale-factor 1.0"
+            "scale_factor": "1.0"
           },
           {
             "id": "statpopgen",
             "subcommand": "statpopgen",
             "name": "Statistical and Population Genetics",
-            "local_dir": "bench-vortex/data/statpopgen",
+            "local_dir": "vortex-bench/data/statpopgen",
             "targets": "duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
-            "scale_factor": "--scale-factor 100"
+            "scale_factor": "100"
           },
           {
             "id": "fineweb",
             "subcommand": "fineweb",
             "name": "FineWeb NVMe",
-            "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
-            "scale_factor": "--scale-factor 100"
+            "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact"
           },
           {
             "id": "fineweb-s3",
             "subcommand": "fineweb",
             "name": "FineWeb S3",
-            "local_dir": "bench-vortex/data/fineweb",
+            "local_dir": "vortex-bench/data/fineweb",
             "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/fineweb/",
-            "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
-            "scale_factor": "--scale-factor 100"
+            "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact"
           },
         ]

.github/workflows/ci.yml

@@ -399,7 +399,8 @@ jobs:
         if: ${{ matrix.suite == 'tpc-h' }}
         # We use i2 to ensure that restarting the duckdb connection succeeds
         run: |
-          cargo run --bin query_bench -- tpch -i2 --targets "datafusion:vortex,datafusion:vortex-compact,duckdb:vortex,duckdb:vortex-compact" --scale-factor 0.1
+          cargo run --bin datafusion-bench -- tpch -i2 --formats "vortex,vortex-compact" --opt scale-factor=0.1
+          cargo run --bin duckdb-bench -- tpch -i2 --formats "vortex,vortex-compact" --opt scale-factor=0.1
       - name: Run FFI Example
         if: ${{ matrix.suite == 'ffi' }}
         run: |
@@ -411,12 +412,12 @@ jobs:
         run: |
           grcov . --binary-path target/debug/ -s . -t lcov --llvm --ignore-not-existing \
             --threads $(nproc) \
-            --ignore '../*' --ignore '/*' --ignore 'fuzz/*' --ignore 'bench-vortex/*' \
+            --ignore '../*' --ignore '/*' --ignore 'fuzz/*' --ignore 'vortex-bench/*' \
             --ignore 'home/*' --ignore 'xtask/*' --ignore 'target/*' --ignore 'vortex-error/*' \
             --ignore 'vortex-python/*' --ignore 'vortex-jni/*' --ignore 'vortex-flatbuffers/*' \
             --ignore 'vortex-proto/*' --ignore 'vortex-tui/*' --ignore 'vortex-datafusion/examples/*' \
             --ignore 'vortex-ffi/examples/*' --ignore '*/arbitrary/*' --ignore '*/arbitrary.rs' --ignore 'vortex-cxx/*' \
-            --ignore 'vortex-gpu/*' \
+            --ignore 'vortex-gpu/*' --ignore benchmarks/* \
             -o ${{ env.GRCOV_OUTPUT_FILE }}
       - name: Codecov
         uses: codecov/codecov-action@v5
@@ -528,10 +529,11 @@ jobs:
           tool: nextest
       - name: Rust Tests (Windows)
         if: matrix.os == 'windows-x64'
-        run: cargo nextest run --locked --workspace --all-features --no-fail-fast --exclude bench-vortex --exclude vortex-python --exclude vortex-duckdb --exclude vortex-fuzz
+        run: |
+          cargo nextest run --locked --workspace --all-features --no-fail-fast --exclude vortex-bench --exclude vortex-python --exclude vortex-duckdb --exclude vortex-fuzz --exclude duckdb-bench --exclude lance-bench --exclude datafusion-bench --exclude random-access-bench --exclude compress-bench
       - name: Rust Tests (Other)
         if: matrix.os != 'windows-x64'
-        run: cargo nextest run --locked --workspace --all-features --no-fail-fast --exclude bench-vortex --exclude vortex-duckdb
+        run: cargo nextest run --locked --workspace --all-features --no-fail-fast --exclude vortex-bench --exclude vortex-duckdb
 
   build-java:
     name: "Java"

.github/workflows/nightly-bench.yml

@@ -38,34 +38,34 @@ jobs:
             "subcommand": "tpch",
             "name": "TPC-H on NVME",
             "targets": "datafusion:parquet,datafusion:vortex,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:duckdb",
-            "scale_factor": "--scale-factor 10.0",
+            "scale_factor": "10.0",
             "build_args": "--features lance"
           },
           {
             "id": "tpch-s3",
             "subcommand": "tpch",
             "name": "TPC-H on S3",
-            "local_dir": "bench-vortex/data/tpch/10.0",
+            "local_dir": "vortex-bench/data/tpch/10.0",
             "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/",
             "targets": "datafusion:parquet,datafusion:vortex,datafusion:lance,duckdb:parquet,duckdb:vortex",
-            "scale_factor": "--scale-factor 10.0",
+            "scale_factor": "10.0",
             "build_args": "--features lance"
           },
           {
             "id": "tpch-nvme",
             "subcommand": "tpch",
             "name": "TPC-H on NVME",
             "targets": "datafusion:parquet,duckdb:parquet,duckdb:vortex",
-            "scale_factor": "--scale-factor 100"
+            "scale_factor": "100"
           },
           {
             "id": "tpch-s3",
             "subcommand": "tpch",
             "name": "TPC-H on S3",
-            "local_dir": "bench-vortex/data/tpch/100.0",
+            "local_dir": "vortex-bench/data/tpch/100.0",
             "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/100.0/",
             "targets": "datafusion:parquet,duckdb:parquet,duckdb:vortex",
-            "scale_factor": "--scale-factor 100.0"
+            "scale_factor": "100.0"
           },
         ]
     strategy: