Convert compress benchmark to a binary (#2399)

Author: robert3005

.github/workflows/bench-pr.yml

@@ -22,81 +22,6 @@ jobs:
         with:
           labels: benchmark
 
-  bench-criterion:
-    needs: label_trigger
-    strategy:
-      matrix:
-        benchmark:
-          - id: compress
-            name: Vortex Compression
-    runs-on: [ self-hosted, gcp ]
-    if: ${{ contains(github.event.head_commit.message, '[benchmark]') || github.event.label.name == 'benchmark' && github.event_name == 'pull_request' }}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/cleanup
-      - uses: ./.github/actions/setup-rust
-
-      # The compression benchmarks rely on DuckDB being installed to convert CSV to Parquet
-      - name: Install DuckDB
-        uses: opt-nc/[email protected]
-        if: runner.environment != 'self-hosted'
-        with:
-          version: v1.0.0
-
-      - name: Set tempdir
-        if: runner.environment == 'self-hosted'
-        run: |
-          echo "TMPDIR=/work" >> $GITHUB_ENV
-
-      - name: Run benchmark
-        shell: bash
-        env:
-          BENCH_VORTEX_RATIOS: '.*'
-          RUSTFLAGS: '-C target-cpu=native'
-        run: |
-          cargo install cargo-criterion
-          sudo apt-get update && sudo apt-get install -y jq
-
-          cargo criterion \
-                --bench ${{ matrix.benchmark.id }} \
-                --features mimalloc \
-                --message-format=json \
-            > ${{ matrix.benchmark.id }}-raw.json
-
-          cat ${{ matrix.benchmark.id }}-raw.json \
-            | bash scripts/coerce-criterion-json.sh \
-            > ${{ matrix.benchmark.id }}.json
-
-      - name: Setup AWS CLI
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole
-          aws-region: us-east-1
-      - name: Install uv
-        uses: astral-sh/setup-uv@v5
-      - name: Compare results
-        shell: bash
-        run: |
-          set -Eeu -o pipefail -x
-
-          base_commit_sha=${{ github.event.pull_request.base.sha }}
-
-          aws s3 cp s3://vortex-benchmark-results-database/data.json - \
-            | grep $base_commit_sha \
-            > base.json
-
-          echo '# Benchmarks: ${{ matrix.benchmark.id }}' > comment.md
-          echo '<details>' >> comment.md
-          echo '<summary>Table of Results</summary>' >> comment.md
-          echo '' >> comment.md
-          uv run scripts/compare-benchmark-jsons.py base.json ${{ matrix.benchmark.id }}.json \
-            >> comment.md
-          echo '</details>' >> comment.md
-      - name: Comment PR
-        uses: thollander/actions-comment-pull-request@v3
-        with:
-          file-path: comment.md
-          comment-tag: bench-pr-comment-${{ matrix.benchmark.id }}
   bench:
     needs: label_trigger
     runs-on: [ self-hosted, gcp ]
@@ -105,6 +30,8 @@ jobs:
         benchmark:
           - id: random_access
             name: Random Access
+          - id: compress
+            name: Compression
     if: ${{ contains(github.event.head_commit.message, '[benchmark]') || github.event.label.name == 'benchmark' && github.event_name == 'pull_request' }}
     steps:
       - uses: actions/checkout@v4
@@ -125,10 +52,9 @@ jobs:
       - name: Run ${{ matrix.benchmark.name }} benchmark
         shell: bash
         env:
-          BENCH_VORTEX_RATIOS: '.*'
           RUSTFLAGS: '-C target-cpu=native'
         run: |
-          cargo run --bin ${{ matrix.benchmark.id }} --release -- -d gh-json -t 1 | tee ${{ matrix.benchmark.id }}.json
+          cargo run --bin ${{ matrix.benchmark.id }} --release -- -d gh-json | tee ${{ matrix.benchmark.id }}.json
 
       - name: Setup AWS CLI
         uses: aws-actions/configure-aws-credentials@v4

.github/workflows/bench.yml

@@ -29,65 +29,15 @@ jobs:
           bash scripts/commit-json.sh > new-commit.json
           bash scripts/cat-s3.sh vortex-benchmark-results-database commits.json new-commit.json
 
-  bench-criterion:
-    strategy:
-      matrix:
-        benchmark:
-          - id: compress
-            name: Vortex Compression
-    runs-on: [ self-hosted, gcp ]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/cleanup
-      - uses: ./.github/actions/setup-rust
-
-      # The compression benchmarks rely on DuckDB being installed to convert CSV to Parquet
-      - name: Install DuckDB
-        uses: opt-nc/[email protected]
-        if: runner.environment != 'self-hosted'
-        with:
-          version: v1.0.0
-
-      - name: Set tempdir
-        if: runner.environment == 'self-hosted'
-        run: |
-          echo "TMPDIR=/work" >> $GITHUB_ENV
-
-      - name: Run benchmark
-        shell: bash
-        env:
-          BENCH_VORTEX_RATIOS: '.*'
-          RUSTFLAGS: '-C target-cpu=native'
-        run: |
-          cargo install cargo-criterion
-          sudo apt-get update && sudo apt-get install -y jq
-
-          cargo criterion \
-                --bench ${{ matrix.benchmark.id }} \
-                --features mimalloc \
-                --message-format=json \
-            > ${{ matrix.benchmark.id }}-raw.json
-
-          cat ${{ matrix.benchmark.id }}-raw.json \
-            | bash scripts/coerce-criterion-json.sh \
-            > ${{ matrix.benchmark.id }}.json
-
-      - name: Setup AWS CLI
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole
-          aws-region: us-east-1
-      - name: Upload Benchmark Results
-        shell: bash
-        run: |
-          bash scripts/cat-s3.sh vortex-benchmark-results-database data.json ${{ matrix.benchmark.id }}.json
   bench:
     runs-on: [ self-hosted, gcp ]
     strategy:
       matrix:
         benchmark:
           - id: random_access
             name: Random Access
+          - id: compress
+            name: Compression
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/cleanup
@@ -107,7 +57,6 @@ jobs:
       - name: Run ${{ matrix.benchmark.name }} benchmark
         shell: bash
         env:
-          BENCH_VORTEX_RATIOS: '.*'
           RUSTFLAGS: '-C target-cpu=native'
         run: |
           cargo run --bin ${{ matrix.benchmark.id }} --release -- -d gh-json | tee ${{ matrix.benchmark.id }}.json

.github/workflows/sql-benchmarks.yml

@@ -44,7 +44,6 @@ jobs:
         if: matrix.remote_storage == null
         shell: bash
         env:
-          BENCH_VORTEX_RATIOS: '.*'
           RUSTFLAGS: '-C target-cpu=native'
         run: |
           cargo run \
@@ -58,7 +57,6 @@ jobs:
         if: matrix.remote_storage != null
         shell: bash
         env:
-          BENCH_VORTEX_RATIOS: '.*'
           RUSTFLAGS: '-C target-cpu=native'
         run: |
           cargo run \