Skip to content

Commit 2f80cb1

Browse files
AdamGSAdamGS
committed
New benchmarks
Signed-off-by: Adam Gutglick <[email protected]>
1 parent afe1196 commit 2f80cb1

File tree

263 files changed

+6504
-6611
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

263 files changed

+6504
-6611
lines changed

‎.github/workflows/bench-pr.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ jobs:
7373
env:
7474
RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes"
7575
run: |
76-
cargo build --bin ${{ matrix.benchmark.id }} --package bench-vortex --profile release_debug
76+
cargo build --bin ${{ matrix.benchmark.id }} --package vortex-bench --profile release_debug
7777
7878
- name: Setup Polar Signals
7979
if: github.event.pull_request.head.repo.fork == false
@@ -90,7 +90,7 @@ jobs:
9090
env:
9191
RUST_BACKTRACE: full
9292
run: |
93-
target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o ${{ matrix.benchmark.id }}.json
93+
target/release_debug/${{ matrix.benchmark.id }} -d gh-json
9494
9595
- name: Setup AWS CLI
9696
if: github.event.pull_request.head.repo.fork == false
@@ -124,7 +124,7 @@ jobs:
124124
125125
echo '# Benchmarks: ${{ matrix.benchmark.name }}' > comment.md
126126
echo '' >> comment.md
127-
uv run --no-project scripts/compare-benchmark-jsons.py base.json ${{ matrix.benchmark.id }}.json "${{ matrix.benchmark.name }}" \
127+
uv run --no-project scripts/compare-benchmark-jsons.py base.json target/vortex-bench/${{ matrix.benchmark.id }}/results.json "${{ matrix.benchmark.name }}" \
128128
>> comment.md
129129
130130
- name: Comment PR

‎.github/workflows/bench.yml‎

Lines changed: 21 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ jobs:
6969
RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes"
7070
# The main difference between this and `bench-pr.yml` is that we add the `lance` feature.
7171
run: |
72-
cargo build --bin ${{ matrix.benchmark.id }} --package bench-vortex --profile release_debug --features lance
72+
cargo build --bin ${{ matrix.benchmark.id }} --package vortex-bench --profile release_debug --features lance
7373
7474
- name: Setup Polar Signals
7575
uses: polarsignals/[email protected]
@@ -85,7 +85,7 @@ jobs:
8585
env:
8686
RUST_BACKTRACE: full
8787
run: |
88-
target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o ${{ matrix.benchmark.id }}.json --formats parquet,lance,vortex
88+
target/release_debug/${{ matrix.benchmark.id }} -d gh-json --formats parquet,lance,vortex
8989
9090
- name: Setup AWS CLI
9191
uses: aws-actions/configure-aws-credentials@v5
@@ -96,7 +96,8 @@ jobs:
9696
- name: Upload Benchmark Results
9797
shell: bash
9898
run: |
99-
bash scripts/cat-s3.sh vortex-benchmark-results-database data.json.gz ${{ matrix.benchmark.id }}.json
99+
bash scripts/cat-s3.sh vortex-benchmark-results-database data.json.gz target/vortex-bench/${{ matrix.benchmark.id }}/results.json
100+
100101
sql:
101102
uses: ./.github/workflows/sql-benchmarks.yml
102103
secrets: inherit
@@ -109,73 +110,71 @@ jobs:
109110
"subcommand": "clickbench",
110111
"name": "Clickbench on NVME",
111112
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb",
112-
"build_args": "--features lance"
113+
"build_lance": true
113114
},
114115
{
115116
"id": "tpch-nvme",
116117
"subcommand": "tpch",
117118
"name": "TPC-H SF=1 on NVME",
118119
"targets": "datafusion:arrow,datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb",
119-
"scale_factor": "--scale-factor 1.0",
120-
"build_args": "--features lance"
120+
"scale_factor": "1.0",
121+
"build_lance": true
121122
},
122123
{
123124
"id": "tpch-s3",
124125
"subcommand": "tpch",
125126
"name": "TPC-H SF=1 on S3",
126-
"local_dir": "bench-vortex/data/tpch/1.0",
127+
"local_dir": "vortex-bench/data/tpch/1.0",
127128
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/",
128129
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
129-
"scale_factor": "--scale-factor 1.0",
130-
"build_args": "--features lance"
130+
"scale_factor": "1.0",
131+
"build_lance": true
131132
},
132133
{
133134
"id": "tpch-nvme-10",
134135
"subcommand": "tpch",
135136
"name": "TPC-H SF=10 on NVME",
136137
"targets": "datafusion:arrow,datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb",
137-
"scale_factor": "--scale-factor 10.0",
138-
"build_args": "--features lance"
138+
"scale_factor": "10.0",
139+
"build_lance": true
139140
},
140141
{
141142
"id": "tpch-s3-10",
142143
"subcommand": "tpch",
143144
"name": "TPC-H SF=10 on S3",
144-
"local_dir": "bench-vortex/data/tpch/10.0",
145+
"local_dir": "vortex-bench/data/tpch/10.0",
145146
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/",
146147
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
147-
"scale_factor": "--scale-factor 10.0",
148-
"build_args": "--features lance"
148+
"scale_factor": "10.0",
149+
"build_lance": true
149150
},
150151
{
151152
"id": "tpcds-nvme",
152153
"subcommand": "tpcds",
153154
"name": "TPC-DS SF=1 on NVME",
154155
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb",
155-
"scale_factor": "--scale-factor 1.0"
156+
"scale_factor": "1.0"
156157
},
157158
{
158159
"id": "statpopgen",
159160
"subcommand": "statpopgen",
160161
"name": "Statistical and Population Genetics",
161-
"local_dir": "bench-vortex/data/statpopgen",
162+
"local_dir": "vortex-bench/data/statpopgen",
162163
"targets": "duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
163-
"scale_factor": "--scale-factor 100"
164+
"scale_factor": "100"
164165
},
165166
{
166167
"id": "fineweb",
167168
"subcommand": "fineweb",
168169
"name": "FineWeb NVMe",
169-
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
170-
"scale_factor": "--scale-factor 100"
170+
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact"
171171
},
172172
{
173173
"id": "fineweb-s3",
174174
"subcommand": "fineweb",
175175
"name": "FineWeb S3",
176-
"local_dir": "bench-vortex/data/fineweb",
176+
"local_dir": "vortex-bench/data/fineweb",
177177
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/fineweb/",
178-
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
179-
"scale_factor": "--scale-factor 100"
178+
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact"
180179
},
181180
]

‎.github/workflows/ci.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -411,12 +411,12 @@ jobs:
411411
run: |
412412
grcov . --binary-path target/debug/ -s . -t lcov --llvm --ignore-not-existing \
413413
--threads $(nproc) \
414-
--ignore '../*' --ignore '/*' --ignore 'fuzz/*' --ignore 'bench-vortex/*' \
414+
--ignore '../*' --ignore '/*' --ignore 'fuzz/*' --ignore 'vortex-bench/*' \
415415
--ignore 'home/*' --ignore 'xtask/*' --ignore 'target/*' --ignore 'vortex-error/*' \
416416
--ignore 'vortex-python/*' --ignore 'vortex-jni/*' --ignore 'vortex-flatbuffers/*' \
417417
--ignore 'vortex-proto/*' --ignore 'vortex-tui/*' --ignore 'vortex-datafusion/examples/*' \
418418
--ignore 'vortex-ffi/examples/*' --ignore '*/arbitrary/*' --ignore '*/arbitrary.rs' --ignore 'vortex-cxx/*' \
419-
--ignore 'vortex-gpu/*' \
419+
--ignore 'vortex-gpu/*' --ignore benchmarks/* \
420420
-o ${{ env.GRCOV_OUTPUT_FILE }}
421421
- name: Codecov
422422
uses: codecov/codecov-action@v5
@@ -533,10 +533,10 @@ jobs:
533533
tool: nextest
534534
- name: Rust Tests (Windows)
535535
if: matrix.os == 'windows-x64'
536-
run: cargo nextest run --locked --workspace --all-features --no-fail-fast --exclude bench-vortex --exclude vortex-python --exclude vortex-duckdb --exclude vortex-fuzz
536+
run: cargo nextest run --locked --workspace --all-features --no-fail-fast --exclude vortex-bench --exclude vortex-python --exclude vortex-duckdb --exclude vortex-fuzz
537537
- name: Rust Tests (Other)
538538
if: matrix.os != 'windows-x64'
539-
run: cargo nextest run --locked --workspace --all-features --no-fail-fast --exclude bench-vortex --exclude vortex-duckdb
539+
run: cargo nextest run --locked --workspace --all-features --no-fail-fast --exclude vortex-bench --exclude vortex-duckdb
540540

541541
build-java:
542542
name: "Java"

‎.github/workflows/nightly-bench.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ jobs:
4646
"id": "tpch-s3",
4747
"subcommand": "tpch",
4848
"name": "TPC-H on S3",
49-
"local_dir": "bench-vortex/data/tpch/10.0",
49+
"local_dir": "vortex-bench/data/tpch/10.0",
5050
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/",
5151
"targets": "datafusion:parquet,datafusion:vortex,datafusion:lance,duckdb:parquet,duckdb:vortex",
5252
"scale_factor": "--scale-factor 10.0",
@@ -63,7 +63,7 @@ jobs:
6363
"id": "tpch-s3",
6464
"subcommand": "tpch",
6565
"name": "TPC-H on S3",
66-
"local_dir": "bench-vortex/data/tpch/100.0",
66+
"local_dir": "vortex-bench/data/tpch/100.0",
6767
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/100.0/",
6868
"targets": "datafusion:parquet,duckdb:parquet,duckdb:vortex",
6969
"scale_factor": "--scale-factor 100.0"

0 commit comments

Comments
 (0)