Skip to content

Commit 0d1336d

Browse files
gatesngatesn
committed
Layouts 2
Signed-off-by: Nicholas Gates <[email protected]>
2 parents 7f5705a + c2303e5 commit 0d1336d

File tree

274 files changed

+11207
-6838
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

274 files changed

+11207
-6838
lines changed

‎.github/workflows/bench.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ jobs:
124124
"subcommand": "tpch",
125125
"name": "TPC-H SF=1 on S3",
126126
"local_dir": "bench-vortex/data/tpch/1.0",
127-
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/tpch/1.0/",
127+
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/",
128128
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
129129
"scale_factor": "--scale-factor 1.0",
130130
"build_args": "--features lance"
@@ -142,7 +142,7 @@ jobs:
142142
"subcommand": "tpch",
143143
"name": "TPC-H SF=10 on S3",
144144
"local_dir": "bench-vortex/data/tpch/10.0",
145-
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/tpch/10.0/",
145+
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/",
146146
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
147147
"scale_factor": "--scale-factor 10.0",
148148
"build_args": "--features lance"
@@ -174,7 +174,7 @@ jobs:
174174
"subcommand": "fineweb",
175175
"name": "FineWeb S3",
176176
"local_dir": "bench-vortex/data/fineweb",
177-
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/fineweb/",
177+
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/fineweb/",
178178
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
179179
"scale_factor": "--scale-factor 100"
180180
},
@@ -190,7 +190,7 @@ jobs:
190190
"subcommand": "gharchive",
191191
"name": "GitHub Archive (S3)",
192192
"local_dir": "bench-vortex/data/gharchive",
193-
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/gharchive/",
193+
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/gharchive/",
194194
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
195195
"scale_factor": "--scale-factor 100"
196196
},

‎.github/workflows/ci.yml‎

Lines changed: 7 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -140,9 +140,9 @@ jobs:
140140
repo-token: ${{ secrets.GITHUB_TOKEN }}
141141
- name: Docs
142142
run: |
143-
RUSTFLAGS="--cfg gpu_unstable" RUSTDOCFLAGS="-D warnings" cargo doc --no-deps
143+
RUSTDOCFLAGS="-D warnings" cargo doc --no-deps
144144
# nextest doesn't support doc tests, so we run it here
145-
RUSTFLAGS="--cfg gpu_unstable" cargo test --doc --workspace --all-features --exclude vortex-cxx --exclude vortex-jni --exclude vortex-ffi --exclude xtask --no-fail-fast
145+
cargo test --doc --workspace --all-features --exclude vortex-cxx --exclude vortex-jni --exclude vortex-ffi --exclude xtask --no-fail-fast
146146
147147
build-rust:
148148
name: "Rust build (${{matrix.config.name}})"
@@ -156,7 +156,7 @@ jobs:
156156
- tag=${{ matrix.config.name }}
157157
env:
158158
# disable lints for build, they will be caught in Rust lint job.
159-
RUSTFLAGS: "-A warnings --cfg gpu_unstable"
159+
RUSTFLAGS: "-A warnings"
160160
strategy:
161161
fail-fast: false
162162
matrix:
@@ -175,7 +175,7 @@ jobs:
175175
target: wasm32-unknown-unknown
176176
env:
177177
rustflags: "RUSTFLAGS='-A warnings --cfg getrandom_backend=\"wasm_js\"'"
178-
args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd --exclude vortex-gpu"
178+
args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd"
179179
steps:
180180
- uses: runs-on/action@v2
181181
with:
@@ -199,8 +199,6 @@ jobs:
199199
check-min-deps:
200200
name: "Check build with minimal dependencies"
201201
timeout-minutes: 120
202-
env:
203-
RUSTFLAGS: "--cfg gpu_unstable"
204202
runs-on:
205203
- runs-on=${{ github.run_id }}
206204
- family=m7i+m7i-flex+m7a
@@ -223,8 +221,6 @@ jobs:
223221
rust-lint:
224222
name: "Rust (lint)"
225223
timeout-minutes: 120
226-
env:
227-
RUSTFLAGS: "--cfg gpu_unstable"
228224
runs-on:
229225
- runs-on=${{ github.run_id }}
230226
- family=m7i+m7i-flex+m7a
@@ -253,8 +249,6 @@ jobs:
253249
rust-lint-no-default:
254250
name: "Rust (lint, no default)"
255251
timeout-minutes: 120
256-
env:
257-
RUSTFLAGS: "--cfg gpu_unstable"
258252
runs-on:
259253
- runs-on=${{ github.run_id }}
260254
- family=m7i+m7i-flex+m7a
@@ -319,7 +313,7 @@ jobs:
319313
- name: Rust Tests
320314
if: ${{ matrix.suite == 'tests' }}
321315
run: |
322-
cargo +nightly nextest run --locked --workspace --exclude vortex-gpu --all-features --no-fail-fast
316+
cargo +nightly nextest run --locked --workspace --all-features --no-fail-fast
323317
- name: Run TPC-H
324318
if: ${{ matrix.suite == 'tpc-h' }}
325319
# We use i2 to ensure that restarting the duckdb connection succeeds
@@ -396,65 +390,16 @@ jobs:
396390
- name: Rust Tests
397391
run: |
398392
# Build with full debug info first (helps with caching)
399-
cargo +nightly build --locked --workspace --exclude vortex-gpu --all-features --target x86_64-unknown-linux-gnu
393+
cargo +nightly build --locked --workspace --all-features --target x86_64-unknown-linux-gnu
400394
# Run tests with sanitizers and debug output
401395
cargo +nightly nextest run \
402396
--locked \
403397
--workspace \
404-
--exclude vortex-gpu \
405-
--all-features \
406-
--no-fail-fast \
407-
--target x86_64-unknown-linux-gnu \
408-
--verbose
409-
410-
gpu-test:
411-
name: "GPU tests"
412-
timeout-minutes: 120
413-
runs-on:
414-
- runs-on=${{ github.run_id }}
415-
- family=g5
416-
- cpu=8
417-
- image=ubuntu24-gpu-x64
418-
- extras=s3-cache
419-
- tag=cuda-tests
420-
env:
421-
# Keep frame pointers for better stack traces
422-
CARGO_PROFILE_DEV_DEBUG: "true"
423-
CARGO_PROFILE_TEST_DEBUG: "true"
424-
RUSTFLAGS: "--cfg gpu_unstable"
425-
steps:
426-
- uses: runs-on/action@v2
427-
with:
428-
sccache: s3
429-
- name: Display NVIDIA SMI details
430-
run: |
431-
nvidia-smi
432-
nvidia-smi -L
433-
nvidia-smi -q -d Memory
434-
- uses: actions/checkout@v5
435-
- uses: ./.github/actions/setup-rust
436-
with:
437-
repo-token: ${{ secrets.GITHUB_TOKEN }}
438-
toolchain: nightly
439-
components: "rust-src, rustfmt, clippy, llvm-tools-preview"
440-
- name: Install nextest
441-
uses: taiki-e/install-action@v2
442-
with:
443-
tool: nextest
444-
- name: Rust Tests
445-
run: |
446-
# Build with full debug info first (helps with caching)
447-
cargo +nightly build --locked -p vortex-gpu --all-features --target x86_64-unknown-linux-gnu
448-
# Run tests with sanitizers and debug output
449-
cargo +nightly nextest run \
450-
--locked \
451-
-p vortex-gpu \
452398
--all-features \
453399
--no-fail-fast \
454400
--target x86_64-unknown-linux-gnu \
455401
--verbose
456402
457-
458403
build-java:
459404
name: "Java"
460405
runs-on: ubuntu-latest
@@ -518,7 +463,6 @@ jobs:
518463
--exclude vortex-datafusion \
519464
--exclude vortex-duckdb \
520465
--exclude vortex-fuzz \
521-
--exclude vortex-gpu \
522466
--exclude vortex-python \
523467
--exclude vortex-tui \
524468
--exclude xtask \
@@ -530,7 +474,7 @@ jobs:
530474
--profile bench
531475
532476
- name: Run benchmarks
533-
uses: CodSpeedHQ/action@bb005fe1c1eea036d3894f02c049cb6b154a1c27
477+
uses: CodSpeedHQ/action@6a8e2b874c338bf81cc5e8be715ada75908d3871
534478
with:
535479
run: cargo codspeed run
536480
token: ${{ secrets.CODSPEED_TOKEN }}

‎.github/workflows/nightly-bench.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ jobs:
4141
"subcommand": "tpch",
4242
"name": "TPC-H on S3",
4343
"local_dir": "bench-vortex/data/tpch/10.0",
44-
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/tpch/10.0/",
44+
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/",
4545
"targets": "datafusion:parquet,datafusion:vortex,datafusion:lance,duckdb:parquet,duckdb:vortex",
4646
"scale_factor": "--scale-factor 10.0",
4747
"build_args": "--features lance"
@@ -58,7 +58,7 @@ jobs:
5858
"subcommand": "tpch",
5959
"name": "TPC-H on S3",
6060
"local_dir": "bench-vortex/data/tpch/100.0",
61-
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/tpch/100.0/",
61+
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/100.0/",
6262
"targets": "datafusion:parquet,duckdb:parquet,duckdb:vortex",
6363
"scale_factor": "--scale-factor 100.0"
6464
},

‎.github/workflows/sql-benchmarks.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ on:
3535
"subcommand": "tpch",
3636
"name": "TPC-H SF=1 on S3",
3737
"local_dir": "bench-vortex/data/tpch/1.0",
38-
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/tpch/1.0/",
38+
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/",
3939
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
4040
"scale_factor": "--scale-factor 1.0"
4141
},
@@ -51,7 +51,7 @@ on:
5151
"subcommand": "tpch",
5252
"name": "TPC-H SF=10 on S3",
5353
"local_dir": "bench-vortex/data/tpch/10.0",
54-
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/tpch/10.0/",
54+
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/",
5555
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
5656
"scale_factor": "--scale-factor 10.0"
5757
},
@@ -81,7 +81,7 @@ on:
8181
"subcommand": "fineweb",
8282
"name": "FineWeb S3",
8383
"local_dir": "bench-vortex/data/fineweb",
84-
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/fineweb/",
84+
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/fineweb/",
8585
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
8686
"scale_factor": "--scale-factor 100"
8787
},
@@ -97,7 +97,7 @@ on:
9797
"subcommand": "gharchive",
9898
"name": "GitHub Archive (S3)",
9999
"local_dir": "bench-vortex/data/gharchive",
100-
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/gharchive/",
100+
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/gharchive/",
101101
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
102102
"scale_factor": "--scale-factor 100"
103103
},

0 commit comments

Comments
 (0)