Skip to content

Commit 6de0540

Browse files
rschu1zerschu1ze
authored
Merge pull request #563 from vortex-data/ji/update-datafusion-vortex-partitioned-v0.44.0
Update `datafusion-vortex-partitioned` to use vortex 0.44.0
2 parents 92a8309 + 161a78f commit 6de0540

File tree

3 files changed

+51
-51
lines changed

3 files changed

+51
-51
lines changed

‎datafusion-vortex-partitioned/benchmark.sh

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,17 @@ sudo apt-get install -y gcc jq build-essential
1515
# Install Vortex from latest release main branch
1616
git clone https://github.com/spiraldb/vortex.git || true
1717
cd vortex
18-
git checkout 0.34.0
18+
git checkout 0.44.0
1919
git submodule update --init
2020
# We build a release version of the benchmarking utility using mimalloc, just like the datafusion-cli
21-
cargo build --release --bin clickbench --package bench-vortex
21+
cargo build --release --bin query_bench --package bench-vortex
2222
export PATH="`pwd`/target/release:$PATH"
2323
cd ..
2424

2525
# Vortex's benchmarking utility generates appropriate Vortex files by itself, so we just run it to make sure they exist before we start measuring.
2626
# This will download parquet files (with time and string columns already converted to the logically correct datatype) and generate Vortex files from them.
2727
echo -n "Load time: "
28-
command time -f '%e' clickbench -i 1 --targets datafusion:vortex --display-format gh-json -q 0 --hide-progress-bar --flavor partitioned
28+
command time -f '%e' query_bench clickbench -i 1 --targets datafusion:vortex --display-format gh-json -q 0 --hide-progress-bar --flavor partitioned
2929

3030
# Run benchmarks for single parquet and partitioned, our CLI generates the relevant vortex files.
3131
./run.sh partitioned

‎datafusion-vortex-partitioned/results/c6a.4xlarge.json

Lines changed: 47 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,56 @@
11
{
22
"system": "DataFusion (Vortex, partitioned)",
3-
"date": "2025-07-10",
3+
"date": "2025-08-07",
44
"machine": "c6a.4xlarge",
55
"cluster_size": 1,
66
"proprietary": "no",
77
"tuned": "no",
88
"tags": ["Rust","column-oriented","embedded","stateless"],
9-
"load_time": 2025,
10-
"data_size": 16030812776,
9+
"load_time": 208.63,
10+
"data_size": 15812419580,
1111
"result": [
12-
[0.189, 0.035, 0.037],
13-
[0.226, 0.047, 0.044],
14-
[0.29, 0.091, 0.091],
15-
[0.883, 0.141, 0.145],
16-
[1.412, 0.916, 0.902],
17-
[1.536, 0.82, 0.801],
18-
[0.177, 0.037, 0.035],
19-
[0.226, 0.05, 0.049],
20-
[1.79, 0.989, 0.943],
21-
[2.156, 1.065, 1.065],
22-
[1.01, 0.185, 0.179],
23-
[1.08, 0.197, 0.196],
24-
[1.63, 0.76, 0.742],
25-
[3.317, 1.39, 1.425],
26-
[1.727, 0.722, 0.735],
27-
[1.393, 1.06, 1.055],
28-
[3.428, 1.957, 2.005],
29-
[3.266, 1.953, 1.907],
30-
[5.093, 3.731, 3.761],
31-
[0.447, 0.078, 0.082],
32-
[13.758, 0.653, 0.642],
33-
[14.857, 0.765, 0.771],
34-
[17.913, 1.444, 1.471],
35-
[49.105, 4, 4.035],
36-
[2.279, 0.28, 0.279],
37-
[1.309, 0.211, 0.214],
38-
[2.263, 0.318, 0.319],
39-
[13.616, 1.25, 1.256],
40-
[12.664, 9.603, 9.902],
41-
[0.642, 0.427, 0.42],
42-
[2.845, 0.581, 0.571],
43-
[5.975, 0.676, 0.657],
44-
[4.7, 3.333, 3.414],
45-
[13.778, 3.58, 3.572],
46-
[14.066, 3.679, 3.589],
47-
[1.419, 1.227, 1.168],
48-
[0.355, 0.143, 0.136],
49-
[0.275, 0.074, 0.068],
50-
[0.272, 0.071, 0.065],
51-
[0.447, 0.256, 0.258],
52-
[0.278, 0.046, 0.05],
53-
[0.26, 0.048, 0.049],
54-
[0.267, 0.069, 0.062]
55-
]
12+
[0.46253397, 0.037158406, 0.037691171],
13+
[0.610650301, 0.048488383, 0.050934092],
14+
[1.396069485, 0.080961145, 0.084554878],
15+
[3.02162171, 0.133933147, 0.129616406],
16+
[3.041626834, 0.75920063, 0.796099832],
17+
[4.184650763, 0.828085784, 0.78301152],
18+
[0.227687226, 0.047766699, 0.035752126],
19+
[0.348559013, 0.051515242, 0.050747312],
20+
[4.128052101, 0.901855398, 0.932169221],
21+
[4.404706066, 1.00746441, 0.986336246],
22+
[2.673922599, 0.160169697, 0.166423223],
23+
[3.041334404, 0.179402617, 0.182405352],
24+
[4.505253625, 0.675846421, 0.675620887],
25+
[7.270337032, 1.270896868, 1.223746891],
26+
[4.495065472, 0.656599954, 0.654115063],
27+
[2.734807873, 0.979836753, 0.874699098],
28+
[6.795665382, 1.722579261, 1.751881691],
29+
[6.824501787, 1.74540955, 1.738042042],
30+
[9.183125573, 3.337959019, 3.265056724],
31+
[1.704086675, 0.080394864, 0.072437846],
32+
[28.532298301, 0.604708132, 0.612086862],
33+
[31.09728059, 0.730309909, 0.72754013],
34+
[36.869137338, 1.336949381, 1.367562413],
35+
[98.810300479, 4.318801697, 4.390844988],
36+
[5.906679078, 0.236549325, 0.243135004],
37+
[4.143619741, 0.160338307, 0.149354462],
38+
[6.210047753, 0.241249493, 0.243290856],
39+
[28.515009562, 1.149712509, 1.158252015],
40+
[23.22071924, 9.41772195, 9.43845471],
41+
[0.85688345, 0.535619387, 0.54440709],
42+
[6.914319046, 0.57216058, 0.570108651],
43+
[13.230020774, 0.691255195, 0.636682458],
44+
[9.34386667, 3.150708616, 3.200822751],
45+
[28.586140461, 3.48110173, 3.467666154],
46+
[28.622548393, 3.442896776, 3.465528342],
47+
[2.137594646, 1.122478265, 1.15540769],
48+
[0.364387036, 0.149012906, 0.155900777],
49+
[0.550358513, 0.096588634, 0.091583696],
50+
[0.836532657, 0.073034192, 0.082260376],
51+
[1.143149152, 0.274401365, 0.292553024],
52+
[0.499817024, 0.059016133, 0.062895227],
53+
[0.678132818, 0.05422875, 0.055551027],
54+
[0.628570513, 0.072180356, 0.066718048]
55+
]
5656
}

‎datafusion-vortex-partitioned/run.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ for query_num in $(seq 0 42); do
3434
echo -n "["
3535
for i in $(seq 1 $TRIES); do
3636
# Parse query results out of the JSON output, which reports the time in ns
37-
RES=$(RUST_LOG=off clickbench -i 1 --flavor $FLAVOR --targets datafusion:vortex --display-format gh-json --queries-file ./queries.sql -q $query_num --hide-progress-bar | jq ".value / 1000000000")
37+
RES=$(RUST_LOG=off query_bench clickbench -i 1 --flavor $FLAVOR --targets datafusion:vortex --display-format gh-json --queries-file ./queries.sql -q $query_num --hide-progress-bar | jq ".value / 1000000000")
3838

3939
[[ $RES != "" ]] && \
4040
echo -n "$RES" || \

0 commit comments

Comments
 (0)