Skip to content

Commit 3bb6ddc

Browse files
AdamGSrobert3005
authored andcommitted
Upgrade DF to 51
Signed-off-by: Adam Gutglick <[email protected]>
1 parent 4ec0f68 commit 3bb6ddc

File tree

8 files changed

+1786
-956
lines changed

8 files changed

+1786
-956
lines changed

Cargo.lock

Lines changed: 1557 additions & 799 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 45 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -61,16 +61,16 @@ anyhow = "1.0.95"
6161
arbitrary = "1.3.2"
6262
arcref = "0.2.0"
6363
arrayref = "0.3.7"
64-
arrow-arith = "56"
65-
arrow-array = "56"
66-
arrow-buffer = "56"
67-
arrow-cast = "56"
68-
arrow-data = "56"
69-
arrow-ipc = "56"
70-
arrow-ord = "56"
71-
arrow-schema = "56"
72-
arrow-select = "56"
73-
arrow-string = "56"
64+
arrow-arith = "57"
65+
arrow-array = "57"
66+
arrow-buffer = "57"
67+
arrow-cast = "57"
68+
arrow-data = "57"
69+
arrow-ipc = "57"
70+
arrow-ord = "57"
71+
arrow-schema = "57"
72+
arrow-select = "57"
73+
arrow-string = "57"
7474
async-compat = "0.2.5"
7575
async-fs = "2.2.0"
7676
async-stream = "0.3.6"
@@ -83,7 +83,7 @@ bzip2 = "0.6.0"
8383
cbindgen = "0.29.0"
8484
cc = "1.2"
8585
cfg-if = "1"
86-
chrono = "0.4.41"
86+
chrono = "0.4.42"
8787
clap = "4.5"
8888
crossbeam-deque = "0.8.6"
8989
crossbeam-queue = "0.3.12"
@@ -97,19 +97,19 @@ cudarc = { version = "0.17.3", features = [
9797
"nvrtc",
9898
], default-features = false }
9999
dashmap = "6.1.0"
100-
datafusion = { version = "50", default-features = false }
101-
datafusion-catalog = { version = "50" }
102-
datafusion-common = { version = "50" }
103-
datafusion-common-runtime = { version = "50" }
104-
datafusion-datasource = { version = "50", default-features = false }
105-
datafusion-execution = { version = "50" }
106-
datafusion-expr = { version = "50" }
107-
datafusion-functions = { version = "50" }
108-
datafusion-physical-expr = { version = "50" }
109-
datafusion-physical-expr-adapter = { version = "50" }
110-
datafusion-physical-expr-common = { version = "50" }
111-
datafusion-physical-plan = { version = "50" }
112-
datafusion-pruning = { version = "50" }
100+
datafusion = { version = "51", default-features = false, features = ["sql"] }
101+
datafusion-catalog = { version = "51" }
102+
datafusion-common = { version = "51" }
103+
datafusion-common-runtime = { version = "51" }
104+
datafusion-datasource = { version = "51", default-features = false }
105+
datafusion-execution = { version = "51" }
106+
datafusion-expr = { version = "51" }
107+
datafusion-functions = { version = "51" }
108+
datafusion-physical-expr = { version = "51" }
109+
datafusion-physical-expr-adapter = { version = "51" }
110+
datafusion-physical-expr-common = { version = "51" }
111+
datafusion-physical-plan = { version = "51" }
112+
datafusion-pruning = { version = "51" }
113113
dirs = "6.0.0"
114114
divan = { package = "codspeed-divan-compat", version = "4.0.4" }
115115
dyn-hash = "0.2.0"
@@ -123,7 +123,7 @@ futures = { version = "0.3.31", default-features = false }
123123
fuzzy-matcher = "0.3"
124124
glob = "0.3.2"
125125
goldenfile = "1"
126-
half = { version = "2.6", features = ["std", "num-traits"] }
126+
half = { version = "2.7.1", features = ["std", "num-traits"] }
127127
hashbrown = "0.16.0"
128128
humansize = "2.1.3"
129129
indicatif = "0.18.0"
@@ -144,14 +144,14 @@ noodles-bgzf = "0.43.0"
144144
noodles-vcf = "0.81.0"
145145
num-traits = "0.2.19"
146146
num_enum = { version = "0.7.3", default-features = false }
147-
object_store = { version = "0.12.3", default-features = false }
147+
object_store = { version = "0.12.4", default-features = false }
148148
once_cell = "1.21"
149149
oneshot = "0.1.11"
150150
opentelemetry = "0.31.0"
151151
opentelemetry-otlp = "0.31.0"
152152
opentelemetry_sdk = "0.31.0"
153153
parking_lot = { version = "0.12.3", features = ["nightly"] }
154-
parquet = "56"
154+
parquet = "57"
155155
paste = "1.0.15"
156156
pco = "0.4.4"
157157
pin-project-lite = "0.2.15"
@@ -198,11 +198,11 @@ target-lexicon = "0.13"
198198
tempfile = "3"
199199
termtree = { version = "0.5" }
200200
thiserror = "2.0.3"
201-
tokio = { version = "1.47" }
201+
tokio = { version = "1.48" }
202202
tokio-stream = "0.1.17"
203203
tokio-util = { version = "0.7.16" }
204-
tpchgen = { version = "2" }
205-
tpchgen-arrow = { version = "2" }
204+
tpchgen = { version = "2", git = "https://github.com/kevinjqliu/tpchgen-rs.git", rev = "c38f69b0dc84c4b7edfad349d1a9cd1a134b8018" }
205+
tpchgen-arrow = { version = "2", git = "https://github.com/kevinjqliu/tpchgen-rs.git", rev = "c38f69b0dc84c4b7edfad349d1a9cd1a134b8018" }
206206
tracing = { version = "0.1.41" }
207207
tracing-perfetto = "0.1.5"
208208
tracing-subscriber = "0.3.20"
@@ -341,3 +341,18 @@ lto = false
341341
[profile.bench_assert]
342342
debug-assertions = true
343343
inherits = "bench"
344+
345+
[patch.crates-io]
346+
datafusion = { git = "https://github.com/apache/datafusion.git", branch = "branch-51" }
347+
datafusion-common = { git = "https://github.com/apache/datafusion.git", branch = "branch-51" }
348+
datafusion-catalog = { git = "https://github.com/apache/datafusion.git", branch = "branch-51" }
349+
datafusion-common-runtime = { git = "https://github.com/apache/datafusion.git", branch = "branch-51" }
350+
datafusion-datasource = { git = "https://github.com/apache/datafusion.git", branch = "branch-51" }
351+
datafusion-execution = { git = "https://github.com/apache/datafusion.git", branch = "branch-51" }
352+
datafusion-expr = { git = "https://github.com/apache/datafusion.git", branch = "branch-51" }
353+
datafusion-functions = { git = "https://github.com/apache/datafusion.git", branch = "branch-51" }
354+
datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git", branch = "branch-51" }
355+
datafusion-physical-expr-adapter = { git = "https://github.com/apache/datafusion.git", branch = "branch-51" }
356+
datafusion-physical-expr-common = { git = "https://github.com/apache/datafusion.git", branch = "branch-51" }
357+
datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", branch = "branch-51" }
358+
datafusion-pruning = { git = "https://github.com/apache/datafusion.git", branch = "branch-51" }

bench-vortex/src/public_bi.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -414,7 +414,7 @@ impl PBIData {
414414
ListingOptions::new(df_format)
415415
.with_session_config_options(session.state().config()),
416416
)
417-
.with_schema(schema.into());
417+
.with_schema(schema.inner().clone());
418418

419419
let listing_table = Arc::new(ListingTable::try_new(config)?);
420420
session.register_table(table_ref, listing_table)?;

vortex-datafusion/src/convert/scalars.rs

Lines changed: 106 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -40,27 +40,55 @@ impl TryToDataFusion<ScalarValue> for Scalar {
4040
let precision = decimal_type.precision();
4141
let scale = decimal_type.scale();
4242

43-
if precision <= i128::MAX_PRECISION {
43+
if precision <= i32::MAX_PRECISION {
44+
match dscalar.decimal_value() {
45+
None => ScalarValue::Decimal32(None, precision, scale),
46+
Some(value) => match value.cast::<i32>() {
47+
Some(v32) => ScalarValue::Decimal32(Some(v32), precision, scale),
48+
None => {
49+
vortex_bail!(
50+
"invalid ScalarValue {value} for decimal with precision {precision}",
51+
)
52+
}
53+
},
54+
}
55+
} else if precision <= i64::MAX_PRECISION {
56+
match dscalar.decimal_value() {
57+
None => ScalarValue::Decimal64(None, precision, scale),
58+
Some(value) => match value.cast::<i64>() {
59+
Some(v64) => ScalarValue::Decimal64(Some(v64), precision, scale),
60+
None => {
61+
vortex_bail!(
62+
"invalid ScalarValue {value} for decimal with precision {precision}",
63+
)
64+
}
65+
},
66+
}
67+
} else if precision <= i128::MAX_PRECISION {
4468
match dscalar.decimal_value() {
4569
None => ScalarValue::Decimal128(None, precision, scale),
46-
Some(DecimalValue::I128(v128)) => {
47-
ScalarValue::Decimal128(Some(v128), precision, scale)
48-
}
49-
_ => vortex_bail!(
50-
"invalid ScalarValue for decimal with precision {}",
51-
precision
52-
),
70+
Some(value) => match value.cast::<i128>() {
71+
Some(v128) => ScalarValue::Decimal128(Some(v128), precision, scale),
72+
None => {
73+
vortex_bail!(
74+
"invalid ScalarValue {value} for decimal with precision {precision}",
75+
)
76+
}
77+
},
5378
}
5479
} else {
5580
match dscalar.decimal_value() {
5681
None => ScalarValue::Decimal256(None, precision, scale),
57-
Some(DecimalValue::I256(v256)) => {
58-
ScalarValue::Decimal256(Some(v256.into()), precision, scale)
59-
}
60-
_ => vortex_bail!(
61-
"invalid ScalarValue for decimal with precision {}",
62-
precision
63-
),
82+
Some(value) => match value.cast::<i256>() {
83+
Some(v256) => {
84+
ScalarValue::Decimal256(Some(v256.into()), precision, scale)
85+
}
86+
None => {
87+
vortex_bail!(
88+
"invalid ScalarValue {value} for decimal with precision {precision}",
89+
)
90+
}
91+
},
6492
}
6593
}
6694
}
@@ -210,6 +238,32 @@ impl FromDataFusion<ScalarValue> for Scalar {
210238
.unwrap_or_else(vortex::scalar::ScalarValue::null),
211239
)
212240
}
241+
ScalarValue::Decimal32(decimal, precision, scale) => {
242+
let decimal_dtype = DecimalDType::new(*precision, *scale);
243+
let nullable = Nullability::Nullable;
244+
if let Some(value) = decimal {
245+
Scalar::decimal(
246+
DecimalValue::I32(*value),
247+
decimal_dtype,
248+
Nullability::Nullable,
249+
)
250+
} else {
251+
Scalar::null(DType::Decimal(decimal_dtype, nullable))
252+
}
253+
}
254+
ScalarValue::Decimal64(decimal, precision, scale) => {
255+
let decimal_dtype = DecimalDType::new(*precision, *scale);
256+
let nullable = Nullability::Nullable;
257+
if let Some(value) = decimal {
258+
Scalar::decimal(
259+
DecimalValue::I64(*value),
260+
decimal_dtype,
261+
Nullability::Nullable,
262+
)
263+
} else {
264+
Scalar::null(DType::Decimal(decimal_dtype, nullable))
265+
}
266+
}
213267
ScalarValue::Decimal128(decimal, precision, scale) => {
214268
let decimal_dtype = DecimalDType::new(*precision, *scale);
215269
let nullable = Nullability::Nullable;
@@ -351,17 +405,41 @@ mod tests {
351405
}
352406

353407
#[rstest]
408+
#[case::decimal32_some(
409+
Scalar::decimal(
410+
DecimalValue::I32(1234),
411+
DecimalDType::new(5, 2),
412+
Nullability::NonNullable
413+
),
414+
ScalarValue::Decimal32(Some(1234), 5, 2)
415+
)]
416+
#[case::decimal32_null(
417+
Scalar::null(DType::Decimal(DecimalDType::new(5, 2), Nullability::Nullable)),
418+
ScalarValue::Decimal32(None, 5, 2)
419+
)]
420+
#[case::decimal64_some(
421+
Scalar::decimal(
422+
DecimalValue::I64(12345),
423+
DecimalDType::new(10, 2),
424+
Nullability::NonNullable
425+
),
426+
ScalarValue::Decimal64(Some(12345), 10, 2)
427+
)]
428+
#[case::decimal64_null(
429+
Scalar::null(DType::Decimal(DecimalDType::new(10, 2), Nullability::Nullable)),
430+
ScalarValue::Decimal64(None, 10, 2)
431+
)]
354432
#[case::decimal128_some(
355433
Scalar::decimal(
356434
DecimalValue::I128(12345),
357-
DecimalDType::new(10, 2),
435+
DecimalDType::new(20, 2),
358436
Nullability::NonNullable
359437
),
360-
ScalarValue::Decimal128(Some(12345), 10, 2)
438+
ScalarValue::Decimal128(Some(12345), 20, 2)
361439
)]
362440
#[case::decimal128_null(
363-
Scalar::null(DType::Decimal(DecimalDType::new(10, 2), Nullability::Nullable)),
364-
ScalarValue::Decimal128(None, 10, 2)
441+
Scalar::null(DType::Decimal(DecimalDType::new(20, 2), Nullability::Nullable)),
442+
ScalarValue::Decimal128(None, 20, 2)
365443
)]
366444
#[case::decimal256_some(
367445
Scalar::decimal(
@@ -560,8 +638,16 @@ mod tests {
560638
ScalarValue::Binary(None)
561639
)]
562640
#[case::null_decimal128(
641+
Scalar::null(DType::Decimal(DecimalDType::new(20, 2), Nullability::Nullable)),
642+
ScalarValue::Decimal128(None, 20, 2)
643+
)]
644+
#[case::null_decimal64(
563645
Scalar::null(DType::Decimal(DecimalDType::new(10, 2), Nullability::Nullable)),
564-
ScalarValue::Decimal128(None, 10, 2)
646+
ScalarValue::Decimal64(None, 10, 2)
647+
)]
648+
#[case::null_decimal32(
649+
Scalar::null(DType::Decimal(DecimalDType::new(5, 2), Nullability::Nullable)),
650+
ScalarValue::Decimal32(None, 5, 2)
565651
)]
566652
fn test_null_handling(#[case] vortex_null: Scalar, #[case] expected_df_null: ScalarValue) {
567653
// Test Vortex -> DataFusion

vortex-datafusion/src/persistent/mod.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -243,8 +243,9 @@ mod tests {
243243
┌───────────────────────────┐
244244
│ SortPreservingMergeExec │
245245
│ -------------------- │
246-
│ c1 ASC NULLS LASTlimit: │
247-
│ 3 │
246+
│ c1 ASC NULLS LAST │
247+
│ │
248+
│ limit: 3 │
248249
└─────────────┬─────────────┘
249250
┌─────────────┴─────────────┐
250251
│ DataSourceExec │

0 commit comments

Comments
 (0)