Skip to content

Commit 75bebe9

Browse files
committed
Fix conflicts
Signed-off-by: Adam Gutglick <[email protected]>
1 parent ec5c119 commit 75bebe9

File tree

8 files changed

+1780
-875
lines changed

8 files changed

+1780
-875
lines changed

Cargo.lock

Lines changed: 1552 additions & 702 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -61,16 +61,16 @@ anyhow = "1.0.95"
6161
arbitrary = "1.3.2"
6262
arcref = "0.2.0"
6363
arrayref = "0.3.7"
64-
arrow-arith = "56"
65-
arrow-array = "56"
66-
arrow-buffer = "56"
67-
arrow-cast = "56"
68-
arrow-data = "56"
69-
arrow-ipc = "56"
70-
arrow-ord = "56"
71-
arrow-schema = "56"
72-
arrow-select = "56"
73-
arrow-string = "56"
64+
arrow-arith = "57"
65+
arrow-array = "57"
66+
arrow-buffer = "57"
67+
arrow-cast = "57"
68+
arrow-data = "57"
69+
arrow-ipc = "57"
70+
arrow-ord = "57"
71+
arrow-schema = "57"
72+
arrow-select = "57"
73+
arrow-string = "57"
7474
async-compat = "0.2.5"
7575
async-fs = "2.2.0"
7676
async-stream = "0.3.6"
@@ -83,7 +83,7 @@ bzip2 = "0.6.0"
8383
cbindgen = "0.29.0"
8484
cc = "1.2"
8585
cfg-if = "1"
86-
chrono = "0.4.41"
86+
chrono = "0.4.42"
8787
clap = "4.5"
8888
crossbeam-deque = "0.8.6"
8989
crossbeam-queue = "0.3.12"
@@ -97,19 +97,19 @@ cudarc = { version = "0.18.0", features = [
9797
"nvrtc",
9898
], default-features = false }
9999
dashmap = "6.1.0"
100-
datafusion = { version = "50", default-features = false }
101-
datafusion-catalog = { version = "50" }
102-
datafusion-common = { version = "50" }
103-
datafusion-common-runtime = { version = "50" }
104-
datafusion-datasource = { version = "50", default-features = false }
105-
datafusion-execution = { version = "50" }
106-
datafusion-expr = { version = "50" }
107-
datafusion-functions = { version = "50" }
108-
datafusion-physical-expr = { version = "50" }
109-
datafusion-physical-expr-adapter = { version = "50" }
110-
datafusion-physical-expr-common = { version = "50" }
111-
datafusion-physical-plan = { version = "50" }
112-
datafusion-pruning = { version = "50" }
100+
datafusion = { version = "51", default-features = false, features = ["sql"] }
101+
datafusion-catalog = { version = "51" }
102+
datafusion-common = { version = "51" }
103+
datafusion-common-runtime = { version = "51" }
104+
datafusion-datasource = { version = "51", default-features = false }
105+
datafusion-execution = { version = "51" }
106+
datafusion-expr = { version = "51" }
107+
datafusion-functions = { version = "51" }
108+
datafusion-physical-expr = { version = "51" }
109+
datafusion-physical-expr-adapter = { version = "51" }
110+
datafusion-physical-expr-common = { version = "51" }
111+
datafusion-physical-plan = { version = "51" }
112+
datafusion-pruning = { version = "51" }
113113
dirs = "6.0.0"
114114
divan = { package = "codspeed-divan-compat", version = "4.0.4" }
115115
dyn-hash = "1.0.0"
@@ -123,7 +123,7 @@ futures = { version = "0.3.31", default-features = false }
123123
fuzzy-matcher = "0.3"
124124
glob = "0.3.2"
125125
goldenfile = "1"
126-
half = { version = "2.6", features = ["std", "num-traits"] }
126+
half = { version = "2.7.1", features = ["std", "num-traits"] }
127127
hashbrown = "0.16.0"
128128
humansize = "2.1.3"
129129
indicatif = "0.18.0"
@@ -144,14 +144,14 @@ noodles-bgzf = "0.44.0"
144144
noodles-vcf = "0.82.0"
145145
num-traits = "0.2.19"
146146
num_enum = { version = "0.7.3", default-features = false }
147-
object_store = { version = "0.12.3", default-features = false }
147+
object_store = { version = "0.12.4", default-features = false }
148148
once_cell = "1.21"
149149
oneshot = "0.1.11"
150150
opentelemetry = "0.31.0"
151151
opentelemetry-otlp = "0.31.0"
152152
opentelemetry_sdk = "0.31.0"
153153
parking_lot = { version = "0.12.3", features = ["nightly"] }
154-
parquet = "56"
154+
parquet = "57"
155155
paste = "1.0.15"
156156
pco = "0.4.4"
157157
pin-project-lite = "0.2.15"
@@ -198,11 +198,11 @@ target-lexicon = "0.13"
198198
tempfile = "3"
199199
termtree = { version = "0.5" }
200200
thiserror = "2.0.3"
201-
tokio = { version = "1.47" }
201+
tokio = { version = "1.48" }
202202
tokio-stream = "0.1.17"
203203
tokio-util = { version = "0.7.16" }
204-
tpchgen = { version = "2" }
205-
tpchgen-arrow = { version = "2" }
204+
tpchgen = { version = "2.0.1" }
205+
tpchgen-arrow = { version = "2.0.1" }
206206
tracing = { version = "0.1.41" }
207207
tracing-perfetto = "0.1.5"
208208
tracing-subscriber = "0.3.20"

bench-vortex/src/public_bi.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -432,7 +432,7 @@ impl PBIData {
432432
ListingOptions::new(df_format)
433433
.with_session_config_options(session.state().config()),
434434
)
435-
.with_schema(schema.into());
435+
.with_schema(schema.inner().clone());
436436

437437
let listing_table = Arc::new(ListingTable::try_new(config)?);
438438
session.register_table(table_ref, listing_table)?;

vortex-datafusion/src/convert/scalars.rs

Lines changed: 106 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -50,27 +50,55 @@ impl TryToDataFusion<ScalarValue> for Scalar {
5050
let precision = decimal_type.precision();
5151
let scale = decimal_type.scale();
5252

53-
if precision <= i128::MAX_PRECISION {
53+
if precision <= i32::MAX_PRECISION {
54+
match dscalar.decimal_value() {
55+
None => ScalarValue::Decimal32(None, precision, scale),
56+
Some(value) => match value.cast::<i32>() {
57+
Some(v32) => ScalarValue::Decimal32(Some(v32), precision, scale),
58+
None => {
59+
vortex_bail!(
60+
"invalid ScalarValue {value} for decimal with precision {precision}",
61+
)
62+
}
63+
},
64+
}
65+
} else if precision <= i64::MAX_PRECISION {
66+
match dscalar.decimal_value() {
67+
None => ScalarValue::Decimal64(None, precision, scale),
68+
Some(value) => match value.cast::<i64>() {
69+
Some(v64) => ScalarValue::Decimal64(Some(v64), precision, scale),
70+
None => {
71+
vortex_bail!(
72+
"invalid ScalarValue {value} for decimal with precision {precision}",
73+
)
74+
}
75+
},
76+
}
77+
} else if precision <= i128::MAX_PRECISION {
5478
match dscalar.decimal_value() {
5579
None => ScalarValue::Decimal128(None, precision, scale),
56-
Some(DecimalValue::I128(v128)) => {
57-
ScalarValue::Decimal128(Some(v128), precision, scale)
58-
}
59-
_ => vortex_bail!(
60-
"invalid ScalarValue for decimal with precision {}",
61-
precision
62-
),
80+
Some(value) => match value.cast::<i128>() {
81+
Some(v128) => ScalarValue::Decimal128(Some(v128), precision, scale),
82+
None => {
83+
vortex_bail!(
84+
"invalid ScalarValue {value} for decimal with precision {precision}",
85+
)
86+
}
87+
},
6388
}
6489
} else {
6590
match dscalar.decimal_value() {
6691
None => ScalarValue::Decimal256(None, precision, scale),
67-
Some(DecimalValue::I256(v256)) => {
68-
ScalarValue::Decimal256(Some(v256.into()), precision, scale)
69-
}
70-
_ => vortex_bail!(
71-
"invalid ScalarValue for decimal with precision {}",
72-
precision
73-
),
92+
Some(value) => match value.cast::<i256>() {
93+
Some(v256) => {
94+
ScalarValue::Decimal256(Some(v256.into()), precision, scale)
95+
}
96+
None => {
97+
vortex_bail!(
98+
"invalid ScalarValue {value} for decimal with precision {precision}",
99+
)
100+
}
101+
},
74102
}
75103
}
76104
}
@@ -220,6 +248,32 @@ impl FromDataFusion<ScalarValue> for Scalar {
220248
.unwrap_or_else(vortex::scalar::ScalarValue::null),
221249
)
222250
}
251+
ScalarValue::Decimal32(decimal, precision, scale) => {
252+
let decimal_dtype = DecimalDType::new(*precision, *scale);
253+
let nullable = Nullability::Nullable;
254+
if let Some(value) = decimal {
255+
Scalar::decimal(
256+
DecimalValue::I32(*value),
257+
decimal_dtype,
258+
Nullability::Nullable,
259+
)
260+
} else {
261+
Scalar::null(DType::Decimal(decimal_dtype, nullable))
262+
}
263+
}
264+
ScalarValue::Decimal64(decimal, precision, scale) => {
265+
let decimal_dtype = DecimalDType::new(*precision, *scale);
266+
let nullable = Nullability::Nullable;
267+
if let Some(value) = decimal {
268+
Scalar::decimal(
269+
DecimalValue::I64(*value),
270+
decimal_dtype,
271+
Nullability::Nullable,
272+
)
273+
} else {
274+
Scalar::null(DType::Decimal(decimal_dtype, nullable))
275+
}
276+
}
223277
ScalarValue::Decimal128(decimal, precision, scale) => {
224278
let decimal_dtype = DecimalDType::new(*precision, *scale);
225279
let nullable = Nullability::Nullable;
@@ -366,17 +420,41 @@ mod tests {
366420
}
367421

368422
#[rstest]
423+
#[case::decimal32_some(
424+
Scalar::decimal(
425+
DecimalValue::I32(1234),
426+
DecimalDType::new(5, 2),
427+
Nullability::NonNullable
428+
),
429+
ScalarValue::Decimal32(Some(1234), 5, 2)
430+
)]
431+
#[case::decimal32_null(
432+
Scalar::null(DType::Decimal(DecimalDType::new(5, 2), Nullability::Nullable)),
433+
ScalarValue::Decimal32(None, 5, 2)
434+
)]
435+
#[case::decimal64_some(
436+
Scalar::decimal(
437+
DecimalValue::I64(12345),
438+
DecimalDType::new(10, 2),
439+
Nullability::NonNullable
440+
),
441+
ScalarValue::Decimal64(Some(12345), 10, 2)
442+
)]
443+
#[case::decimal64_null(
444+
Scalar::null(DType::Decimal(DecimalDType::new(10, 2), Nullability::Nullable)),
445+
ScalarValue::Decimal64(None, 10, 2)
446+
)]
369447
#[case::decimal128_some(
370448
Scalar::decimal(
371449
DecimalValue::I128(12345),
372-
DecimalDType::new(10, 2),
450+
DecimalDType::new(20, 2),
373451
Nullability::NonNullable
374452
),
375-
ScalarValue::Decimal128(Some(12345), 10, 2)
453+
ScalarValue::Decimal128(Some(12345), 20, 2)
376454
)]
377455
#[case::decimal128_null(
378-
Scalar::null(DType::Decimal(DecimalDType::new(10, 2), Nullability::Nullable)),
379-
ScalarValue::Decimal128(None, 10, 2)
456+
Scalar::null(DType::Decimal(DecimalDType::new(20, 2), Nullability::Nullable)),
457+
ScalarValue::Decimal128(None, 20, 2)
380458
)]
381459
#[case::decimal256_some(
382460
Scalar::decimal(
@@ -575,8 +653,16 @@ mod tests {
575653
ScalarValue::Binary(None)
576654
)]
577655
#[case::null_decimal128(
656+
Scalar::null(DType::Decimal(DecimalDType::new(20, 2), Nullability::Nullable)),
657+
ScalarValue::Decimal128(None, 20, 2)
658+
)]
659+
#[case::null_decimal64(
578660
Scalar::null(DType::Decimal(DecimalDType::new(10, 2), Nullability::Nullable)),
579-
ScalarValue::Decimal128(None, 10, 2)
661+
ScalarValue::Decimal64(None, 10, 2)
662+
)]
663+
#[case::null_decimal32(
664+
Scalar::null(DType::Decimal(DecimalDType::new(5, 2), Nullability::Nullable)),
665+
ScalarValue::Decimal32(None, 5, 2)
580666
)]
581667
fn test_null_handling(#[case] vortex_null: Scalar, #[case] expected_df_null: ScalarValue) {
582668
// Test Vortex -> DataFusion

vortex-datafusion/src/persistent/mod.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -254,8 +254,9 @@ mod tests {
254254
┌───────────────────────────┐
255255
│ SortPreservingMergeExec │
256256
│ -------------------- │
257-
│ c1 ASC NULLS LASTlimit: │
258-
│ 3 │
257+
│ c1 ASC NULLS LAST │
258+
│ │
259+
│ limit: 3 │
259260
└─────────────┬─────────────┘
260261
┌─────────────┴─────────────┐
261262
│ DataSourceExec │

0 commit comments

Comments
 (0)