Skip to content

Commit e7b8cae

Browse files
joseph-isaacsjoseph-isaacs
authored
fix[duckdb]: no to_struct in exporter (#5750)
Signed-off-by: Joe Isaacs <[email protected]>
1 parent e7e1748 commit e7b8cae

File tree

3 files changed

+69
-44
lines changed

3 files changed

+69
-44
lines changed

‎vortex-array/src/vectors.rs‎

Lines changed: 42 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -38,23 +38,23 @@ use crate::arrays::VarBinViewArray;
3838
use crate::validity::Validity;
3939

4040
/// Trait for converting vector types into arrays.
41-
pub trait VectorIntoArray {
41+
pub trait VectorIntoArray<T> {
4242
/// Converts the vector into an array of the specified data type.
43-
fn into_array(self, dtype: &DType) -> ArrayRef;
43+
fn into_array(self, dtype: &DType) -> T;
4444
}
4545

46-
impl VectorIntoArray for Vector {
46+
impl VectorIntoArray<ArrayRef> for Vector {
4747
fn into_array(self, dtype: &DType) -> ArrayRef {
4848
match dtype {
49-
DType::Null => self.into_null().into_array(dtype),
50-
DType::Bool(_) => self.into_bool().into_array(dtype),
51-
DType::Primitive(..) => self.into_primitive().into_array(dtype),
52-
DType::Decimal(..) => self.into_decimal().into_array(dtype),
53-
DType::Utf8(_) => self.into_string().into_array(dtype),
54-
DType::Binary(_) => self.into_binary().into_array(dtype),
55-
DType::List(..) => self.into_list().into_array(dtype),
56-
DType::FixedSizeList(..) => self.into_fixed_size_list().into_array(dtype),
57-
DType::Struct(..) => self.into_struct().into_array(dtype),
49+
DType::Null => self.into_null().into_array(dtype).into_array(),
50+
DType::Bool(_) => self.into_bool().into_array(dtype).into_array(),
51+
DType::Primitive(..) => self.into_primitive().into_array(dtype).into_array(),
52+
DType::Decimal(..) => self.into_decimal().into_array(dtype).into_array(),
53+
DType::Utf8(_) => self.into_string().into_array(dtype).into_array(),
54+
DType::Binary(_) => self.into_binary().into_array(dtype).into_array(),
55+
DType::List(..) => self.into_list().into_array(dtype).into_array(),
56+
DType::FixedSizeList(..) => self.into_fixed_size_list().into_array(dtype).into_array(),
57+
DType::Struct(..) => self.into_struct().into_array(dtype).into_array(),
5858
DType::Extension(ext_dtype) => {
5959
let storage = self.into_array(ext_dtype.storage_dtype());
6060
ExtensionArray::new(ext_dtype.clone(), storage).into_array()
@@ -63,33 +63,32 @@ impl VectorIntoArray for Vector {
6363
}
6464
}
6565

66-
impl VectorIntoArray for NullVector {
67-
fn into_array(self, dtype: &DType) -> ArrayRef {
66+
impl VectorIntoArray<NullArray> for NullVector {
67+
fn into_array(self, dtype: &DType) -> NullArray {
6868
assert!(matches!(dtype, DType::Null));
69-
NullArray::new(self.len()).into_array()
69+
NullArray::new(self.len())
7070
}
7171
}
7272

73-
impl VectorIntoArray for BoolVector {
74-
fn into_array(self, dtype: &DType) -> ArrayRef {
73+
impl VectorIntoArray<BoolArray> for BoolVector {
74+
fn into_array(self, dtype: &DType) -> BoolArray {
7575
assert!(matches!(dtype, DType::Bool(_)));
7676

7777
let (bits, validity) = self.into_parts();
7878
BoolArray::from_bit_buffer(bits, Validity::from_mask(validity, dtype.nullability()))
79-
.into_array()
8079
}
8180
}
8281

83-
impl VectorIntoArray for PrimitiveVector {
84-
fn into_array(self, dtype: &DType) -> ArrayRef {
82+
impl VectorIntoArray<PrimitiveArray> for PrimitiveVector {
83+
fn into_array(self, dtype: &DType) -> PrimitiveArray {
8584
match_each_native_ptype!(self.ptype(), |T| {
8685
<T as NativePType>::downcast(self).into_array(dtype)
8786
})
8887
}
8988
}
9089

91-
impl<T: NativePType> VectorIntoArray for PVector<T> {
92-
fn into_array(self, dtype: &DType) -> ArrayRef {
90+
impl<T: NativePType> VectorIntoArray<PrimitiveArray> for PVector<T> {
91+
fn into_array(self, dtype: &DType) -> PrimitiveArray {
9392
assert!(matches!(dtype, DType::Primitive(_, _)));
9493
assert_eq!(T::PTYPE, dtype.as_ptype());
9594

@@ -101,20 +100,19 @@ impl<T: NativePType> VectorIntoArray for PVector<T> {
101100
Validity::from_mask(validity, dtype.nullability()),
102101
)
103102
}
104-
.into_array()
105103
}
106104
}
107105

108-
impl VectorIntoArray for DecimalVector {
109-
fn into_array(self, dtype: &DType) -> ArrayRef {
106+
impl VectorIntoArray<DecimalArray> for DecimalVector {
107+
fn into_array(self, dtype: &DType) -> DecimalArray {
110108
match_each_decimal_value_type!(self.decimal_type(), |D| {
111109
<D as NativeDecimalType>::downcast(self).into_array(dtype)
112110
})
113111
}
114112
}
115113

116-
impl<D: NativeDecimalType> VectorIntoArray for DVector<D> {
117-
fn into_array(self, dtype: &DType) -> ArrayRef {
114+
impl<D: NativeDecimalType> VectorIntoArray<DecimalArray> for DVector<D> {
115+
fn into_array(self, dtype: &DType) -> DecimalArray {
118116
assert!(matches!(dtype, DType::Decimal(_, _)));
119117

120118
let nullability = dtype.nullability();
@@ -133,12 +131,11 @@ impl<D: NativeDecimalType> VectorIntoArray for DVector<D> {
133131
Validity::from_mask(validity, nullability),
134132
)
135133
}
136-
.into_array()
137134
}
138135
}
139136

140-
impl<T: BinaryViewType> VectorIntoArray for BinaryViewVector<T> {
141-
fn into_array(self, dtype: &DType) -> ArrayRef {
137+
impl<T: BinaryViewType> VectorIntoArray<VarBinViewArray> for BinaryViewVector<T> {
138+
fn into_array(self, dtype: &DType) -> VarBinViewArray {
142139
assert!(matches!(dtype, DType::Utf8(_)));
143140

144141
let (views, buffers, validity) = self.into_parts();
@@ -155,12 +152,11 @@ impl<T: BinaryViewType> VectorIntoArray for BinaryViewVector<T> {
155152
validity,
156153
)
157154
}
158-
.into_array()
159155
}
160156
}
161157

162-
impl VectorIntoArray for ListViewVector {
163-
fn into_array(self, dtype: &DType) -> ArrayRef {
158+
impl VectorIntoArray<ListViewArray> for ListViewVector {
159+
fn into_array(self, dtype: &DType) -> ListViewArray {
164160
assert!(matches!(dtype, DType::List(_, _)));
165161

166162
let (elements, offsets, sizes, validity) = self.into_parts();
@@ -178,12 +174,19 @@ impl VectorIntoArray for ListViewVector {
178174
let sizes = sizes.into_array(&sizes_dtype);
179175

180176
// SAFETY: vectors maintain all invariants required for array creation
181-
unsafe { ListViewArray::new_unchecked(elements, offsets, sizes, validity) }.into_array()
177+
unsafe {
178+
ListViewArray::new_unchecked(
179+
elements,
180+
offsets.into_array(),
181+
sizes.into_array(),
182+
validity,
183+
)
184+
}
182185
}
183186
}
184187

185-
impl VectorIntoArray for FixedSizeListVector {
186-
fn into_array(self, dtype: &DType) -> ArrayRef {
188+
impl VectorIntoArray<FixedSizeListArray> for FixedSizeListVector {
189+
fn into_array(self, dtype: &DType) -> FixedSizeListArray {
187190
assert!(matches!(dtype, DType::FixedSizeList(_, _, _)));
188191

189192
let len = self.len();
@@ -198,12 +201,12 @@ impl VectorIntoArray for FixedSizeListVector {
198201
.into_array(elements_dtype);
199202

200203
// SAFETY: vectors maintain all invariants required for array creation
201-
unsafe { FixedSizeListArray::new_unchecked(elements, size, validity, len) }.into_array()
204+
unsafe { FixedSizeListArray::new_unchecked(elements, size, validity, len) }
202205
}
203206
}
204207

205-
impl VectorIntoArray for StructVector {
206-
fn into_array(self, dtype: &DType) -> ArrayRef {
208+
impl VectorIntoArray<StructArray> for StructVector {
209+
fn into_array(self, dtype: &DType) -> StructArray {
207210
assert!(matches!(dtype, DType::Struct(_, _)));
208211

209212
let len = self.len();
@@ -222,6 +225,5 @@ impl VectorIntoArray for StructVector {
222225

223226
// SAFETY: vectors maintain all invariants required for array creation
224227
unsafe { StructArray::new_unchecked(field_arrays, struct_fields.clone(), len, validity) }
225-
.into_array()
226228
}
227229
}

‎vortex-duckdb/src/scan.rs‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,22 @@ use itertools::Itertools;
2424
use num_traits::AsPrimitive;
2525
use url::Url;
2626
use vortex::VortexSessionDefault;
27+
use vortex::array::Array;
2728
use vortex::array::ArrayRef;
2829
use vortex::array::ToCanonical;
30+
use vortex::array::VectorExecutor;
31+
use vortex::array::arrays::ScalarFnVTable;
32+
use vortex::array::arrays::StructArray;
33+
use vortex::array::arrays::StructVTable;
2934
use vortex::array::optimizer::ArrayOptimizer;
35+
use vortex::array::vectors::VectorIntoArray;
3036
use vortex::dtype::FieldNames;
3137
use vortex::error::VortexExpect;
3238
use vortex::error::VortexResult;
3339
use vortex::error::vortex_bail;
3440
use vortex::error::vortex_err;
3541
use vortex::expr::Expression;
42+
use vortex::expr::Pack;
3643
use vortex::expr::and;
3744
use vortex::expr::and_collect;
3845
use vortex::expr::col;
@@ -330,13 +337,30 @@ impl TableFunction for VortexTableFunction {
330337
let (array_result, conversion_cache) = result?;
331338

332339
let array_result = if *USE_VORTEX_OPERATORS {
333-
array_result.optimize_recursive()?
340+
let array_result = array_result.optimize_recursive()?;
341+
if let Some(array) = array_result.as_opt::<StructVTable>() {
342+
array.clone()
343+
} else if let Some(array) = array_result.as_opt::<ScalarFnVTable>()
344+
&& let Some(pack_options) = array.scalar_fn().as_opt::<Pack>()
345+
{
346+
StructArray::new(
347+
pack_options.names.clone(),
348+
array.children(),
349+
array.len(),
350+
pack_options.nullability.into(),
351+
)
352+
} else {
353+
array_result
354+
.execute_vector(&global_state.session)?
355+
.into_struct()
356+
.into_array(array_result.dtype())
357+
}
334358
} else {
335-
array_result
359+
array_result.to_struct()
336360
};
337361

338362
local_state.exporter = Some(ArrayExporter::try_new(
339-
&array_result.to_struct(),
363+
&array_result,
340364
&conversion_cache,
341365
&global_state.session,
342366
)?);

‎vortex-vector/src/struct_/vector.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ use std::fmt::Debug;
77
use std::ops::BitAnd;
88
use std::ops::RangeBounds;
99
use std::sync::Arc;
10-
1110
use vortex_error::VortexExpect;
1211
use vortex_error::VortexResult;
1312
use vortex_error::vortex_ensure;

0 commit comments

Comments
 (0)