Skip to content

Commit 412aff2

Browse files
gatesngatesn
committed
Flat Layout Execution
Signed-off-by: Nicholas Gates <[email protected]>
1 parent 86a215d commit 412aff2

File tree

23 files changed

+699
-626
lines changed

23 files changed

+699
-626
lines changed

‎Cargo.lock‎

Lines changed: 2 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vortex-array/src/arrays/filter/vtable.rs‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

44
use std::hash::Hasher;
5+
use std::ops::Range;
56

67
use vortex_buffer::BufferHandle;
78
use vortex_compute::filter::Filter;
@@ -10,14 +11,17 @@ use vortex_error::VortexExpect;
1011
use vortex_error::VortexResult;
1112
use vortex_error::vortex_bail;
1213
use vortex_mask::Mask;
14+
use vortex_scalar::Scalar;
1315
use vortex_vector::Vector;
1416

1517
use crate::Array;
1618
use crate::ArrayBufferVisitor;
1719
use crate::ArrayChildVisitor;
1820
use crate::ArrayEq;
1921
use crate::ArrayHash;
22+
use crate::ArrayRef;
2023
use crate::Canonical;
24+
use crate::IntoArray;
2125
use crate::Precision;
2226
use crate::arrays::LEGACY_SESSION;
2327
use crate::arrays::filter::array::FilterArray;
@@ -32,6 +36,7 @@ use crate::vtable::ArrayVTableExt;
3236
use crate::vtable::BaseArrayVTable;
3337
use crate::vtable::CanonicalVTable;
3438
use crate::vtable::NotSupported;
39+
use crate::vtable::OperationsVTable;
3540
use crate::vtable::VTable;
3641
use crate::vtable::ValidityVTable;
3742
use crate::vtable::VisitorVTable;
@@ -46,7 +51,7 @@ impl VTable for FilterVTable {
4651
type Metadata = Mask;
4752
type ArrayVTable = Self;
4853
type CanonicalVTable = Self;
49-
type OperationsVTable = NotSupported;
54+
type OperationsVTable = Self;
5055
type ValidityVTable = Self;
5156
type VisitorVTable = Self;
5257
type ComputeVTable = NotSupported;
@@ -126,6 +131,17 @@ impl CanonicalVTable<FilterVTable> for FilterVTable {
126131
}
127132
}
128133

134+
impl OperationsVTable<FilterVTable> for FilterVTable {
135+
fn slice(array: &FilterArray, range: Range<usize>) -> ArrayRef {
136+
FilterArray::new(array.child.slice(range.clone()), array.mask.slice(range)).into_array()
137+
}
138+
139+
fn scalar_at(array: &FilterArray, index: usize) -> Scalar {
140+
let rank_idx = array.mask.rank(index);
141+
array.child.scalar_at(rank_idx)
142+
}
143+
}
144+
129145
impl ValidityVTable<FilterVTable> for FilterVTable {
130146
fn is_valid(array: &FilterArray, index: usize) -> bool {
131147
let rank_idx = array.mask.rank(index);

‎vortex-compute/Cargo.toml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,15 @@ workspace = true
2121

2222
[dependencies]
2323
arrow-array = { workspace = true }
24+
arrow-buffer = { workspace = true }
2425
arrow-ord = { workspace = true }
26+
arrow-schema = { workspace = true }
2527

2628
vortex-buffer = { workspace = true }
2729
vortex-dtype = { workspace = true }
2830
vortex-error = { workspace = true }
29-
vortex-mask = { workspace = true }
30-
vortex-vector = { workspace = true, features = ["arrow"] }
31+
vortex-mask = { workspace = true, features = ["arrow"] }
32+
vortex-vector = { workspace = true }
3133

3234
log = { workspace = true }
3335
multiversion = { workspace = true }

vortex-vector/src/arrow/binaryview.rs renamed to vortex-compute/src/arrow/binaryview.rs

Lines changed: 19 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -3,26 +3,28 @@
33

44
use std::sync::Arc;
55

6-
use crate::arrow::nulls_to_mask;
7-
use crate::binaryview::BinaryType;
8-
use crate::binaryview::BinaryView;
9-
use crate::binaryview::BinaryViewVector;
10-
use crate::binaryview::StringType;
116
use arrow_array::Array;
127
use arrow_array::ArrayRef;
138
use arrow_array::GenericByteViewArray;
149
use vortex_buffer::Buffer;
1510
use vortex_buffer::ByteBuffer;
16-
use vortex_error::VortexError;
17-
use vortex_error::vortex_err;
11+
use vortex_error::VortexResult;
12+
use vortex_vector::binaryview::BinaryType;
13+
use vortex_vector::binaryview::BinaryView;
14+
use vortex_vector::binaryview::BinaryViewVector;
15+
use vortex_vector::binaryview::StringType;
16+
17+
use crate::arrow::IntoArrow;
18+
use crate::arrow::IntoVector;
19+
use crate::arrow::nulls_to_mask;
1820

1921
macro_rules! impl_binaryview_to_arrow {
2022
($T:ty, $A:ty) => {
21-
impl TryFrom<BinaryViewVector<$T>> for ArrayRef {
22-
type Error = VortexError;
23+
impl IntoArrow for BinaryViewVector<$T> {
24+
type Output = ArrayRef;
2325

24-
fn try_from(value: BinaryViewVector<$T>) -> Result<Self, Self::Error> {
25-
let (views, buffers, validity) = value.into_parts();
26+
fn into_arrow(self) -> VortexResult<Self::Output> {
27+
let (views, buffers, validity) = self.into_parts();
2628

2729
let views = Buffer::<u128>::from_byte_buffer(views.into_byte_buffer())
2830
.into_arrow_scalar_buffer();
@@ -47,30 +49,19 @@ impl_binaryview_to_arrow!(StringType, arrow_array::types::StringViewType);
4749

4850
macro_rules! impl_binaryview_from_arrow {
4951
($T:ty, $A:ty) => {
50-
impl TryFrom<&dyn Array> for BinaryViewVector<$T> {
51-
type Error = VortexError;
52-
53-
fn try_from(value: &dyn Array) -> Result<Self, Self::Error> {
54-
let array = value
55-
.as_any()
56-
.downcast_ref::<GenericByteViewArray<$A>>()
57-
.ok_or_else(|| {
58-
vortex_err!(
59-
"expected GenericByteViewArray<{}>, got {}",
60-
stringify!($A),
61-
value.data_type()
62-
)
63-
})?;
52+
impl IntoVector for &GenericByteViewArray<$A> {
53+
type Output = BinaryViewVector<$T>;
6454

55+
fn into_vector(self) -> VortexResult<Self::Output> {
6556
// Convert views from Arrow's u128 representation to BinaryView
66-
let arrow_views = array.views();
57+
let arrow_views = self.views();
6758
let views = Buffer::<BinaryView>::from_byte_buffer(
6859
Buffer::<u128>::from_arrow_scalar_buffer(arrow_views.clone())
6960
.into_byte_buffer(),
7061
);
7162

7263
// Convert buffers
73-
let buffers: Box<[ByteBuffer]> = array
64+
let buffers: Box<[ByteBuffer]> = self
7465
.data_buffers()
7566
.iter()
7667
.map(|b| {
@@ -81,7 +72,7 @@ macro_rules! impl_binaryview_from_arrow {
8172
})
8273
.collect();
8374

84-
let validity = nulls_to_mask(array.nulls(), array.len());
75+
let validity = nulls_to_mask(self.nulls(), self.len());
8576

8677
// SAFETY: Arrow's GenericByteViewArray maintains the same invariants as our BinaryViewVector
8778
Ok(unsafe { BinaryViewVector::new_unchecked(views, Arc::new(buffers), validity) })

‎vortex-compute/src/arrow/bool.rs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
use std::sync::Arc;
5+
6+
use arrow_array::Array;
7+
use arrow_array::ArrayRef;
8+
use arrow_array::BooleanArray;
9+
use vortex_buffer::BitBuffer;
10+
use vortex_error::VortexResult;
11+
use vortex_vector::bool::BoolVector;
12+
13+
use crate::arrow::IntoArrow;
14+
use crate::arrow::IntoVector;
15+
use crate::arrow::nulls_to_mask;
16+
17+
impl IntoArrow for BoolVector {
18+
type Output = ArrayRef;
19+
20+
fn into_arrow(self) -> VortexResult<Self::Output> {
21+
let (bits, validity) = self.into_parts();
22+
Ok(Arc::new(BooleanArray::new(bits.into(), validity.into())))
23+
}
24+
}
25+
26+
impl IntoVector for &BooleanArray {
27+
type Output = BoolVector;
28+
29+
fn into_vector(self) -> VortexResult<Self::Output> {
30+
let bits = BitBuffer::from(self.values().clone());
31+
let validity = nulls_to_mask(self.nulls(), self.len());
32+
Ok(BoolVector::new(bits, validity))
33+
}
34+
}

‎vortex-compute/src/arrow/decimal.rs‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
use std::sync::Arc;
5+
6+
use arrow_array::Array;
7+
use arrow_array::ArrayRef;
8+
use arrow_array::PrimitiveArray;
9+
use arrow_array::types::Decimal32Type;
10+
use arrow_array::types::Decimal64Type;
11+
use arrow_array::types::Decimal128Type;
12+
use arrow_array::types::Decimal256Type;
13+
use vortex_buffer::Buffer;
14+
use vortex_dtype::PrecisionScale;
15+
use vortex_dtype::i256;
16+
use vortex_error::VortexResult;
17+
use vortex_vector::decimal::DVector;
18+
use vortex_vector::decimal::DecimalVector;
19+
20+
use crate::arrow::IntoArrow;
21+
use crate::arrow::IntoVector;
22+
use crate::arrow::nulls_to_mask;
23+
24+
impl IntoArrow for DecimalVector {
25+
type Output = ArrayRef;
26+
27+
fn into_arrow(self) -> VortexResult<Self::Output> {
28+
match self {
29+
DecimalVector::D8(v) => v.into_arrow(),
30+
DecimalVector::D16(v) => v.into_arrow(),
31+
DecimalVector::D32(v) => v.into_arrow(),
32+
DecimalVector::D64(v) => v.into_arrow(),
33+
DecimalVector::D128(v) => v.into_arrow(),
34+
DecimalVector::D256(v) => v.into_arrow(),
35+
}
36+
}
37+
}
38+
39+
macro_rules! impl_decimal_upcast_i32 {
40+
($T:ty) => {
41+
impl IntoArrow for DVector<$T> {
42+
type Output = ArrayRef;
43+
44+
fn into_arrow(self) -> VortexResult<Self::Output> {
45+
let (_, elements, validity) = self.into_parts();
46+
// Upcast the DVector to Arrow's smallest decimal type (Decimal32)
47+
let elements =
48+
Buffer::<i32>::from_trusted_len_iter(elements.iter().map(|i| *i as i32));
49+
Ok(Arc::new(PrimitiveArray::<Decimal32Type>::new(
50+
elements.into_arrow_scalar_buffer(),
51+
validity.into(),
52+
)))
53+
}
54+
}
55+
};
56+
}
57+
58+
impl_decimal_upcast_i32!(i8);
59+
impl_decimal_upcast_i32!(i16);
60+
61+
/// Direct Arrow conversion for vectors that map directly to Arrow decimal types.
62+
macro_rules! impl_decimal_to_arrow {
63+
($T:ty, $A:ty) => {
64+
impl IntoArrow for DVector<$T> {
65+
type Output = ArrayRef;
66+
67+
fn into_arrow(self) -> VortexResult<Self::Output> {
68+
let (_, elements, validity) = self.into_parts();
69+
Ok(Arc::new(PrimitiveArray::<$A>::new(
70+
elements.into_arrow_scalar_buffer(),
71+
validity.into(),
72+
)))
73+
}
74+
}
75+
};
76+
}
77+
78+
impl_decimal_to_arrow!(i32, Decimal32Type);
79+
impl_decimal_to_arrow!(i64, Decimal64Type);
80+
impl_decimal_to_arrow!(i128, Decimal128Type);
81+
82+
impl IntoArrow for DVector<i256> {
83+
type Output = ArrayRef;
84+
85+
fn into_arrow(self) -> VortexResult<Self::Output> {
86+
let (_, elements, validity) = self.into_parts();
87+
88+
// Transmute the elements from our i256 to Arrow's.
89+
// SAFETY: we use Arrow's type internally for our layout.
90+
let elements =
91+
unsafe { std::mem::transmute::<Buffer<i256>, Buffer<arrow_buffer::i256>>(elements) };
92+
93+
Ok(Arc::new(PrimitiveArray::<Decimal256Type>::new(
94+
elements.into_arrow_scalar_buffer(),
95+
validity.into(),
96+
)))
97+
}
98+
}
99+
100+
/// Convert a Decimal32 Arrow array to a DecimalVector.
101+
impl IntoVector for &PrimitiveArray<Decimal32Type> {
102+
type Output = DecimalVector;
103+
104+
fn into_vector(self) -> VortexResult<Self::Output> {
105+
let (precision, scale) = match self.data_type() {
106+
arrow_schema::DataType::Decimal32(p, s) => (*p, *s),
107+
_ => unreachable!("PrimitiveArray<Decimal32Type> must have Decimal32 data type"),
108+
};
109+
110+
let elements = Buffer::<i32>::from_arrow_scalar_buffer(self.values().clone());
111+
let validity = nulls_to_mask(self.nulls(), self.len());
112+
let ps = PrecisionScale::<i32>::new(precision, scale);
113+
114+
Ok(DecimalVector::D32(DVector::new(ps, elements, validity)))
115+
}
116+
}
117+
118+
/// Convert a Decimal64 Arrow array to a DecimalVector.
119+
impl IntoVector for &PrimitiveArray<Decimal64Type> {
120+
type Output = DecimalVector;
121+
122+
fn into_vector(self) -> VortexResult<Self::Output> {
123+
let (precision, scale) = match self.data_type() {
124+
arrow_schema::DataType::Decimal64(p, s) => (*p, *s),
125+
_ => unreachable!("PrimitiveArray<Decimal64Type> must have Decimal64 data type"),
126+
};
127+
128+
let elements = Buffer::<i64>::from_arrow_scalar_buffer(self.values().clone());
129+
let validity = nulls_to_mask(self.nulls(), self.len());
130+
let ps = PrecisionScale::<i64>::new(precision, scale);
131+
132+
Ok(DecimalVector::D64(DVector::new(ps, elements, validity)))
133+
}
134+
}
135+
136+
/// Convert a Decimal128 Arrow array to a DecimalVector.
137+
impl IntoVector for &PrimitiveArray<Decimal128Type> {
138+
type Output = DecimalVector;
139+
140+
fn into_vector(self) -> VortexResult<Self::Output> {
141+
let (precision, scale) = match self.data_type() {
142+
arrow_schema::DataType::Decimal128(p, s) => (*p, *s),
143+
_ => unreachable!("PrimitiveArray<Decimal128Type> must have Decimal128 data type"),
144+
};
145+
146+
let elements = Buffer::<i128>::from_arrow_scalar_buffer(self.values().clone());
147+
let validity = nulls_to_mask(self.nulls(), self.len());
148+
let ps = PrecisionScale::<i128>::new(precision, scale);
149+
150+
Ok(DecimalVector::D128(DVector::new(ps, elements, validity)))
151+
}
152+
}
153+
154+
/// Convert a Decimal256 Arrow array to a DecimalVector.
155+
impl IntoVector for &PrimitiveArray<Decimal256Type> {
156+
type Output = DecimalVector;
157+
158+
fn into_vector(self) -> VortexResult<Self::Output> {
159+
let (precision, scale) = match self.data_type() {
160+
arrow_schema::DataType::Decimal256(p, s) => (*p, *s),
161+
_ => unreachable!("PrimitiveArray<Decimal256Type> must have Decimal256 data type"),
162+
};
163+
164+
let elements =
165+
Buffer::<arrow_buffer::i256>::from_arrow_scalar_buffer(self.values().clone());
166+
// SAFETY: we use Arrow's type internally for our layout.
167+
let elements =
168+
unsafe { std::mem::transmute::<Buffer<arrow_buffer::i256>, Buffer<i256>>(elements) };
169+
let validity = nulls_to_mask(self.nulls(), self.len());
170+
let ps = PrecisionScale::<i256>::new(precision, scale);
171+
172+
Ok(DecimalVector::D256(DVector::new(ps, elements, validity)))
173+
}
174+
}

0 commit comments

Comments
 (0)