Skip to content

Commit aca77c4

Browse files
committed
Merge branch 'develop' into ngates/inline-rules
2 parents 157579e + e16faa6 commit aca77c4

File tree

51 files changed

+2058
-115
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+2058
-115
lines changed

encodings/fsst/src/array.rs

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ use vortex_buffer::BufferHandle;
4242
use vortex_dtype::DType;
4343
use vortex_dtype::Nullability;
4444
use vortex_dtype::PType;
45-
use vortex_error::VortexExpect;
4645
use vortex_error::VortexResult;
4746
use vortex_error::vortex_bail;
4847
use vortex_error::vortex_ensure;
@@ -157,7 +156,10 @@ impl VTable for FSSTVTable {
157156
);
158157

159158
let mut children_iter = children.into_iter();
160-
let codes = children_iter.next().vortex_expect("codes child");
159+
let codes = children_iter
160+
.next()
161+
.ok_or_else(|| vortex_err!("FSSTArray with_children missing codes"))?;
162+
161163
let codes = codes
162164
.as_opt::<VarBinVTable>()
163165
.ok_or_else(|| {
@@ -169,7 +171,7 @@ impl VTable for FSSTVTable {
169171
.clone();
170172
let uncompressed_lengths = children_iter
171173
.next()
172-
.vortex_expect("uncompressed_lengths child");
174+
.ok_or_else(|| vortex_err!("FSSTArray with_children missing uncompressed_lengths"))?;
173175

174176
array.codes = codes;
175177
array.uncompressed_lengths = uncompressed_lengths;

vortex-array/src/arrays/bool/vtable/rules.rs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,12 @@ impl ArrayParentReduceRule<BoolVTable> for BoolMaskedValidityRule {
3535
&self,
3636
array: &BoolArray,
3737
parent: &MaskedArray,
38-
_child_idx: usize,
38+
child_idx: usize,
3939
) -> VortexResult<Option<ArrayRef>> {
40+
if child_idx > 0 {
41+
return Ok(None);
42+
}
43+
4044
// Merge the parent's validity mask into the child's validity
4145
// TODO(joe): make this lazy
4246
Ok(Some(

vortex-array/src/arrays/dict/vtable/mod.rs

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

44
use vortex_buffer::BufferHandle;
5+
use vortex_compute::take::Take;
56
use vortex_dtype::DType;
67
use vortex_dtype::Nullability;
78
use vortex_dtype::PType;
@@ -16,6 +17,9 @@ use crate::ArrayRef;
1617
use crate::DeserializeMetadata;
1718
use crate::ProstMetadata;
1819
use crate::SerializeMetadata;
20+
use crate::kernel::BindCtx;
21+
use crate::kernel::KernelRef;
22+
use crate::kernel::kernel;
1923
use crate::serde::ArrayChildren;
2024
use crate::vtable;
2125
use crate::vtable::ArrayId;
@@ -124,4 +128,16 @@ impl VTable for DictVTable {
124128
array.values = values;
125129
Ok(())
126130
}
131+
132+
fn bind_kernel(array: &Self::Array, ctx: &mut BindCtx) -> VortexResult<KernelRef> {
133+
let values_kernel = array.values().bind_kernel(ctx)?;
134+
let codes_kernel = array.codes().bind_kernel(ctx)?;
135+
136+
Ok(kernel(move || {
137+
let values = values_kernel.execute()?;
138+
let codes = codes_kernel.execute()?.into_primitive();
139+
140+
Ok(values.take(&codes))
141+
}))
142+
}
127143
}

vortex-array/src/arrays/scalar_fn/kernel.rs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,10 @@ impl Kernel for ScalarFnKernel {
5757
return_dtype: self.return_dtype,
5858
};
5959

60-
Ok(self.scalar_fn.execute(args)?.ensure_vector(self.row_count))
60+
Ok(self
61+
.scalar_fn
62+
.execute(args)?
63+
.unwrap_into_vector(self.row_count))
6164
}
6265

6366
fn push_down_filter(self: Box<Self>, selection: &Mask) -> VortexResult<PushDownResult> {

vortex-array/src/arrays/scalar_fn/vtable/canonical.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ impl CanonicalVTable<ScalarFnVTable> for ScalarFnVTable {
3737
.scalar_fn
3838
.execute(ctx)
3939
.vortex_expect("Canonicalize should be fallible")
40-
.ensure_vector(len);
40+
.unwrap_into_vector(len);
4141

4242
result_vector.into_array(&array.dtype).to_canonical()
4343
}

vortex-array/src/builders/fixed_size_list.rs

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,33 @@ impl FixedSizeListBuilder {
7373
}
7474
}
7575

76+
/// Appends an array as a single non-null list entry to the builder.
77+
///
78+
/// The input `array` must have the same dtype as the element dtype of this list builder, and
79+
/// its length must match the fixed list size.
80+
///
81+
/// Note that the list entry will be non-null but the elements themselves are allowed to be null
82+
/// (only if the elements [`DType`] is nullable, of course).
83+
pub fn append_array_as_list(&mut self, array: &dyn Array) -> VortexResult<()> {
84+
vortex_ensure!(
85+
array.dtype() == self.element_dtype(),
86+
"Array dtype {:?} does not match list element dtype {:?}",
87+
array.dtype(),
88+
self.element_dtype()
89+
);
90+
vortex_ensure!(
91+
array.len() == self.list_size() as usize,
92+
"Array length {} does not match fixed list size {}",
93+
array.len(),
94+
self.list_size()
95+
);
96+
97+
self.elements_builder.extend_from_array(array);
98+
self.nulls.append_non_null();
99+
100+
Ok(())
101+
}
102+
76103
/// Appends a fixed-size list `value` to the builder.
77104
///
78105
/// Note that a [`ListScalar`] can represent both a [`ListArray`] scalar **and** a
@@ -770,4 +797,63 @@ mod tests {
770797
let wrong_scalar = Scalar::from(42i32);
771798
assert!(builder.append_scalar(&wrong_scalar).is_err());
772799
}
800+
801+
#[test]
802+
fn test_append_array_as_list() {
803+
let dtype: Arc<DType> = Arc::new(I32.into());
804+
let mut builder = FixedSizeListBuilder::with_capacity(dtype.clone(), 3, NonNullable, 10);
805+
806+
// Append a primitive array as a single list entry.
807+
let arr1 = buffer![1i32, 2, 3].into_array();
808+
builder.append_array_as_list(&arr1).unwrap();
809+
810+
// Interleave with a list scalar.
811+
builder
812+
.append_value(
813+
Scalar::fixed_size_list(
814+
dtype.clone(),
815+
vec![10i32.into(), 11i32.into(), 12i32.into()],
816+
NonNullable,
817+
)
818+
.as_list(),
819+
)
820+
.unwrap();
821+
822+
// Append another primitive array as a single list entry.
823+
let arr2 = buffer![4i32, 5, 6].into_array();
824+
builder.append_array_as_list(&arr2).unwrap();
825+
826+
// Interleave with another list scalar.
827+
builder
828+
.append_value(
829+
Scalar::fixed_size_list(
830+
dtype.clone(),
831+
vec![20i32.into(), 21i32.into(), 22i32.into()],
832+
NonNullable,
833+
)
834+
.as_list(),
835+
)
836+
.unwrap();
837+
838+
let fsl = builder.finish_into_fixed_size_list();
839+
assert_eq!(fsl.len(), 4);
840+
assert_eq!(fsl.list_size(), 3);
841+
842+
// Verify elements array: [1, 2, 3, 10, 11, 12, 4, 5, 6, 20, 21, 22].
843+
let elements = fsl.elements().to_primitive();
844+
assert_eq!(
845+
elements.as_slice::<i32>(),
846+
&[1, 2, 3, 10, 11, 12, 4, 5, 6, 20, 21, 22]
847+
);
848+
849+
// Test dtype mismatch error.
850+
let mut builder = FixedSizeListBuilder::with_capacity(dtype.clone(), 3, NonNullable, 10);
851+
let wrong_dtype_arr = buffer![1i64, 2, 3].into_array();
852+
assert!(builder.append_array_as_list(&wrong_dtype_arr).is_err());
853+
854+
// Test length mismatch error.
855+
let mut builder = FixedSizeListBuilder::with_capacity(dtype, 3, NonNullable, 10);
856+
let wrong_len_arr = buffer![1i32, 2].into_array();
857+
assert!(builder.append_array_as_list(&wrong_len_arr).is_err());
858+
}
773859
}

vortex-array/src/builders/list.rs

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,30 @@ impl<O: IntegerPType> ListBuilder<O> {
8787
}
8888
}
8989

90+
/// Appends an array as a single non-null list entry to the builder.
91+
///
92+
/// The input `array` must have the same dtype as the element dtype of this list builder.
93+
///
94+
/// Note that the list entry will be non-null but the elements themselves are allowed to be null
95+
/// (only if the elements [`DType`] in nullable, of course).
96+
pub fn append_array_as_list(&mut self, array: &dyn Array) -> VortexResult<()> {
97+
vortex_ensure!(
98+
array.dtype() == self.element_dtype(),
99+
"Array dtype {:?} does not match list element dtype {:?}",
100+
array.dtype(),
101+
self.element_dtype()
102+
);
103+
104+
self.elements_builder.extend_from_array(array);
105+
self.nulls.append_non_null();
106+
self.offsets_builder.append_value(
107+
O::from_usize(self.elements_builder.len())
108+
.vortex_expect("Failed to convert from usize to O"),
109+
);
110+
111+
Ok(())
112+
}
113+
90114
/// Appends a list `value` to the builder.
91115
pub fn append_value(&mut self, value: ListScalar) -> VortexResult<()> {
92116
match value.elements() {
@@ -547,4 +571,51 @@ mod tests {
547571
let wrong_scalar = Scalar::from(42i32);
548572
assert!(builder.append_scalar(&wrong_scalar).is_err());
549573
}
574+
575+
#[test]
576+
fn test_append_array_as_list() {
577+
let dtype: Arc<DType> = Arc::new(I32.into());
578+
let mut builder = ListBuilder::<u32>::with_capacity(dtype.clone(), NonNullable, 20, 10);
579+
580+
// Append a primitive array as a single list entry.
581+
let arr1 = buffer![1i32, 2, 3].into_array();
582+
builder.append_array_as_list(&arr1).unwrap();
583+
584+
// Interleave with a list scalar.
585+
builder
586+
.append_value(
587+
Scalar::list(dtype.clone(), vec![10i32.into(), 11i32.into()], NonNullable)
588+
.as_list(),
589+
)
590+
.unwrap();
591+
592+
// Append another primitive array as a single list entry.
593+
let arr2 = buffer![4i32, 5].into_array();
594+
builder.append_array_as_list(&arr2).unwrap();
595+
596+
// Append an empty array as a single list entry (empty list).
597+
let arr3 = buffer![0i32; 0].into_array();
598+
builder.append_array_as_list(&arr3).unwrap();
599+
600+
// Interleave with another list scalar (empty list).
601+
builder
602+
.append_value(Scalar::list_empty(dtype.clone(), NonNullable).as_list())
603+
.unwrap();
604+
605+
let list = builder.finish_into_list();
606+
assert_eq!(list.len(), 5);
607+
608+
// Verify elements array: [1, 2, 3, 10, 11, 4, 5].
609+
let elements = list.elements().to_primitive();
610+
assert_eq!(elements.as_slice::<i32>(), &[1, 2, 3, 10, 11, 4, 5]);
611+
612+
// Verify offsets array.
613+
let offsets = list.offsets().to_primitive();
614+
assert_eq!(offsets.as_slice::<u32>(), &[0, 3, 5, 7, 7, 7]);
615+
616+
// Test dtype mismatch error.
617+
let mut builder = ListBuilder::<u32>::with_capacity(dtype, NonNullable, 20, 10);
618+
let wrong_dtype_arr = buffer![1i64, 2, 3].into_array();
619+
assert!(builder.append_array_as_list(&wrong_dtype_arr).is_err());
620+
}
550621
}

vortex-array/src/builders/listview.rs

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,43 @@ impl<O: IntegerPType, S: IntegerPType> ListViewBuilder<O, S> {
110110
}
111111
}
112112

113+
/// Appends an array as a single non-null list entry to the builder.
114+
///
115+
/// The input `array` must have the same dtype as the element dtype of this list builder.
116+
///
117+
/// Note that the list entry will be non-null but the elements themselves are allowed to be null
118+
/// (only if the elements [`DType`] is nullable, of course).
119+
pub fn append_array_as_list(&mut self, array: &dyn Array) -> VortexResult<()> {
120+
vortex_ensure!(
121+
array.dtype() == self.element_dtype(),
122+
"Array dtype {:?} does not match list element dtype {:?}",
123+
array.dtype(),
124+
self.element_dtype()
125+
);
126+
127+
let curr_offset = self.elements_builder.len();
128+
let num_elements = array.len();
129+
130+
// We must assert this even in release mode to ensure that the safety comment in
131+
// `finish_into_listview` is correct.
132+
assert!(
133+
((curr_offset + num_elements) as u64) < O::max_value_as_u64(),
134+
"appending this list would cause an offset overflow"
135+
);
136+
137+
self.elements_builder.extend_from_array(array);
138+
self.nulls.append_non_null();
139+
140+
self.offsets_builder.append_value(
141+
O::from_usize(curr_offset).vortex_expect("Failed to convert from usize to `O`"),
142+
);
143+
self.sizes_builder.append_value(
144+
S::from_usize(num_elements).vortex_expect("Failed to convert from usize to `S`"),
145+
);
146+
147+
Ok(())
148+
}
149+
113150
/// Append a list of values to the builder.
114151
///
115152
/// This method extends the value builder with the provided values and records
@@ -633,4 +670,60 @@ mod tests {
633670
.contains("null value to non-nullable")
634671
);
635672
}
673+
674+
#[test]
675+
fn test_append_array_as_list() {
676+
use vortex_buffer::buffer;
677+
678+
use crate::ToCanonical;
679+
680+
let dtype: Arc<DType> = Arc::new(I32.into());
681+
let mut builder =
682+
ListViewBuilder::<u32, u32>::with_capacity(dtype.clone(), NonNullable, 20, 10);
683+
684+
// Append a primitive array as a single list entry.
685+
let arr1 = buffer![1i32, 2, 3].into_array();
686+
builder.append_array_as_list(&arr1).unwrap();
687+
688+
// Interleave with a list scalar.
689+
builder
690+
.append_value(
691+
Scalar::list(dtype.clone(), vec![10i32.into(), 11i32.into()], NonNullable)
692+
.as_list(),
693+
)
694+
.unwrap();
695+
696+
// Append another primitive array as a single list entry.
697+
let arr2 = buffer![4i32, 5].into_array();
698+
builder.append_array_as_list(&arr2).unwrap();
699+
700+
// Append an empty array as a single list entry (empty list).
701+
let arr3 = buffer![0i32; 0].into_array();
702+
builder.append_array_as_list(&arr3).unwrap();
703+
704+
// Interleave with another list scalar.
705+
builder
706+
.append_value(Scalar::list_empty(dtype.clone(), NonNullable).as_list())
707+
.unwrap();
708+
709+
let listview = builder.finish_into_listview();
710+
assert_eq!(listview.len(), 5);
711+
712+
// Verify elements array: [1, 2, 3, 10, 11, 4, 5].
713+
let elements = listview.elements().to_primitive();
714+
assert_eq!(elements.as_slice::<i32>(), &[1, 2, 3, 10, 11, 4, 5]);
715+
716+
// Verify offsets array.
717+
let offsets = listview.offsets().to_primitive();
718+
assert_eq!(offsets.as_slice::<u32>(), &[0, 3, 5, 7, 7]);
719+
720+
// Verify sizes array.
721+
let sizes = listview.sizes().to_primitive();
722+
assert_eq!(sizes.as_slice::<u32>(), &[3, 2, 2, 0, 0]);
723+
724+
// Test dtype mismatch error.
725+
let mut builder = ListViewBuilder::<u32, u32>::with_capacity(dtype, NonNullable, 20, 10);
726+
let wrong_dtype_arr = buffer![1i64, 2, 3].into_array();
727+
assert!(builder.append_array_as_list(&wrong_dtype_arr).is_err());
728+
}
636729
}

vortex-array/src/executor.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,6 @@ impl VectorExecutor for ArrayRef {
5555

5656
fn execute_vector(&self, session: &VortexSession) -> VortexResult<Vector> {
5757
let len = self.len();
58-
Ok(self.execute_datum(session)?.ensure_vector(len))
58+
Ok(self.execute_datum(session)?.unwrap_into_vector(len))
5959
}
6060
}

0 commit comments

Comments
 (0)