merge

Signed-off-by: Nicholas Gates <[email protected]>

Author: gatesn

encodings/alp/src/alp/operator.rs

@@ -1,16 +1,18 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-use crate::{match_each_alp_float_ptype, ALPArray, ALPFloat, ALPVTable, Exponents};
 use std::marker::PhantomData;
+
 use vortex_array::pipeline::{BindContext, PipelineTransform, TransformKernel};
 use vortex_array::vtable::{OperatorVTable, PipelineNode};
 use vortex_buffer::Buffer;
-use vortex_dtype::{match_each_integer_ptype, NativePType, PTypeDowncastExt};
+use vortex_dtype::{NativePType, PTypeDowncastExt, match_each_integer_ptype};
 use vortex_error::VortexResult;
 use vortex_vector::primitive::PVector;
 use vortex_vector::{Vector, VectorMut};
 
+use crate::{ALPArray, ALPFloat, ALPVTable, Exponents, match_each_alp_float_ptype};
+
 impl OperatorVTable<ALPVTable> for ALPVTable {
     fn pipeline_node(array: &ALPArray) -> Option<PipelineNode<'_>> {
         Some(PipelineNode::Transform(array))
@@ -40,7 +42,7 @@ impl PipelineTransform for ALPArray {
 
                 match_each_alp_float_ptype!(self.ptype(), |A| {
                     match_each_integer_ptype!(patches.indices_ptype(), |P| {
-                        let patch_indices: Buffer<P> = P::downcast(patch_idxs).into_buffer();
+                        let patch_indices: Buffer<P> = patch_idxs.downcast::<P>().into_elements();
                         let patch_values: PVector<A> = A::downcast(patch_vals);
                         Ok(Box::new(PatchedALPKernel {
                             exponents,
@@ -61,10 +63,10 @@ struct ALPKernel<A: ALPFloat> {
 }
 
 impl<A: ALPFloat> TransformKernel for ALPKernel<A> {
-    fn step(&mut self, input: &VectorMut, out: &mut VectorMut) -> VortexResult<()> {
-        let encoded = input.into_primitive().downcast::<A::ALPInt>().into_buffer();
+    fn step(&mut self, input: &Vector, out: &mut VectorMut) -> VortexResult<()> {
+        let encoded = input.as_primitive().downcast::<A::ALPInt>().elements();
 
-        let mut decoded = A::downcast(out.into_primitive());
+        let decoded = out.as_primitive_mut().downcast::<A>();
         decoded.extend(
             encoded
                 .iter()
@@ -74,6 +76,7 @@ impl<A: ALPFloat> TransformKernel for ALPKernel<A> {
     }
 }
 
+#[allow(dead_code)] // TODO(ngates): implement patching
 struct PatchedALPKernel<A: ALPFloat, P: NativePType> {
     // The ALP exponents
     exponents: Exponents,
@@ -84,9 +87,9 @@ struct PatchedALPKernel<A: ALPFloat, P: NativePType> {
 
 impl<A: ALPFloat, P: NativePType> TransformKernel for PatchedALPKernel<A, P> {
     fn step(&mut self, input: &Vector, out: &mut VectorMut) -> VortexResult<()> {
-        let encoded = input.into_primitive().downcast::<A::ALPInt>().into_buffer();
+        let encoded = input.as_primitive().downcast::<A::ALPInt>().elements();
 
-        let mut decoded = out.into_primitive().downcast::<A>();
+        let decoded = out.as_primitive_mut().downcast::<A>();
         decoded.extend(
             encoded
                 .iter()

encodings/sequence/src/array.rs

@@ -160,7 +160,7 @@ impl VTable for SequenceVTable {
     type ComputeVTable = NotSupported;
     type EncodeVTable = Self;
     type SerdeVTable = Self;
-    type OperatorVTable = Self;
+    type OperatorVTable = NotSupported;
 
     fn id(_encoding: &Self::Encoding) -> EncodingId {
         EncodingId::new_ref("vortex.sequence")

encodings/sequence/src/lib.rs

@@ -4,7 +4,7 @@
 mod array;
 mod compress;
 mod compute;
-mod operator;
+// mod operator;
 mod serde;
 
 /// Represents the equation A\[i\] = a * i + b.

vortex-array/src/array/operator.rs

@@ -4,7 +4,6 @@
 use std::sync::Arc;
 
 use vortex_error::{VortexResult, vortex_panic};
-use vortex_mask::Mask;
 use vortex_vector::{Vector, VectorOps, vector_matches_dtype};
 
 use crate::execution::{BatchKernelRef, BindCtx, DummyExecutionCtx, ExecutionCtx};
@@ -17,13 +16,12 @@ use crate::{Array, ArrayAdapter, ArrayRef};
 /// Note: the public functions such as "execute" should move onto the main `Array` trait when
 /// operators is stabilized. The other functions should remain on a `pub(crate)` trait.
 pub trait ArrayOperator: 'static + Send + Sync {
-    /// Execute the array's batch kernel with the given selection mask.
+    /// Execute the array's batch kernel.
     ///
     /// # Panics
     ///
-    /// If the mask length does not match the array length.
     /// If the array's implementation returns an invalid vector (wrong length, wrong type, etc.).
-    fn execute_batch(&self, selection: &Mask, ctx: &mut dyn ExecutionCtx) -> VortexResult<Vector>;
+    fn execute_batch(&self, ctx: &mut dyn ExecutionCtx) -> VortexResult<Vector>;
 
     /// Optimize the array by running the optimization rules.
     fn reduce_children(&self) -> VortexResult<Option<ArrayRef>>;
@@ -40,8 +38,8 @@ pub trait ArrayOperator: 'static + Send + Sync {
 }
 
 impl ArrayOperator for Arc<dyn Array> {
-    fn execute_batch(&self, selection: &Mask, ctx: &mut dyn ExecutionCtx) -> VortexResult<Vector> {
-        self.as_ref().execute_batch(selection, ctx)
+    fn execute_batch(&self, ctx: &mut dyn ExecutionCtx) -> VortexResult<Vector> {
+        self.as_ref().execute_batch(ctx)
     }
 
     fn reduce_children(&self) -> VortexResult<Option<ArrayRef>> {
@@ -62,23 +60,22 @@ impl ArrayOperator for Arc<dyn Array> {
 }
 
 impl<V: VTable> ArrayOperator for ArrayAdapter<V> {
-    fn execute_batch(&self, selection: &Mask, ctx: &mut dyn ExecutionCtx) -> VortexResult<Vector> {
+    fn execute_batch(&self, ctx: &mut dyn ExecutionCtx) -> VortexResult<Vector> {
         // Check if the array is a pipeline node
         if let Some(pipeline_node) =
             <V::OperatorVTable as OperatorVTable<V>>::pipeline_node(&self.0)
             && let PipelineNode::Source(source) = pipeline_node
         {
-            return PipelineSourceDriver::new(source).execute(selection);
+            return PipelineSourceDriver::new(source).execute();
         }
 
-        let vector =
-            <V::OperatorVTable as OperatorVTable<V>>::execute_batch(&self.0, selection, ctx)?;
+        let vector = <V::OperatorVTable as OperatorVTable<V>>::execute_batch(&self.0, ctx)?;
 
         // Such a cheap check that we run it always. More expensive DType checks live in
         // debug_assertions.
         assert_eq!(
             vector.len(),
-            selection.true_count(),
+            self.len(),
             "Batch execution returned vector of incorrect length"
         );
 
@@ -126,11 +123,6 @@ impl BindCtx for () {
 
 impl dyn Array + '_ {
     pub fn execute(&self) -> VortexResult<Vector> {
-        self.execute_batch(&Mask::new_true(self.len()), &mut DummyExecutionCtx)
-    }
-
-    pub fn execute_with_selection(&self, mask: &Mask) -> VortexResult<Vector> {
-        assert_eq!(self.len(), mask.len());
-        self.execute_batch(mask, &mut DummyExecutionCtx)
+        self.execute_batch(&mut DummyExecutionCtx)
     }
 }

vortex-array/src/arrays/listview/vtable/operator.rs

@@ -47,12 +47,7 @@ mod tests {
     use vortex_mask::Mask;
     use vortex_vector::VectorOps;
 
-    use crate::IntoArray;
-    use crate::arrays::listview::tests::common::{
-        create_basic_listview, create_nullable_listview, create_overlapping_listview,
-    };
-    use crate::arrays::{ListViewArray, PrimitiveArray};
-    use crate::validity::Validity;
+    use crate::arrays::listview::tests::common::create_basic_listview;
 
     #[test]
     fn test_listview_operator_basic() {
@@ -85,106 +80,4 @@ mod tests {
         // Verify validity is all valid.
         assert!(matches!(listview_vector.validity(), Mask::AllTrue(_)));
     }
-
-    #[test]
-    fn test_listview_operator_with_selection() {
-        // Create a ListView with 6 lists: [[10,11], [20,21], [30,31], [40,41], [50,51], [60,61]]
-        let elements =
-            PrimitiveArray::from_iter([10i32, 11, 20, 21, 30, 31, 40, 41, 50, 51, 60, 61])
-                .into_array();
-        let offsets = PrimitiveArray::from_iter([0u32, 2, 4, 6, 8, 10]).into_array();
-        let sizes = PrimitiveArray::from_iter([2u32, 2, 2, 2, 2, 2]).into_array();
-        let listview = ListViewArray::new(elements, offsets, sizes, Validity::AllValid);
-
-        // Create selection mask: [true, false, true, false, true, false].
-        let selection = Mask::from_iter([true, false, true, false, true, false]);
-
-        // Execute with selection.
-        let result = listview.execute_with_selection(&selection).unwrap();
-
-        // Verify filtered length (3 lists selected).
-        assert_eq!(result.len(), 3);
-
-        let listview_vector = result.as_list();
-
-        // Verify offsets are filtered to indices 0, 2, 4.
-        let offsets = listview_vector.offsets().clone().into_u32();
-        assert_eq!(offsets.elements().as_slice(), &[0, 4, 8]);
-
-        // Verify sizes are filtered to indices 0, 2, 4.
-        let sizes = listview_vector.sizes().clone().into_u32();
-        assert_eq!(sizes.elements().as_slice(), &[2, 2, 2]);
-
-        // Verify elements remain complete (not filtered).
-        let elements = listview_vector.elements().as_primitive().clone().into_i32();
-        assert_eq!(
-            elements.elements().as_slice(),
-            &[10, 11, 20, 21, 30, 31, 40, 41, 50, 51, 60, 61]
-        );
-    }
-
-    #[test]
-    fn test_listview_operator_with_nulls_and_selection() {
-        // Use the nullable listview: [[10,20], null, [50]]
-        let listview = create_nullable_listview();
-
-        // Create selection mask: [true, true, false].
-        let selection = Mask::from_iter([true, true, false]);
-
-        // Execute with selection.
-        let result = listview.execute_with_selection(&selection).unwrap();
-
-        // Verify filtered length (2 lists selected, including the null).
-        assert_eq!(result.len(), 2);
-
-        let listview_vector = result.as_list();
-
-        // Verify offsets are filtered to indices 0 and 1.
-        let offsets = listview_vector.offsets().clone().into_u32();
-        assert_eq!(offsets.elements().as_slice(), &[0, 2]);
-
-        // Verify sizes are filtered to indices 0 and 1.
-        let sizes = listview_vector.sizes().clone().into_u32();
-        assert_eq!(sizes.elements().as_slice(), &[2, 2]);
-
-        // Verify validity mask correctly shows first list valid, second list null.
-        assert!(listview_vector.validity().value(0)); // First list is valid.
-        assert!(!listview_vector.validity().value(1)); // Second list is null.
-
-        // Verify elements remain complete.
-        let elements = listview_vector.elements().as_primitive().clone().into_i32();
-        assert_eq!(elements.elements().as_slice(), &[10, 20, 30, 40, 50]);
-    }
-
-    #[test]
-    fn test_listview_operator_overlapping_with_selection() {
-        // Use the overlapping listview: [[5,6,7], [2,3], [8,9], [0,1], [1,2,3,4]]
-        let listview = create_overlapping_listview();
-
-        // Create selection mask: [true, false, true, true, false].
-        let selection = Mask::from_iter([true, false, true, true, false]);
-
-        // Execute with selection.
-        let result = listview.execute_with_selection(&selection).unwrap();
-
-        // Verify filtered length (3 lists selected).
-        assert_eq!(result.len(), 3);
-
-        let listview_vector = result.as_list();
-
-        // Verify offsets are filtered to indices 0, 2, 3 (out-of-order preserved).
-        let offsets = listview_vector.offsets().clone().into_u32();
-        assert_eq!(offsets.elements().as_slice(), &[5, 8, 0]);
-
-        // Verify sizes are filtered.
-        let sizes = listview_vector.sizes().clone().into_u32();
-        assert_eq!(sizes.elements().as_slice(), &[3, 2, 2]);
-
-        // Verify elements remain complete (all 10 elements).
-        let elements = listview_vector.elements().as_primitive().clone().into_i32();
-        assert_eq!(
-            elements.elements().as_slice(),
-            &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
-        );
-    }
 }

vortex-array/src/arrays/struct_/vtable/operator.rs

@@ -41,8 +41,8 @@ impl OperatorVTable<StructVTable> for StructVTable {
 
 #[cfg(test)]
 mod tests {
+    use vortex_buffer::bitbuffer;
     use vortex_dtype::{FieldNames, PTypeDowncast};
-    use vortex_mask::Mask;
     use vortex_vector::VectorOps;
 
     use crate::IntoArray;
@@ -82,44 +82,6 @@ mod tests {
         assert_eq!(bool_values, vec![true, false, true, false, true]);
     }
 
-    #[test]
-    fn test_struct_operator_with_mask() {
-        // Create a struct array with two fields.
-        let int_field = PrimitiveArray::from_iter([10i32, 20, 30, 40, 50, 60]);
-        let bool_field = BoolArray::from_iter([true, false, true, false, true, false]);
-
-        let struct_array = StructArray::try_new(
-            FieldNames::from(["numbers", "flags"]),
-            vec![int_field.into_array(), bool_field.into_array()],
-            6,
-            Validity::AllValid,
-        )
-        .unwrap();
-
-        // Create a selection mask that selects indices 0, 2, 4 (alternating pattern).
-        let selection = Mask::from_iter([true, false, true, false, true, false]);
-
-        // Execute with selection mask.
-        let result = struct_array.execute_with_selection(&selection).unwrap();
-
-        // Verify the result has the filtered length.
-        assert_eq!(result.len(), 3);
-
-        // Verify the struct vector fields.
-        let struct_vector = result.as_struct();
-        let fields = struct_vector.fields();
-        assert_eq!(fields.len(), 2);
-
-        // Verify the integer field has the correct filtered values (indices 0, 2, 4).
-        let int_vector = fields[0].as_primitive().clone().into_i32();
-        assert_eq!(int_vector.elements().as_slice(), &[10, 30, 50]);
-
-        // Verify the boolean field has the correct filtered values (indices 0, 2, 4).
-        let bool_vector = fields[1].as_bool();
-        let bool_values: Vec<bool> = (0..3).map(|i| bool_vector.bits().value(i)).collect();
-        assert_eq!(bool_values, vec![true, true, true]);
-    }
-
     #[test]
     fn test_struct_operator_null_handling() {
         // Create fields with nulls.
@@ -148,13 +110,9 @@ mod tests {
         )
         .unwrap();
 
-        // Create a selection mask that selects indices 0, 1, 2, 4, 5.
-        let selection = Mask::from_iter([true, true, true, false, true, true]);
-
-        // Execute with selection mask.
-        let result = struct_array.execute_with_selection(&selection).unwrap();
+        let result = struct_array.execute().unwrap();
 
-        assert_eq!(result.len(), 5);
+        assert_eq!(result.len(), 6);
 
         // Verify the struct vector fields.
         let struct_vector = result.as_struct();
@@ -164,23 +122,19 @@ mod tests {
         // Verify integer field has the correct filtered values with nulls.
         // Selected indices: 0, 1, 2, 4, 5 from [Some(100), None, Some(200), Some(300), None, Some(400)].
         let int_vector = fields[0].as_primitive().clone().into_i32();
-        let int_values: Vec<Option<i32>> = (0..5).map(|i| int_vector.get(i).copied()).collect();
+        let int_values: Vec<Option<i32>> = (0..6).map(|i| int_vector.get(i).copied()).collect();
         assert_eq!(
             int_values,
-            vec![Some(100), None, Some(200), None, Some(400)]
+            vec![Some(100), None, Some(200), Some(300), None, Some(400)]
         );
 
-        // Verify boolean field values.
-        // Selected indices: 0, 1, 2, 4, 5 from [T, F, T, F, T, F].
+        // Verify boolean field values from [T, F, T, F, T, F].
         let bool_vector = fields[1].as_bool();
-        let bool_values: Vec<bool> = (0..5).map(|i| bool_vector.bits().value(i)).collect();
-        assert_eq!(bool_values, vec![true, false, true, true, false]);
+        assert_eq!(bool_vector.bits(), &bitbuffer![1 0 1 0 1 0]);
 
         // Verify the struct-level validity is correctly propagated.
         // Original struct validity: [T, F, T, T, F, T]
-        // Selected indices: 0, 1, 2, 4, 5 -> validity: [T, F, T, F, T].
         let validity_mask = struct_vector.validity();
-        let struct_validity_values: Vec<bool> = (0..5).map(|i| validity_mask.value(i)).collect();
-        assert_eq!(struct_validity_values, vec![true, false, true, false, true]);
+        assert_eq!(validity_mask.to_bit_buffer(), bitbuffer![1 0 1 1 0 1]);
     }
 }