Merge branch 'develop' into operators-bench

Author: 0ax1

encodings/alp/src/alp/array.rs

@@ -432,7 +432,7 @@ impl ALPArray {
 }
 
 impl ValidityChild<ALPVTable> for ALPVTable {
-    fn validity_child(array: &ALPArray) -> &dyn Array {
+    fn validity_child(array: &ALPArray) -> &ArrayRef {
         array.encoded()
     }
 }

encodings/alp/src/alp_rd/array.rs

@@ -368,7 +368,7 @@ impl ALPRDArray {
 }
 
 impl ValidityChild<ALPRDVTable> for ALPRDVTable {
-    fn validity_child(array: &ALPRDArray) -> &dyn Array {
+    fn validity_child(array: &ALPRDArray) -> &ArrayRef {
         array.left_parts()
     }
 }

encodings/datetime-parts/src/array.rs

@@ -288,7 +288,7 @@ impl BaseArrayVTable<DateTimePartsVTable> for DateTimePartsVTable {
 }
 
 impl ValidityChild<DateTimePartsVTable> for DateTimePartsVTable {
-    fn validity_child(array: &DateTimePartsArray) -> &dyn Array {
+    fn validity_child(array: &DateTimePartsArray) -> &ArrayRef {
         array.days()
     }
 }

encodings/datetime-parts/src/compute/rules.rs

@@ -1,14 +1,14 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+use vortex_array::Array;
 use vortex_array::ArrayRef;
 use vortex_array::IntoArray;
 use vortex_array::arrays::ConstantArray;
 use vortex_array::arrays::ConstantVTable;
 use vortex_array::arrays::FilterArray;
 use vortex_array::arrays::FilterVTable;
 use vortex_array::matchers::Exact;
-use vortex_array::optimizer::ArrayOptimizer;
 use vortex_array::optimizer::rules::ArrayParentReduceRule;
 use vortex_array::optimizer::rules::ParentRuleSet;
 use vortex_error::VortexExpect;
@@ -38,19 +38,15 @@ impl ArrayParentReduceRule<DateTimePartsVTable> for DTPFilterPushDownRule {
         parent: &FilterArray,
         child_idx: usize,
     ) -> VortexResult<Option<ArrayRef>> {
-        if child_idx != 0 {
-            return Ok(None);
-        }
+        debug_assert_eq!(child_idx, 0);
 
         if !child.seconds().is::<ConstantVTable>() || !child.subseconds().is::<ConstantVTable>() {
             return Ok(None);
         }
 
         DateTimePartsArray::try_new(
             child.dtype().clone(),
-            FilterArray::new(child.days().clone(), parent.filter_mask().clone())
-                .into_array()
-                .optimize()?,
+            child.days().clone().filter(parent.filter_mask().clone())?,
             ConstantArray::new(
                 child.seconds().as_constant().vortex_expect("constant"),
                 parent.filter_mask().true_count(),

encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs

@@ -269,9 +269,9 @@ impl OperationsVTable<DecimalBytePartsVTable> for DecimalBytePartsVTable {
 }
 
 impl ValidityChild<DecimalBytePartsVTable> for DecimalBytePartsVTable {
-    fn validity_child(array: &DecimalBytePartsArray) -> &dyn Array {
+    fn validity_child(array: &DecimalBytePartsArray) -> &ArrayRef {
         // validity stored in 0th child
-        array.msp.as_ref()
+        &array.msp
     }
 }
 

encodings/fastlanes/src/for/vtable/validity.rs

@@ -1,14 +1,14 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-use vortex_array::Array;
+use vortex_array::ArrayRef;
 use vortex_array::vtable::ValidityChild;
 
 use super::FoRVTable;
 use crate::FoRArray;
 
 impl ValidityChild<FoRVTable> for FoRVTable {
-    fn validity_child(array: &FoRArray) -> &dyn Array {
-        array.encoded().as_ref()
+    fn validity_child(array: &FoRArray) -> &ArrayRef {
+        array.encoded()
     }
 }

encodings/fastlanes/src/rle/vtable/validity.rs

@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-use vortex_array::Array;
 use vortex_array::ArrayRef;
 use vortex_array::vtable::ValidityChild;
 use vortex_array::vtable::ValidityChildSliceHelper;
@@ -10,8 +9,8 @@ use super::RLEVTable;
 use crate::RLEArray;
 
 impl ValidityChild<RLEVTable> for RLEVTable {
-    fn validity_child(array: &RLEArray) -> &dyn Array {
-        array.indices().as_ref()
+    fn validity_child(array: &RLEArray) -> &ArrayRef {
+        array.indices()
     }
 }
 

encodings/fsst/src/array.rs

@@ -198,6 +198,8 @@ pub struct FSSTArray {
     symbols: Buffer<Symbol>,
     symbol_lengths: Buffer<u8>,
     codes: VarBinArray,
+    /// NOTE(ngates): this === codes, but is stored as an ArrayRef so we can return &ArrayRef!
+    codes_array: ArrayRef,
     /// Lengths of the original values before compression, can be compressed.
     uncompressed_lengths: ArrayRef,
     stats_set: ArrayStats,
@@ -283,12 +285,14 @@ impl FSSTArray {
             Compressor::rebuild_from(symbols2.as_slice(), symbol_lengths2.as_slice())
         })
             as Box<dyn Fn() -> Compressor + Send>));
+        let codes_array = codes.to_array();
 
         Self {
             dtype,
             symbols,
             symbol_lengths,
             codes,
+            codes_array,
             uncompressed_lengths,
             stats_set: Default::default(),
             compressor,
@@ -329,8 +333,6 @@ impl FSSTArray {
 
     /// Build a [`Decompressor`][fsst::Decompressor] that can be used to decompress values from
     /// this array.
-    ///
-    /// This is private to the crate to avoid leaking `fsst-rs` types as part of the public API.
     pub fn decompressor(&self) -> Decompressor<'_> {
         Decompressor::new(self.symbols().as_slice(), self.symbol_lengths().as_slice())
     }
@@ -379,8 +381,8 @@ impl BaseArrayVTable<FSSTVTable> for FSSTVTable {
 }
 
 impl ValidityChild<FSSTVTable> for FSSTVTable {
-    fn validity_child(array: &FSSTArray) -> &dyn Array {
-        array.codes().as_ref()
+    fn validity_child(array: &FSSTArray) -> &ArrayRef {
+        &array.codes_array
     }
 }
 

encodings/runend/src/array.rs

@@ -21,6 +21,7 @@ use vortex_array::search_sorted::SearchSortedSide;
 use vortex_array::serde::ArrayChildren;
 use vortex_array::stats::ArrayStats;
 use vortex_array::stats::StatsSetRef;
+use vortex_array::validity::Validity;
 use vortex_array::vtable;
 use vortex_array::vtable::ArrayId;
 use vortex_array::vtable::ArrayVTable;
@@ -427,6 +428,22 @@ impl ValidityVTable<RunEndVTable> for RunEndVTable {
         array.values().all_invalid()
     }
 
+    fn validity(array: &RunEndArray) -> VortexResult<Validity> {
+        Ok(match array.values().validity()? {
+            Validity::NonNullable | Validity::AllValid => Validity::AllValid,
+            Validity::AllInvalid => Validity::AllInvalid,
+            Validity::Array(values_validity) => Validity::Array(unsafe {
+                RunEndArray::new_unchecked(
+                    array.ends().clone(),
+                    values_validity,
+                    array.offset(),
+                    array.len(),
+                )
+                .into_array()
+            }),
+        })
+    }
+
     fn validity_mask(array: &RunEndArray) -> Mask {
         match array.values().validity_mask() {
             Mask::AllTrue(_) => Mask::AllTrue(array.len()),

encodings/sequence/src/array.rs

@@ -19,6 +19,7 @@ use vortex_array::arrays::PrimitiveArray;
 use vortex_array::serde::ArrayChildren;
 use vortex_array::stats::ArrayStats;
 use vortex_array::stats::StatsSetRef;
+use vortex_array::validity::Validity;
 use vortex_array::vtable;
 use vortex_array::vtable::ArrayId;
 use vortex_array::vtable::ArrayVTable;
@@ -413,6 +414,10 @@ impl ValidityVTable<SequenceVTable> for SequenceVTable {
         false
     }
 
+    fn validity(_array: &SequenceArray) -> VortexResult<Validity> {
+        Ok(Validity::AllValid)
+    }
+
     fn validity_mask(array: &SequenceArray) -> Mask {
         Mask::AllTrue(array.len())
     }