Add datums

Signed-off-by: Nicholas Gates <[email protected]>

Author: gatesn

encodings/fastlanes/benches/pipeline_bitpacking_compare_scalar.rs

@@ -13,7 +13,7 @@ use vortex_array::{Array, ArrayRef, IntoArray, ToCanonical};
 use vortex_buffer::{BitBuffer, BufferMut};
 use vortex_dtype::NativePType;
 use vortex_error::VortexResult;
-use vortex_expr::{Scope, lit, lt, root};
+use vortex_expr::{lit, lt, root};
 use vortex_fastlanes::FoRArray;
 use vortex_fastlanes::bitpack_compress::bitpack_to_best_bit_width;
 use vortex_mask::Mask;
@@ -68,7 +68,7 @@ pub fn eval<T: NativePType + Into<Scalar>>(bencher: Bencher, fraction_kept: f64)
         .bench_local_values(|(mask, array)| {
             // We run the filter first, then compare.
             let array = filter(array.as_ref(), &mask).unwrap();
-            expr.evaluate(&Scope::new(array)).unwrap().to_canonical()
+            expr.evaluate(&array).unwrap().to_canonical()
         });
 }
 

encodings/sequence/src/compute/min_max.rs

@@ -3,6 +3,7 @@
 
 use vortex_array::compute::{MinMaxKernel, MinMaxKernelAdapter, MinMaxResult};
 use vortex_array::register_kernel;
+use vortex_dtype::Nullability::NonNullable;
 use vortex_error::VortexResult;
 use vortex_scalar::Scalar;
 
@@ -19,8 +20,8 @@ impl MinMaxKernel for SequenceVTable {
             (last, base)
         };
         Ok(Some(MinMaxResult {
-            min: Scalar::new(array.dtype().clone(), min.into()),
-            max: Scalar::new(array.dtype().clone(), max.into()),
+            min: Scalar::primitive_value(min, array.ptype(), NonNullable),
+            max: Scalar::primitive_value(max, array.ptype(), NonNullable),
         }))
     }
 }

fuzz/fuzz_targets/array_ops.rs

@@ -8,7 +8,7 @@ use std::backtrace::Backtrace;
 
 use libfuzzer_sys::{Corpus, fuzz_target};
 use vortex_array::arrays::ConstantArray;
-use vortex_array::compute::{cast, compare, fill_null, filter, min_max, sum, take};
+use vortex_array::compute::{cast, compare, fill_null, filter, mask, min_max, sum, take};
 use vortex_array::search_sorted::{SearchResult, SearchSorted, SearchSortedSide};
 use vortex_array::{Array, ArrayRef, IntoArray};
 use vortex_btrblocks::BtrBlocksCompressor;
@@ -99,7 +99,6 @@ fuzz_target!(|fuzz_action: FuzzArrayAction| -> Corpus {
                 assert_array_eq(&expected.array(), &current_array, i).unwrap();
             }
             Action::Mask(mask_val) => {
-                use vortex_array::compute::mask;
                 current_array = mask(&current_array, &mask_val).vortex_unwrap();
                 assert_array_eq(&expected.array(), &current_array, i).unwrap();
             }

fuzz/fuzz_targets/file_io.rs

@@ -12,7 +12,7 @@ use vortex_array::{Array, Canonical, IntoArray, ToCanonical};
 use vortex_buffer::ByteBufferMut;
 use vortex_dtype::{DType, StructFields};
 use vortex_error::{VortexExpect, VortexUnwrap, vortex_panic};
-use vortex_expr::{Scope, lit, root};
+use vortex_expr::{lit, root};
 use vortex_file::{OpenOptionsSessionExt, WriteOptionsSessionExt, WriteStrategyBuilder};
 use vortex_fuzz::{CompressorStrategy, FuzzFileAction, RUNTIME, SESSION};
 use vortex_layout::layouts::compact::CompactCompressor;
@@ -36,14 +36,14 @@ fuzz_target!(|fuzz: FuzzFileAction| -> Corpus {
         let bool_mask = filter_expr
             .clone()
             .unwrap_or_else(|| lit(true))
-            .evaluate(&Scope::new(array_data.clone()))
+            .evaluate(&array_data)
             .vortex_unwrap();
         let mask = bool_mask.to_bool().to_mask_fill_null_false();
         let filtered = filter(&array_data, &mask).vortex_unwrap();
         projection_expr
             .clone()
             .unwrap_or_else(root)
-            .evaluate(&Scope::new(filtered))
+            .evaluate(&filtered)
             .vortex_unwrap()
     };
 

vortex-array/src/arrays/bool/compute/min_max.rs

@@ -3,6 +3,8 @@
 
 use std::ops::BitAnd;
 
+use Nullability::NonNullable;
+use vortex_dtype::Nullability;
 use vortex_error::VortexResult;
 use vortex_mask::Mask;
 use vortex_scalar::Scalar;
@@ -30,15 +32,15 @@ impl MinMaxKernel for BoolVTable {
         let Some(slice) = true_slices.next() else {
             // all false
             return Ok(Some(MinMaxResult {
-                min: Scalar::bool(false, array.dtype().nullability()),
-                max: Scalar::bool(false, array.dtype().nullability()),
+                min: Scalar::bool(false, NonNullable),
+                max: Scalar::bool(false, NonNullable),
             }));
         };
         if slice.0 == 0 && slice.1 == array.len() {
             // all true
             return Ok(Some(MinMaxResult {
-                min: Scalar::bool(true, array.dtype().nullability()),
-                max: Scalar::bool(true, array.dtype().nullability()),
+                min: Scalar::bool(true, NonNullable),
+                max: Scalar::bool(true, NonNullable),
             }));
         };
 
@@ -53,16 +55,16 @@ impl MinMaxKernel for BoolVTable {
                 let Some(_) = false_slices.next() else {
                     // In this case we don't have any false values which means we are all true and null
                     return Ok(Some(MinMaxResult {
-                        min: Scalar::bool(true, array.dtype().nullability()),
-                        max: Scalar::bool(true, array.dtype().nullability()),
+                        min: Scalar::bool(true, NonNullable),
+                        max: Scalar::bool(true, NonNullable),
                     }));
                 };
             }
         }
 
         Ok(Some(MinMaxResult {
-            min: Scalar::bool(false, array.dtype().nullability()),
-            max: Scalar::bool(true, array.dtype().nullability()),
+            min: Scalar::bool(false, NonNullable),
+            max: Scalar::bool(true, NonNullable),
         }))
     }
 }
@@ -71,47 +73,47 @@ register_kernel!(MinMaxKernelAdapter(BoolVTable).lift());
 
 #[cfg(test)]
 mod tests {
-    use vortex_dtype::{DType, Nullability};
+    use Nullability::NonNullable;
+    use vortex_dtype::Nullability;
     use vortex_scalar::Scalar;
 
     use crate::arrays::BoolArray;
     use crate::compute::{MinMaxResult, min_max};
 
     #[test]
     fn test_min_max_nulls() {
-        let dtype = DType::Bool(Nullability::Nullable);
         assert_eq!(
             min_max(BoolArray::from_iter(vec![Some(true), Some(true), None, None]).as_ref())
                 .unwrap(),
             Some(MinMaxResult {
-                min: Scalar::new(dtype.clone(), true.into()),
-                max: Scalar::new(dtype.clone(), true.into()),
+                min: Scalar::bool(true, NonNullable),
+                max: Scalar::bool(true, NonNullable),
             })
         );
 
         assert_eq!(
             min_max(BoolArray::from_iter(vec![None, Some(true), Some(true)]).as_ref()).unwrap(),
             Some(MinMaxResult {
-                min: Scalar::new(dtype.clone(), true.into()),
-                max: Scalar::new(dtype.clone(), true.into()),
+                min: Scalar::bool(true, NonNullable),
+                max: Scalar::bool(true, NonNullable),
             })
         );
 
         assert_eq!(
             min_max(BoolArray::from_iter(vec![None, Some(true), Some(true), None]).as_ref())
                 .unwrap(),
             Some(MinMaxResult {
-                min: Scalar::new(dtype.clone(), true.into()),
-                max: Scalar::new(dtype.clone(), true.into()),
+                min: Scalar::bool(true, NonNullable),
+                max: Scalar::bool(true, NonNullable),
             })
         );
 
         assert_eq!(
             min_max(BoolArray::from_iter(vec![Some(false), Some(false), None, None]).as_ref())
                 .unwrap(),
             Some(MinMaxResult {
-                min: Scalar::new(dtype.clone(), false.into()),
-                max: Scalar::new(dtype, false.into()),
+                min: Scalar::bool(false, NonNullable),
+                max: Scalar::bool(false, NonNullable),
             })
         );
     }

vortex-array/src/arrays/constant/compute/min_max.rs

@@ -10,7 +10,7 @@ use crate::register_kernel;
 impl MinMaxKernel for ConstantVTable {
     fn min_max(&self, array: &ConstantArray) -> VortexResult<Option<MinMaxResult>> {
         let scalar = array.scalar();
-        if scalar.is_null() {
+        if scalar.is_null() || scalar.as_primitive_opt().is_some_and(|p| p.is_nan()) {
             return Ok(None);
         }
         let non_nullable_dtype = scalar.dtype().as_nonnullable();
@@ -22,3 +22,21 @@ impl MinMaxKernel for ConstantVTable {
 }
 
 register_kernel!(MinMaxKernelAdapter(ConstantVTable).lift());
+
+#[cfg(test)]
+mod test {
+    use vortex_dtype::Nullability;
+    use vortex_dtype::half::f16;
+    use vortex_scalar::Scalar;
+
+    use crate::arrays::ConstantArray;
+    use crate::compute::min_max;
+
+    #[test]
+    fn test_min_max_nan() {
+        let scalar = Scalar::primitive(f16::NAN, Nullability::NonNullable);
+        let array = ConstantArray::new(scalar, 2).to_array();
+        let result = min_max(&array).unwrap();
+        assert_eq!(result, None);
+    }
+}

vortex-array/src/arrays/decimal/compute/min_max.rs

@@ -2,10 +2,11 @@
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
 use itertools::Itertools;
-use vortex_dtype::{DType, NativeDecimalType, match_each_decimal_value_type};
+use vortex_dtype::Nullability::NonNullable;
+use vortex_dtype::{DecimalDType, NativeDecimalType, match_each_decimal_value_type};
 use vortex_error::VortexResult;
 use vortex_mask::Mask;
-use vortex_scalar::{DecimalValue, Scalar, ScalarValue};
+use vortex_scalar::{DecimalValue, Scalar};
 
 use crate::arrays::{DecimalArray, DecimalVTable};
 use crate::compute::{MinMaxKernel, MinMaxKernelAdapter, MinMaxResult};
@@ -27,35 +28,38 @@ where
     D: Into<DecimalValue> + NativeDecimalType,
 {
     Ok(match array.validity_mask() {
-        Mask::AllTrue(_) => compute_min_max(array.buffer::<D>().iter(), array.dtype()),
+        Mask::AllTrue(_) => compute_min_max(array.buffer::<D>().iter(), array.decimal_dtype()),
         Mask::AllFalse(_) => None,
         Mask::Values(v) => compute_min_max(
             array
                 .buffer::<D>()
                 .iter()
                 .zip(v.bit_buffer().iter())
                 .filter_map(|(v, m)| m.then_some(v)),
-            array.dtype(),
+            array.decimal_dtype(),
         ),
     })
 }
 
-fn compute_min_max<'a, T>(iter: impl Iterator<Item = &'a T>, dtype: &DType) -> Option<MinMaxResult>
+fn compute_min_max<'a, T>(
+    iter: impl Iterator<Item = &'a T>,
+    decimal_dtype: DecimalDType,
+) -> Option<MinMaxResult>
 where
     T: Into<DecimalValue> + NativeDecimalType + Ord + Copy + 'a,
 {
     match iter.minmax_by(|a, b| a.cmp(b)) {
         itertools::MinMaxResult::NoElements => None,
         itertools::MinMaxResult::OneElement(&x) => {
-            let scalar = Scalar::new(dtype.clone(), ScalarValue::from(x.into()));
+            let scalar = Scalar::decimal(x.into(), decimal_dtype, NonNullable);
             Some(MinMaxResult {
                 min: scalar.clone(),
                 max: scalar,
             })
         }
         itertools::MinMaxResult::MinMax(&min, &max) => Some(MinMaxResult {
-            min: Scalar::new(dtype.clone(), ScalarValue::from(min.into())),
-            max: Scalar::new(dtype.clone(), ScalarValue::from(max.into())),
+            min: Scalar::decimal(min.into(), decimal_dtype, NonNullable),
+            max: Scalar::decimal(max.into(), decimal_dtype, NonNullable),
         }),
     }
 }
@@ -80,13 +84,14 @@ mod tests {
 
         let min_max = min_max(decimal.as_ref()).unwrap();
 
+        let non_nullable_dtype = decimal.dtype().as_nonnullable();
         let expected = MinMaxResult {
             min: Scalar::new(
-                decimal.dtype().clone(),
+                non_nullable_dtype.clone(),
                 ScalarValue::from(DecimalValue::from(100i32)),
             ),
             max: Scalar::new(
-                decimal.dtype().clone(),
+                non_nullable_dtype,
                 ScalarValue::from(DecimalValue::from(200i32)),
             ),
         };

vortex-array/src/arrays/decimal/vtable/mod.rs

@@ -8,6 +8,7 @@ use crate::{EncodingId, EncodingRef, vtable};
 mod array;
 mod canonical;
 mod operations;
+mod operator;
 mod serde;
 mod validity;
 mod visitor;

vortex-array/src/arrays/decimal/vtable/operator.rs

@@ -0,0 +1,65 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+use vortex_compute::filter::Filter;
+use vortex_dtype::{PrecisionScale, match_each_decimal_value_type};
+use vortex_error::VortexResult;
+use vortex_vector::decimal::DVector;
+
+use crate::arrays::{DecimalArray, DecimalVTable, MaskedVTable};
+use crate::execution::{BatchKernelRef, BindCtx, kernel};
+use crate::vtable::{OperatorVTable, ValidityHelper};
+use crate::{ArrayRef, IntoArray};
+
+impl OperatorVTable<DecimalVTable> for DecimalVTable {
+    fn bind(
+        array: &DecimalArray,
+        selection: Option<&ArrayRef>,
+        ctx: &mut dyn BindCtx,
+    ) -> VortexResult<BatchKernelRef> {
+        let mask = ctx.bind_selection(array.len(), selection)?;
+        let validity = ctx.bind_validity(array.validity(), array.len(), selection)?;
+
+        match_each_decimal_value_type!(array.values_type(), |D| {
+            let elements = array.buffer::<D>();
+            let ps = PrecisionScale::<D>::try_from(&array.decimal_dtype())?;
+
+            Ok(kernel(move || {
+                let mask = mask.execute()?;
+                let validity = validity.execute()?;
+
+                // Note that validity already has the mask applied so we only need to apply it to
+                // the elements.
+                let elements = elements.filter(&mask);
+
+                Ok(DVector::<D>::try_new(ps, elements, validity)?.into())
+            }))
+        })
+    }
+
+    fn reduce_parent(
+        array: &DecimalArray,
+        parent: &ArrayRef,
+        _child_idx: usize,
+    ) -> VortexResult<Option<ArrayRef>> {
+        // Push-down masking of `validity` from the parent `MaskedArray`.
+        if let Some(masked) = parent.as_opt::<MaskedVTable>() {
+            let masked_array = match_each_decimal_value_type!(array.values_type(), |D| {
+                // SAFETY: Since we are only flipping some bits in the validity, all invariants that
+                // were upheld are still upheld.
+                unsafe {
+                    DecimalArray::new_unchecked(
+                        array.buffer::<D>(),
+                        array.decimal_dtype(),
+                        array.validity().clone().and(masked.validity().clone()),
+                    )
+                }
+                .into_array()
+            });
+
+            return Ok(Some(masked_array));
+        }
+
+        Ok(None)
+    }
+}

vortex-array/src/arrays/extension/compute/min_max.rs

@@ -1,6 +1,9 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+use std::sync::Arc;
+
+use vortex_dtype::Nullability;
 use vortex_error::VortexResult;
 use vortex_scalar::Scalar;
 
@@ -10,10 +13,12 @@ use crate::register_kernel;
 
 impl MinMaxKernel for ExtensionVTable {
     fn min_max(&self, array: &ExtensionArray) -> VortexResult<Option<MinMaxResult>> {
+        let non_nullable_ext_dtype =
+            Arc::new(array.ext_dtype().with_nullability(Nullability::NonNullable));
         Ok(
             compute::min_max(array.storage())?.map(|MinMaxResult { min, max }| MinMaxResult {
-                min: Scalar::extension(array.ext_dtype().clone(), min),
-                max: Scalar::extension(array.ext_dtype().clone(), max),
+                min: Scalar::extension(non_nullable_ext_dtype.clone(), min),
+                max: Scalar::extension(non_nullable_ext_dtype, max),
             }),
         )
     }