feat: teach PrimitiveArray fill_null (#2006)

Author: danking

vortex-array/src/array/primitive/compute/fill.rs

@@ -1,9 +1,10 @@
 use vortex_buffer::Buffer;
 use vortex_dtype::{match_each_native_ptype, Nullability};
 use vortex_error::{vortex_err, VortexResult};
+use vortex_scalar::Scalar;
 
 use crate::array::primitive::PrimitiveArray;
-use crate::array::PrimitiveEncoding;
+use crate::array::{ConstantArray, PrimitiveEncoding};
 use crate::compute::FillForwardFn;
 use crate::validity::{ArrayValidity, Validity};
 use crate::variants::PrimitiveArrayTrait;
@@ -27,10 +28,8 @@ impl FillForwardFn<PrimitiveArray> for PrimitiveEncoding {
 
         if validity.all_invalid() {
             match_each_native_ptype!(array.ptype(), |$T| {
-                return Ok(PrimitiveArray::new(
-                    Buffer::<$T>::zeroed(array.len()),
-                    Validity::AllValid
-                ).into_array());
+                let fill_value = Scalar::from($T::default()).cast(array.dtype())?;
+                return Ok(ConstantArray::new(fill_value, array.len()).into_array())
             })
         }
 
@@ -90,12 +89,12 @@ mod test {
     #[test]
     fn nullable_non_null() {
         let arr = PrimitiveArray::new(
-            buffer![8u8, 10u8, 12u8, 14u8, 16u8],
+            buffer![8u8, 10, 12, 14, 16],
             Validity::Array(BoolArray::from_iter([true, true, true, true, true]).into_array()),
         )
         .into_array();
         let p = fill_forward(&arr).unwrap().into_primitive().unwrap();
-        assert_eq!(p.as_slice::<u8>(), vec![8, 10, 12, 14, 16]);
+        assert_eq!(p.as_slice::<u8>(), vec![8u8, 10, 12, 14, 16]);
         assert!(p.logical_validity().all_valid());
     }
 }

vortex-array/src/array/primitive/compute/fill_null.rs

@@ -0,0 +1,109 @@
+use std::ops::Not;
+
+use vortex_buffer::BufferMut;
+use vortex_dtype::{match_each_native_ptype, Nullability};
+use vortex_error::{VortexExpect, VortexResult};
+use vortex_scalar::Scalar;
+
+use crate::array::primitive::PrimitiveArray;
+use crate::array::{ConstantArray, PrimitiveEncoding};
+use crate::compute::FillNullFn;
+use crate::validity::Validity;
+use crate::variants::PrimitiveArrayTrait;
+use crate::{ArrayData, ArrayLen, IntoArrayData, IntoArrayVariant as _};
+
+impl FillNullFn<PrimitiveArray> for PrimitiveEncoding {
+    fn fill_null(&self, array: &PrimitiveArray, fill_value: Scalar) -> VortexResult<ArrayData> {
+        let result_validity = match fill_value.dtype().nullability() {
+            Nullability::NonNullable => Validity::NonNullable,
+            Nullability::Nullable => Validity::AllValid,
+        };
+
+        Ok(match array.validity() {
+            Validity::NonNullable | Validity::AllValid => {
+                match_each_native_ptype!(array.ptype(), |$T| {
+                    PrimitiveArray::new::<$T>(array.buffer().clone(), result_validity).into_array()
+                })
+            }
+            Validity::AllInvalid => ConstantArray::new(fill_value, array.len()).into_array(),
+            Validity::Array(is_valid) => {
+                // TODO(danking): when we take PrimitiveArray by value, we should mutate in-place
+                let is_invalid = is_valid.into_bool()?.boolean_buffer().not();
+                match_each_native_ptype!(array.ptype(), |$T| {
+                    let mut buffer = BufferMut::copy_from(array.as_slice::<$T>());
+                    let fill_value = fill_value
+                        .as_primitive()
+                        .typed_value::<$T>()
+                        .vortex_expect("top-level fill_null ensure non-null fill value");
+                    for invalid_index in is_invalid.set_indices() {
+                        buffer[invalid_index] = fill_value;
+                    }
+                    PrimitiveArray::new(buffer, result_validity).into_array()
+                })
+            }
+        })
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use vortex_buffer::buffer;
+    use vortex_scalar::Scalar;
+
+    use crate::array::primitive::PrimitiveArray;
+    use crate::array::BoolArray;
+    use crate::compute::fill_null;
+    use crate::validity::{ArrayValidity, Validity};
+    use crate::{IntoArrayData, IntoArrayVariant};
+
+    #[test]
+    fn fill_null_leading_none() {
+        let arr =
+            PrimitiveArray::from_option_iter([None, Some(8u8), None, Some(10), None]).into_array();
+        let p = fill_null(&arr, Scalar::from(42u8))
+            .unwrap()
+            .into_primitive()
+            .unwrap();
+        assert_eq!(p.as_slice::<u8>(), vec![42, 8, 42, 10, 42]);
+        assert!(p.logical_validity().all_valid());
+    }
+
+    #[test]
+    fn fill_null_all_none() {
+        let arr = PrimitiveArray::from_option_iter([Option::<u8>::None, None, None, None, None])
+            .into_array();
+
+        let p = fill_null(&arr, Scalar::from(255u8))
+            .unwrap()
+            .into_primitive()
+            .unwrap();
+        assert_eq!(p.as_slice::<u8>(), vec![255, 255, 255, 255, 255]);
+        assert!(p.logical_validity().all_valid());
+    }
+
+    #[test]
+    fn fill_null_nullable_non_null() {
+        let arr = PrimitiveArray::new(
+            buffer![8u8, 10, 12, 14, 16],
+            Validity::Array(BoolArray::from_iter([true, true, true, true, true]).into_array()),
+        )
+        .into_array();
+        let p = fill_null(&arr, Scalar::from(255u8))
+            .unwrap()
+            .into_primitive()
+            .unwrap();
+        assert_eq!(p.as_slice::<u8>(), vec![8, 10, 12, 14, 16]);
+        assert!(p.logical_validity().all_valid());
+    }
+
+    #[test]
+    fn fill_null_non_nullable() {
+        let arr = buffer![8u8, 10, 12, 14, 16].into_array();
+        let p = fill_null(&arr, Scalar::from(255u8))
+            .unwrap()
+            .into_primitive()
+            .unwrap();
+        assert_eq!(p.as_slice::<u8>(), vec![8u8, 10, 12, 14, 16]);
+        assert!(p.logical_validity().all_valid());
+    }
+}

vortex-array/src/array/primitive/compute/mod.rs

@@ -1,12 +1,13 @@
 use crate::array::PrimitiveEncoding;
 use crate::compute::{
-    CastFn, ComputeVTable, FillForwardFn, FilterFn, ScalarAtFn, SearchSortedFn,
+    CastFn, ComputeVTable, FillForwardFn, FillNullFn, FilterFn, ScalarAtFn, SearchSortedFn,
     SearchSortedUsizeFn, SliceFn, TakeFn,
 };
 use crate::ArrayData;
 
 mod cast;
 mod fill;
+mod fill_null;
 mod filter;
 mod scalar_at;
 mod search_sorted;
@@ -22,6 +23,10 @@ impl ComputeVTable for PrimitiveEncoding {
         Some(self)
     }
 
+    fn fill_null_fn(&self) -> Option<&dyn FillNullFn<ArrayData>> {
+        Some(self)
+    }
+
     fn filter_fn(&self) -> Option<&dyn FilterFn<ArrayData>> {
         Some(self)
     }