Arrays to require validity

Signed-off-by: Nicholas Gates <[email protected]>

Author: gatesn

encodings/sparse/src/lib.rs

@@ -412,10 +412,24 @@ impl ValidityVTable<SparseVTable> for SparseVTable {
     }
 
     fn validity(array: &SparseArray) -> VortexResult<Validity> {
-        // TODO(ngates): should this be able to execute arrays?
-        Ok(Validity::from_mask(
-            Self::validity_mask(array),
-            Nullability::Nullable,
+        let patches = unsafe {
+            Patches::new_unchecked(
+                array.patches.array_len(),
+                array.patches.offset().clone(),
+                array.patches.indices().clone(),
+                array
+                    .patches
+                    .values()
+                    .validity()?
+                    .to_array(array.patches.values().len()),
+                array.patches.chunk_offsets().clone(),
+                array.patches.offset_within_chunk(),
+            )
+        };
+
+        Ok(Validity::Array(
+            unsafe { SparseArray::new_unchecked(patches, array.fill_value.is_valid().into()) }
+                .into_array(),
         ))
     }
 

vortex-array/src/arrow/array.rs

@@ -6,6 +6,7 @@ use std::hash::Hash;
 use std::ops::Range;
 
 use arrow_array::ArrayRef as ArrowArrayRef;
+use vortex_buffer::BitBuffer;
 use vortex_buffer::BufferHandle;
 use vortex_dtype::DType;
 use vortex_dtype::Nullability;
@@ -25,6 +26,7 @@ use crate::Canonical;
 use crate::EmptyMetadata;
 use crate::IntoArray;
 use crate::Precision;
+use crate::arrays::BoolArray;
 use crate::arrow::FromArrowArray;
 use crate::serde::ArrayChildren;
 use crate::stats::ArrayStats;
@@ -185,10 +187,20 @@ impl ValidityVTable<ArrowVTable> for ArrowVTable {
     }
 
     fn validity(array: &ArrowArray) -> VortexResult<Validity> {
-        Ok(Validity::from_mask(
-            Self::validity_mask(array),
-            Nullability::Nullable,
-        ))
+        Ok(match array.inner.logical_nulls() {
+            None => Validity::AllValid,
+            Some(null_buffer) => match null_buffer.null_count() {
+                0 => Validity::AllValid,
+                n if n == array.inner.len() => Validity::AllInvalid,
+                _ => Validity::Array(
+                    BoolArray::new(
+                        BitBuffer::from(null_buffer.inner().clone()),
+                        Validity::NonNullable,
+                    )
+                    .into_array(),
+                ),
+            },
+        })
     }
 
     fn validity_mask(array: &ArrowArray) -> Mask {