Chore: Remove nullability from Vectors (#5029)

Tracking Issue: #5028

Removes explicit nullability in Vectors, meaning our physical vectors
now behave even more like Arrow.

---------

Signed-off-by: Connor Tsui <[email protected]>

Author: connortsui20

vortex-vector/src/bool/vector.rs

@@ -4,42 +4,33 @@
 //! Definition and implementation of [`BoolVector`].
 
 use vortex_buffer::BitBuffer;
-use vortex_dtype::Nullability;
 use vortex_mask::Mask;
 
 use super::BoolVectorMut;
 use crate::VectorOps;
 
 /// An immutable vector of boolean values.
 ///
-/// Internally, the boolean values are stored as the bits of a [`BitBuffer`] plus an optional
-/// [`Mask`] for null booleans (where `true` represents a _valid_ boolean and `false` represents a
-/// `null` boolean).
-///
 /// The mutable equivalent of this type is [`BoolVectorMut`].
 #[derive(Debug, Clone)]
 pub struct BoolVector {
     pub(super) bits: BitBuffer,
-    pub(super) validity: Option<Mask>,
+    pub(super) validity: Mask,
 }
 
 impl VectorOps for BoolVector {
     type Mutable = BoolVectorMut;
 
-    fn nullability(&self) -> Nullability {
-        Nullability::from(self.validity.is_some())
-    }
-
     fn len(&self) -> usize {
-        debug_assert!(
-            self.validity
-                .as_ref()
-                .is_none_or(|mask| mask.len() == self.bits.len())
-        );
+        debug_assert!(self.validity.len() == self.bits.len());
 
         self.bits.len()
     }
 
+    fn validity(&self) -> &Mask {
+        &self.validity
+    }
+
     fn try_into_mut(self) -> Result<Self::Mutable, Self>
     where
         Self: Sized,
@@ -54,19 +45,15 @@ impl VectorOps for BoolVector {
             }
         };
 
-        let validity = match self.validity {
-            Some(v) => match v.try_into_mut() {
-                Ok(v) => Some(v),
-                Err(v) => {
-                    return Err(BoolVector {
-                        bits: bits.freeze(),
-                        validity: Some(v),
-                    });
-                }
-            },
-            None => None,
-        };
-
-        Ok(BoolVectorMut { bits, validity })
+        match self.validity.try_into_mut() {
+            Ok(validity_mut) => Ok(BoolVectorMut {
+                bits,
+                validity: validity_mut,
+            }),
+            Err(validity) => Err(BoolVector {
+                bits: bits.freeze(),
+                validity,
+            }),
+        }
     }
 }

vortex-vector/src/bool/vector_mut.rs

@@ -4,53 +4,35 @@
 //! Definition and implementation of [`BoolVectorMut`].
 
 use vortex_buffer::BitBufferMut;
-use vortex_dtype::Nullability;
-use vortex_error::vortex_panic;
 use vortex_mask::MaskMut;
 
 use super::BoolVector;
 use crate::{VectorMutOps, VectorOps};
 
 /// A mutable vector of boolean values.
 ///
-/// Internally, the boolean values are stored as the bits of a [`BitBufferMut`] plus an optional
-/// [`MaskMut`] for null booleans (where `true` represents a _valid_ boolean and `false` represents
-/// a `null` boolean).
-///
 /// The immutable equivalent of this type is [`BoolVector`].
 #[derive(Debug, Clone)]
 pub struct BoolVectorMut {
     pub(super) bits: BitBufferMut,
-    pub(super) validity: Option<MaskMut>,
+    pub(super) validity: MaskMut,
 }
 
 impl BoolVectorMut {
-    /// Creates a new mutable boolean vector with the given `capacity` and `nullability`.
-    pub fn with_capacity(capacity: usize, nullability: Nullability) -> Self {
-        let validity = nullability
-            .is_nullable()
-            .then(|| MaskMut::with_capacity(capacity));
-
+    /// Creates a new mutable boolean vector with the given `capacity`.
+    pub fn with_capacity(capacity: usize) -> Self {
         Self {
             bits: BitBufferMut::with_capacity(capacity),
-            validity,
+            validity: MaskMut::with_capacity(capacity),
         }
     }
 }
 
 impl VectorMutOps for BoolVectorMut {
     type Immutable = BoolVector;
 
-    fn nullability(&self) -> Nullability {
-        Nullability::from(self.validity.is_some())
-    }
-
     fn len(&self) -> usize {
-        debug_assert!(
-            self.validity
-                .as_ref()
-                .is_none_or(|mask| mask.len() == self.bits.len())
-        );
+        debug_assert!(self.validity.len() == self.bits.len());
 
         self.bits.len()
     }
@@ -61,74 +43,35 @@ impl VectorMutOps for BoolVectorMut {
 
     fn reserve(&mut self, additional: usize) {
         self.bits.reserve(additional);
-
-        if let Some(v) = self.validity.as_mut() {
-            v.reserve(additional);
-        }
+        self.validity.reserve(additional);
     }
 
     fn extend_from_vector(&mut self, other: &BoolVector) {
-        assert_eq!(
-            self.nullability(),
-            other.nullability(),
-            "tried to extend a vector with nullability {} with another vector with nullability {}",
-            self.nullability(),
-            other.nullability(),
-        );
-
         self.bits.append_buffer(&other.bits);
-
-        match (&mut self.validity, &other.validity) {
-            (Some(self_v), Some(other_v)) => self_v.append_mask(other_v),
-            (Some(self_v), None) => self_v.append_n(true, other.bits.len()),
-            (None, Some(other_v)) => {
-                let mut new_validity = MaskMut::new_true(self.bits.len() - other.bits.len());
-                new_validity.append_mask(other_v);
-                self.validity = Some(new_validity);
-            }
-            (None, None) => {}
-        }
+        self.validity.append_mask(other.validity());
     }
 
     fn append_nulls(&mut self, n: usize) {
-        let Some(mask) = &mut self.validity else {
-            vortex_panic!("tried to append nulls to a non-nullable vector")
-        };
-
-        mask.append_n(false, n);
-
         self.bits.append_n(false, n);
+        self.validity.append_n(false, n);
     }
 
     fn freeze(self) -> Self::Immutable {
         BoolVector {
             bits: self.bits.freeze(),
-            validity: self.validity.map(|v| v.freeze()),
+            validity: self.validity.freeze(),
         }
     }
 
     fn split_off(&mut self, at: usize) -> Self {
         BoolVectorMut {
             bits: self.bits.split_off(at),
-            validity: self.validity.as_mut().map(|v| v.split_off(at)),
+            validity: self.validity.split_off(at),
         }
     }
 
     fn unsplit(&mut self, other: Self) {
-        // TODO(connor): We must check `other`'s nullability in relation to `self`.
-
-        let other_len = other.bits.len();
         self.bits.unsplit(other.bits);
-
-        match (&mut self.validity, other.validity) {
-            (Some(self_v), Some(other_v)) => self_v.unsplit(other_v),
-            (Some(self_v), None) => self_v.append_n(true, other_len),
-            (None, Some(other_v)) => {
-                let mut new_validity = MaskMut::new_true(self.bits.len() - other_len);
-                new_validity.unsplit(other_v);
-                self.validity = Some(new_validity);
-            }
-            (None, None) => {}
-        }
+        self.validity.unsplit(other.validity);
     }
 }

vortex-vector/src/null/vector.rs

@@ -3,7 +3,7 @@
 
 //! Definition and implementation of [`NullVector`].
 
-use vortex_dtype::Nullability;
+use vortex_mask::Mask;
 
 use crate::{NullVectorMut, VectorOps};
 
@@ -13,29 +13,33 @@ use crate::{NullVectorMut, VectorOps};
 /// single `length` counter.
 ///
 /// The mutable equivalent of this type is [`NullVectorMut`].
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone)]
 pub struct NullVector {
     pub(super) len: usize,
+    pub(super) validity: Mask,
 }
 
 impl NullVector {
     /// Creates a new immutable vector of nulls with the given length.
     pub fn new(len: usize) -> Self {
-        Self { len }
+        Self {
+            len,
+            validity: Mask::AllFalse(len),
+        }
     }
 }
 
 impl VectorOps for NullVector {
     type Mutable = NullVectorMut;
 
-    fn nullability(&self) -> Nullability {
-        Nullability::Nullable
-    }
-
     fn len(&self) -> usize {
         self.len
     }
 
+    fn validity(&self) -> &Mask {
+        &self.validity
+    }
+
     fn try_into_mut(self) -> Result<Self::Mutable, Self>
     where
         Self: Sized,

vortex-vector/src/null/vector_mut.rs

@@ -3,8 +3,6 @@
 
 //! Definition and implementation of [`NullVectorMut`].
 
-use vortex_dtype::Nullability;
-
 use super::NullVector;
 use crate::VectorMutOps;
 
@@ -29,10 +27,6 @@ impl NullVectorMut {
 impl VectorMutOps for NullVectorMut {
     type Immutable = NullVector;
 
-    fn nullability(&self) -> Nullability {
-        Nullability::Nullable
-    }
-
     fn len(&self) -> usize {
         self.len
     }

vortex-vector/src/ops.rs

@@ -4,7 +4,7 @@
 //! Definition and implementation of [`VectorOps`] and [`VectorMutOps`] for [`Vector`] and
 //! [`VectorMut`], respectively.
 
-use vortex_dtype::Nullability;
+use vortex_mask::Mask;
 
 use crate::{Vector, VectorMut, private};
 
@@ -13,16 +13,6 @@ pub trait VectorOps: private::Sealed + Into<Vector> {
     /// The mutable equivalent of this immutable vector.
     type Mutable: VectorMutOps<Immutable = Self>;
 
-    /// Returns the [`Nullability`] of the vector.
-    fn nullability(&self) -> Nullability;
-
-    /// Returns `true` if the nullability of this vector is [`Nullable`].
-    ///
-    /// [`Nullable`]: Nullability::Nullable
-    fn is_nullable(&self) -> bool {
-        self.nullability().is_nullable()
-    }
-
     /// Returns the number of elements in the vector, also referred to as its "length".
     fn len(&self) -> usize;
 
@@ -31,6 +21,14 @@ pub trait VectorOps: private::Sealed + Into<Vector> {
         self.len() == 0
     }
 
+    /// Returns the validity mask of the vector, where `true` represents a _valid_ element and
+    /// `false` represents a `null` element.
+    ///
+    /// Note that vectors are **always** considered nullable. "Non-nullable" data will simply have a
+    /// [`Mask`] of [`AllTrue(len)`](Mask::AllTrue). It is on the caller to ensure that they do not
+    /// add nullable data to a vector they want to keep as non-nullable.
+    fn validity(&self) -> &Mask;
+
     /// Tries to convert `self` into a mutable vector (implementing [`VectorMutOps`]).
     ///
     /// This method will only succeed if `self` is the only unique strong reference (it effectively
@@ -50,16 +48,6 @@ pub trait VectorMutOps: private::Sealed + Into<VectorMut> {
     /// The immutable equivalent of this mutable vector.
     type Immutable: VectorOps<Mutable = Self>;
 
-    /// Returns the [`Nullability`] of the vector.
-    fn nullability(&self) -> Nullability;
-
-    /// Returns `true` if the nullability of this vector is [`Nullable`].
-    ///
-    /// [`Nullable`]: Nullability::Nullable
-    fn is_nullable(&self) -> bool {
-        self.nullability().is_nullable()
-    }
-
     /// Returns the number of elements in the vector, also referred to as its "length".
     fn len(&self) -> usize;
 
@@ -82,20 +70,12 @@ pub trait VectorMutOps: private::Sealed + Into<VectorMut> {
     fn reserve(&mut self, additional: usize);
 
     /// Extends the vector by appending elements from another vector.
-    ///
-    /// # Panics
-    ///
-    /// Panics if `other` does not have the same nullability as `self`.
     fn extend_from_vector(&mut self, other: &Self::Immutable);
 
-    /// Appends `n` null elements to the vector (if it is nullable).
+    /// Appends `n` null elements to the vector.
     ///
     /// Implementors should ensure that they correctly append "null" or garbage values to their
-    /// elements in addition to adding nulls to their validity mask.
-    ///
-    /// # Panics
-    ///
-    /// If `self` is a non-nullable vector, implementors should ensure that this function panics.
+    /// elements in addition to adding nulls to their validity mask.s.
     fn append_nulls(&mut self, n: usize);
 
     /// Converts `self` into an immutable vector.
@@ -115,7 +95,6 @@ pub trait VectorMutOps: private::Sealed + Into<VectorMut> {
     /// `at > capacity`).
     fn split_off(&mut self, at: usize) -> Self;
 
-    // TODO(connor): Should this panic if other has a different nullability?
     /// Absorbs a mutable vector that was previously split off.
     ///
     /// If the two vectors were previously contiguous and not mutated in a way that causes