feat: rest of BinaryViewVector{,Mut} (#5133)

Tracking Issue: #5028

---------

Signed-off-by: Andrew Duffy <[email protected]>

Author: a10y

Cargo.lock

@@ -226,6 +226,12 @@ impl<T> Buffer<T> {
         unsafe { std::slice::from_raw_parts(self.bytes.as_ptr().cast(), self.length) }
     }
 
+    /// Return a view over the buffer as an opaque byte slice.
+    #[inline(always)]
+    pub fn as_bytes(&self) -> &[u8] {
+        self.bytes.as_ref()
+    }
+
     /// Returns an iterator over the buffer of elements of type T.
     pub fn iter(&self) -> Iter<'_, T> {
         Iter {
@@ -319,7 +325,7 @@ impl<T> Buffer<T> {
 
     /// Returns a slice of self that is equivalent to the given subset.
     ///
-    /// When processing the buffer you will often end up with &\[T\] that is a subset
+    /// When processing the buffer you will often end up with `&[T]` that is a subset
     /// of the underlying buffer. This function turns the slice into a slice of the buffer
     /// it has been taken from.
     ///
@@ -332,7 +338,7 @@ impl<T> Buffer<T> {
 
     /// Returns a slice of self that is equivalent to the given subset.
     ///
-    /// When processing the buffer you will often end up with &\[T\] that is a subset
+    /// When processing the buffer you will often end up with `&[T]` that is a subset
     /// of the underlying buffer. This function turns the slice into a slice of the buffer
     /// it has been taken from.
     ///

vortex-buffer/src/buffer.rs

@@ -6,13 +6,13 @@ use std::sync::Arc;
 use arrow_array::{ArrayRef, GenericByteViewArray};
 use vortex_buffer::Buffer;
 use vortex_error::VortexResult;
-use vortex_vector::{BinaryType, StringType, VarBinVector};
+use vortex_vector::{BinaryType, BinaryViewVector, StringType};
 
 use crate::arrow::IntoArrow;
 
 macro_rules! impl_varbin {
     ($T:ty, $A:ty) => {
-        impl IntoArrow<ArrayRef> for VarBinVector<$T> {
+        impl IntoArrow<ArrayRef> for BinaryViewVector<$T> {
             fn into_arrow(self) -> VortexResult<ArrayRef> {
                 let (views, buffers, validity) = self.into_parts();
 

vortex-compute/src/arrow/varbin.rs vortex-compute/src/arrow/binaryview.rsvortex-compute/src/arrow/varbin.rs renamed to vortex-compute/src/arrow/binaryview.rs

@@ -5,13 +5,13 @@
 
 use vortex_error::VortexResult;
 
+mod binaryview;
 mod bool;
 mod decimal;
 mod mask;
 mod null;
 mod primitive;
 mod struct_;
-mod varbin;
 mod vector;
 
 /// Trait for converting Vortex vector types into Arrow types.

vortex-compute/src/arrow/mod.rs

@@ -8,8 +8,9 @@ use std::ops::BitAnd;
 use vortex_dtype::{NativeDecimalType, NativePType};
 use vortex_mask::Mask;
 use vortex_vector::{
-    BoolVector, DVector, DecimalVector, NullVector, PVector, PrimitiveVector, StructVector,
-    VarBinType, VarBinVector, Vector, match_each_dvector, match_each_pvector, match_each_vector,
+    BinaryViewType, BinaryViewVector, BoolVector, DVector, DecimalVector, NullVector, PVector,
+    PrimitiveVector, StructVector, Vector, match_each_dvector, match_each_pvector,
+    match_each_vector,
 };
 
 /// Trait for masking the validity of an array or vector.
@@ -70,7 +71,7 @@ impl<T: NativePType> MaskValidity for PVector<T> {
     }
 }
 
-impl<T: VarBinType> MaskValidity for VarBinVector<T> {
+impl<T: BinaryViewType> MaskValidity for BinaryViewVector<T> {
     fn mask_validity(self, mask: &Mask) -> Self {
         let (views, buffers, validity) = self.into_parts();
         // SAFETY: we are preserving the original views and buffers, only modifying the validity.

vortex-compute/src/mask/mod.rs

@@ -1,6 +1,13 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+//! Definition and implementation of variable-length binary types.
+//!
+//! All types are specializations of the [`BinaryViewVector`] type, which is represented internally
+//! by `BinaryView`s. `BinaryView`s are identical to the `BinaryView` type defined by the Arrow
+//! [specification](https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-view-layout),
+//! which are inspired by "German" strings.
+
 pub use types::*;
 pub use vector::*;
 pub use vector_mut::*;
@@ -14,16 +21,16 @@ mod vector_mut;
 mod view;
 
 /// Type alias for non-utf8 variable-length binary vectors.
-pub type BinaryVector = VarBinVector<BinaryType>;
+pub type BinaryVector = BinaryViewVector<BinaryType>;
 /// Type alias for mutable non-utf8 variable-length binary vectors.
-pub type BinaryVectorMut = VarBinVectorMut<BinaryType>;
+pub type BinaryVectorMut = BinaryViewVectorMut<BinaryType>;
 /// Type alias for UTF-8 variable-length string vectors.
-pub type StringVector = VarBinVector<StringType>;
+pub type StringVector = BinaryViewVector<StringType>;
 /// Type alias for mutable UTF-8 variable-length string vectors.
-pub type StringVectorMut = VarBinVectorMut<StringType>;
+pub type StringVectorMut = BinaryViewVectorMut<StringType>;
 
-impl VarBinTypeDowncast for Vector {
-    type Output<T: VarBinType> = VarBinVector<T>;
+impl BinaryViewDowncast for Vector {
+    type Output<T: BinaryViewType> = BinaryViewVector<T>;
 
     fn into_binary(self) -> Self::Output<BinaryType> {
         if let Vector::Binary(v) = self {
@@ -40,8 +47,8 @@ impl VarBinTypeDowncast for Vector {
     }
 }
 
-impl VarBinTypeUpcast for Vector {
-    type Input<T: VarBinType> = VarBinVector<T>;
+impl BinaryViewTypeUpcast for Vector {
+    type Input<T: BinaryViewType> = BinaryViewVector<T>;
 
     fn from_binary(input: Self::Input<BinaryType>) -> Self {
         Vector::Binary(input)
@@ -52,8 +59,8 @@ impl VarBinTypeUpcast for Vector {
     }
 }
 
-impl VarBinTypeDowncast for VectorMut {
-    type Output<T: VarBinType> = VarBinVectorMut<T>;
+impl BinaryViewDowncast for VectorMut {
+    type Output<T: BinaryViewType> = BinaryViewVectorMut<T>;
 
     fn into_binary(self) -> Self::Output<BinaryType> {
         if let VectorMut::Binary(v) = self {
@@ -70,8 +77,8 @@ impl VarBinTypeDowncast for VectorMut {
     }
 }
 
-impl VarBinTypeUpcast for VectorMut {
-    type Input<T: VarBinType> = VarBinVectorMut<T>;
+impl BinaryViewTypeUpcast for VectorMut {
+    type Input<T: BinaryViewType> = BinaryViewVectorMut<T>;
 
     fn from_binary(input: Self::Input<BinaryType>) -> Self {
         VectorMut::Binary(input)

vortex-vector/src/varbin/mod.rs vortex-vector/src/binaryview/mod.rsvortex-vector/src/varbin/mod.rs renamed to vortex-vector/src/binaryview/mod.rs

@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+//! Variable-length binary types and related traits.
+
+use std::fmt::Debug;
+
+use crate::{BinaryViewVector, BinaryViewVectorMut, Vector, VectorMut};
+
+impl<T: BinaryViewType> From<BinaryViewVector<T>> for Vector {
+    fn from(value: BinaryViewVector<T>) -> Self {
+        T::upcast(value)
+    }
+}
+
+impl<T: BinaryViewType> From<BinaryViewVectorMut<T>> for VectorMut {
+    fn from(value: BinaryViewVectorMut<T>) -> Self {
+        T::upcast(value)
+    }
+}
+
+/// Trait to mark supported binary view types.
+pub trait BinaryViewType: Debug + Sized + private::Sealed {
+    /// The slice type for this variable binary type.
+    type Slice: ?Sized + AsRef<[u8]>;
+
+    /// Validate if a set of bytes conforms to the logical type constraints of the native `Slice`.
+    fn validate(bytes: &[u8]) -> bool;
+
+    /// Returns the bytes as the native `Slice` type
+    /// for this binary view vector.
+    ///
+    /// # Safety
+    ///
+    /// The caller must check beforehand that bytes return from the vector conform to the type
+    /// requirements of this binary type.
+    ///
+    /// Failure to do so can result in undefined behavior or incorrect results in downstream
+    /// vector operations.
+    unsafe fn from_bytes_unchecked(bytes: &[u8]) -> &Self::Slice;
+
+    /// Downcast the provided object to a type-specific instance.
+    fn downcast<V: BinaryViewDowncast>(visitor: V) -> V::Output<Self>;
+
+    /// Upcast a type-specific instance to a generic instance.
+    fn upcast<V: BinaryViewTypeUpcast>(input: V::Input<Self>) -> V;
+}
+
+/// [`BinaryType`] for UTF-8 strings.
+#[derive(Clone, Debug)]
+pub struct StringType;
+impl BinaryViewType for StringType {
+    type Slice = str;
+
+    #[inline(always)]
+    fn validate(bytes: &[u8]) -> bool {
+        std::str::from_utf8(bytes).is_ok()
+    }
+
+    unsafe fn from_bytes_unchecked(bytes: &[u8]) -> &Self::Slice {
+        // SAFETY: vectors should be checked at the boundary for upholding the UTF8 variant,
+        //  or only be built from vectors that are known to satisfy the variant.
+        unsafe { std::str::from_utf8_unchecked(bytes) }
+    }
+
+    fn downcast<V: BinaryViewDowncast>(visitor: V) -> V::Output<Self> {
+        visitor.into_string()
+    }
+
+    fn upcast<V: BinaryViewTypeUpcast>(input: V::Input<Self>) -> V {
+        V::from_string(input)
+    }
+}
+
+/// [`BinaryType`] for raw binary data.
+#[derive(Clone, Debug)]
+pub struct BinaryType;
+impl BinaryViewType for BinaryType {
+    type Slice = [u8];
+
+    #[inline(always)]
+    fn validate(_bytes: &[u8]) -> bool {
+        true
+    }
+
+    unsafe fn from_bytes_unchecked(bytes: &[u8]) -> &Self::Slice {
+        bytes
+    }
+
+    fn downcast<V: BinaryViewDowncast>(visitor: V) -> V::Output<Self> {
+        visitor.into_binary()
+    }
+
+    fn upcast<V: BinaryViewTypeUpcast>(input: V::Input<Self>) -> V {
+        V::from_binary(input)
+    }
+}
+
+/// Trait for downcasting generic variable binary types to specific types.
+pub trait BinaryViewDowncast {
+    /// The output type after downcasting.
+    type Output<T: BinaryViewType>;
+
+    /// Downcast to a binary type.
+    fn into_binary(self) -> Self::Output<BinaryType>;
+    /// Downcast to a string type.
+    fn into_string(self) -> Self::Output<StringType>;
+}
+
+/// Trait for upcasting specific variable binary types to generic types.
+pub trait BinaryViewTypeUpcast {
+    /// The input type for upcasting.
+    type Input<T: BinaryViewType>;
+
+    /// Upcast from a binary type.
+    fn from_binary(input: Self::Input<BinaryType>) -> Self;
+    /// Upcast from a string type.
+    fn from_string(input: Self::Input<StringType>) -> Self;
+}
+
+/// Private module to seal the `BinaryViewType` trait.
+mod private {
+    /// Sealed trait to prevent external implementations of [`VarBinType`].
+    pub trait Sealed {}
+
+    impl Sealed for super::StringType {}
+    impl Sealed for super::BinaryType {}
+}