Feature: ListViewArray (#4708)

Tracking Issue: #4699

Adds a new `ListViewArray` encoding.

The compute functions and the `ListViewBuilder` will come in a future
PR.

---------

Signed-off-by: Connor Tsui <[email protected]>

Author: connortsui20

encodings/sparse/src/canonical.rs

@@ -7,13 +7,13 @@ use itertools::Itertools;
 use num_traits::NumCast;
 use vortex_array::arrays::{
     BinaryView, BoolArray, BooleanBuffer, ConstantArray, FixedSizeListArray, ListArray, NullArray,
-    OffsetPType, PrimitiveArray, StructArray, VarBinViewArray, smallest_storage_type,
+    PrimitiveArray, StructArray, VarBinViewArray, smallest_storage_type,
 };
 use vortex_array::builders::{ArrayBuilder, DecimalBuilder, ListBuilder, builder_with_capacity};
 use vortex_array::patches::Patches;
 use vortex_array::validity::Validity;
 use vortex_array::vtable::{CanonicalVTable, ValidityHelper};
-use vortex_array::{Array, ArrayRef, Canonical, IntoArray as _, ToCanonical as _};
+use vortex_array::{Array, ArrayRef, Canonical, IntoArray as _, OffsetPType, ToCanonical as _};
 use vortex_buffer::{Buffer, BufferMut, BufferString, ByteBuffer, buffer, buffer_mut};
 use vortex_dtype::{
     DType, DecimalDType, NativePType, Nullability, StructFields, match_each_integer_ptype,

vortex-array/src/arrays/arbitrary.rs

@@ -14,13 +14,12 @@ use vortex_scalar::arbitrary::random_scalar;
 use vortex_scalar::{Scalar, match_each_decimal_value_type};
 
 use super::{
-    BoolArray, ChunkedArray, NullArray, OffsetPType, PrimitiveArray, StructArray,
-    smallest_storage_type,
+    BoolArray, ChunkedArray, NullArray, PrimitiveArray, StructArray, smallest_storage_type,
 };
 use crate::arrays::{VarBinArray, VarBinViewArray};
 use crate::builders::{ArrayBuilder, DecimalBuilder, FixedSizeListBuilder};
 use crate::validity::Validity;
-use crate::{Array, ArrayRef, IntoArray, ToCanonical, builders};
+use crate::{Array, ArrayRef, IntoArray, OffsetPType, ToCanonical, builders};
 
 /// A wrapper type to implement `Arbitrary` for `ArrayRef`.
 #[derive(Clone, Debug)]

vortex-array/src/arrays/fixed_size_list/mod.rs

@@ -83,9 +83,9 @@ pub struct FixedSizeListArray {
 
     /// The validity / null map of the array.
     ///
-    /// Note that this null map refers to the fixed-size list scalars, **not** the elements of the
-    /// _individual_ fixed-size list scalars. The `elements` array will track individual value
-    /// nullability.
+    /// Note that this null map refers to which fixed-size list scalars are null, **not** which
+    /// sub-elements of fixed-size list scalars are null. The `elements` array will track individual
+    /// value nullability.
     validity: Validity,
 
     /// The length of the array.

vortex-array/src/arrays/fixed_size_list/vtable/serde.rs

@@ -32,9 +32,13 @@ impl SerdeVTable<FixedSizeListVTable> for FixedSizeListVTable {
     ) -> VortexResult<FixedSizeListArray> {
         vortex_ensure!(
             buffers.is_empty(),
-            "`FixedSizeListVTable::build` got some buffers"
+            "`FixedSizeListVTable::build` expects no buffers"
         );
 
+        let DType::FixedSizeList(element_dtype, list_size, _) = &dtype else {
+            vortex_bail!("Expected `DType::FixedSizeList`, got {:?}", dtype);
+        };
+
         let validity = {
             if children.len() > 2 {
                 vortex_bail!("`FixedSizeListVTable::build` method expected 1 or 2 children")
@@ -49,10 +53,6 @@ impl SerdeVTable<FixedSizeListVTable> for FixedSizeListVTable {
             }
         };
 
-        let DType::FixedSizeList(element_dtype, list_size, _) = &dtype else {
-            vortex_bail!("Expected `DType::FixedSizeList`, got {:?}", dtype);
-        };
-
         let num_elements = len * (*list_size as usize);
         let elements = children.get(0, element_dtype.as_ref(), num_elements)?;
 

vortex-array/src/arrays/list/compute/take.rs

@@ -7,12 +7,12 @@ use vortex_dtype::{NativePType, Nullability, match_each_integer_ptype};
 use vortex_error::{VortexExpect, VortexResult, vortex_panic};
 use vortex_mask::Mask;
 
-use crate::arrays::{ListArray, ListVTable, OffsetPType, PrimitiveArray};
+use crate::arrays::{ListArray, ListVTable, PrimitiveArray};
 use crate::builders::{ArrayBuilder, PrimitiveBuilder};
 use crate::compute::{TakeKernel, TakeKernelAdapter, take};
 use crate::validity::Validity;
 use crate::vtable::ValidityHelper;
-use crate::{Array, ArrayRef, ToCanonical, register_kernel};
+use crate::{Array, ArrayRef, OffsetPType, ToCanonical, register_kernel};
 
 impl TakeKernel for ListVTable {
     fn take(&self, array: &ListArray, indices: &dyn Array) -> VortexResult<ArrayRef> {

vortex-array/src/arrays/list/mod.rs

@@ -9,11 +9,13 @@ use std::sync::Arc;
 
 #[cfg(feature = "test-harness")]
 use itertools::Itertools;
-use num_traits::{AsPrimitive, PrimInt};
-use vortex_dtype::{DType, NativePType, match_each_integer_ptype, match_each_native_ptype};
+use num_traits::AsPrimitive;
+use vortex_dtype::{DType, match_each_integer_ptype, match_each_native_ptype};
 use vortex_error::{VortexExpect, VortexResult, vortex_bail, vortex_ensure};
 use vortex_scalar::Scalar;
 
+#[cfg(feature = "test-harness")]
+use crate::OffsetPType;
 use crate::arrays::PrimitiveVTable;
 #[cfg(feature = "test-harness")]
 use crate::builders::{ArrayBuilder, ListBuilder};
@@ -115,10 +117,6 @@ pub struct ListArray {
 #[derive(Clone, Debug)]
 pub struct ListEncoding;
 
-pub trait OffsetPType: NativePType + PrimInt + AsPrimitive<usize> + Into<Scalar> {}
-
-impl<T> OffsetPType for T where T: NativePType + PrimInt + AsPrimitive<usize> + Into<Scalar> {}
-
 impl ListArray {
     /// Creates a new [`ListArray`].
     ///