Move SparseArray into a crate (#2090)

Author: gatesn

Cargo.lock

@@ -162,9 +162,10 @@ vortex-layout = { version = "0.22.1", path = "./vortex-layout" }
 vortex-mask = { version = "0.22.1", path = "./vortex-mask" }
 vortex-proto = { version = "0.22.1", path = "./vortex-proto" }
 vortex-runend = { version = "0.22.1", path = "./encodings/runend" }
+vortex-sampling-compressor = { version = "0.22.1", path = "./vortex-sampling-compressor" }
 vortex-scalar = { version = "0.22.1", path = "./vortex-scalar", default-features = false }
 vortex-scan = { version = "0.22.1", path = "./vortex-scan", default-features = false }
-vortex-sampling-compressor = { version = "0.22.1", path = "./vortex-sampling-compressor" }
+vortex-sparse = { version = "0.22.1", path = "./encodings/sparse" }
 vortex-zigzag = { version = "0.22.1", path = "./encodings/zigzag" }
 # END crates published by this project
 

Cargo.toml

@@ -8,8 +8,8 @@ use tabled::settings::themes::Colorization;
 use tabled::settings::{Color, Style};
 use vortex::array::builder::VarBinBuilder;
 use vortex::array::{
-    BoolArray, ChunkedArray, ConstantArray, ListArray, NullArray, PrimitiveArray, SparseArray,
-    StructArray, VarBinViewArray,
+    BoolArray, ChunkedArray, ConstantArray, ListArray, NullArray, PrimitiveArray, StructArray,
+    VarBinViewArray,
 };
 use vortex::buffer::buffer;
 use vortex::datetime_dtype::{TemporalMetadata, TimeUnit, TIME_ID};
@@ -21,6 +21,7 @@ use vortex::encodings::dict::DictArray;
 use vortex::encodings::fastlanes::{BitPackedArray, DeltaArray, FoRArray};
 use vortex::encodings::fsst::{fsst_compress, fsst_train_compressor};
 use vortex::encodings::runend::RunEndArray;
+use vortex::encodings::sparse::SparseArray;
 use vortex::encodings::zigzag::ZigZagArray;
 use vortex::scalar::Scalar;
 use vortex::validity::Validity;

bench-vortex/src/bin/notimplemented.rs

@@ -23,6 +23,7 @@ vortex-dtype = { workspace = true }
 vortex-error = { workspace = true }
 vortex-mask = { workspace = true }
 vortex-scalar = { workspace = true }
+vortex-sparse = { workspace = true }
 
 [lints]
 workspace = true

encodings/dict/Cargo.toml

@@ -4,7 +4,7 @@ use num_traits::AsPrimitive;
 use vortex_array::accessor::ArrayAccessor;
 use vortex_array::aliases::hash_map::{DefaultHashBuilder, Entry, HashMap, HashTable, RandomState};
 use vortex_array::array::{
-    BinaryView, ConstantArray, PrimitiveArray, SparseArray, VarBinArray, VarBinViewArray,
+    BinaryView, ConstantArray, PrimitiveArray, VarBinArray, VarBinViewArray,
 };
 use vortex_array::validity::Validity;
 use vortex_array::variants::PrimitiveArrayTrait;
@@ -13,6 +13,7 @@ use vortex_buffer::{BufferMut, ByteBufferMut};
 use vortex_dtype::{match_each_native_ptype, DType, NativePType, Nullability, PType, ToBytes};
 use vortex_error::{vortex_bail, VortexExpect as _, VortexResult, VortexUnwrap};
 use vortex_scalar::Scalar;
+use vortex_sparse::SparseArray;
 
 use crate::DictArray;
 

encodings/dict/src/compress.rs

@@ -30,6 +30,7 @@ vortex-dtype = { workspace = true }
 vortex-error = { workspace = true }
 vortex-mask = { workspace = true }
 vortex-scalar = { workspace = true }
+vortex-sparse = { workspace = true }
 
 [dev-dependencies]
 criterion = { workspace = true }

encodings/fastlanes/Cargo.toml

@@ -1,5 +1,5 @@
 use num_traits::{PrimInt, WrappingAdd, WrappingSub};
-use vortex_array::array::{ConstantArray, PrimitiveArray, SparseArray};
+use vortex_array::array::{ConstantArray, PrimitiveArray};
 use vortex_array::stats::{trailing_zeros, ArrayStatistics, Stat};
 use vortex_array::validity::LogicalValidity;
 use vortex_array::variants::PrimitiveArrayTrait;
@@ -10,6 +10,7 @@ use vortex_dtype::{
 };
 use vortex_error::{vortex_bail, vortex_err, VortexExpect, VortexResult};
 use vortex_scalar::Scalar;
+use vortex_sparse::SparseArray;
 
 use crate::FoRArray;
 

encodings/fastlanes/src/for/compress.rs

@@ -0,0 +1,32 @@
+[package]
+name = "vortex-sparse"
+description = "Vortex Sparse array"
+version = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+keywords = { workspace = true }
+include = { workspace = true }
+edition = { workspace = true }
+rust-version = { workspace = true }
+categories = { workspace = true }
+readme = { workspace = true }
+
+[lints]
+workspace = true
+
+[dependencies]
+itertools = { workspace = true }
+rkyv = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+vortex-array = { workspace = true }
+vortex-buffer = { workspace = true }
+vortex-dtype = { workspace = true }
+vortex-error = { workspace = true }
+vortex-mask = { workspace = true }
+vortex-scalar = { workspace = true }
+
+[dev-dependencies]
+vortex-array = { workspace = true, features = ["test-harness"] }
+rstest = { workspace = true }

encodings/sparse/Cargo.toml

@@ -1,15 +1,13 @@
-use arrow_buffer::{ArrowNativeType, BooleanBuffer};
+use vortex_array::array::{BoolArray, BooleanBuffer, ConstantArray, PrimitiveArray};
+use vortex_array::patches::Patches;
+use vortex_array::validity::Validity;
+use vortex_array::{ArrayDType, ArrayLen, Canonical, IntoCanonical};
 use vortex_buffer::buffer;
 use vortex_dtype::{match_each_native_ptype, DType, NativePType, Nullability, PType};
 use vortex_error::{VortexError, VortexResult};
 use vortex_scalar::Scalar;
 
-use crate::array::primitive::PrimitiveArray;
-use crate::array::sparse::SparseArray;
-use crate::array::{BoolArray, ConstantArray};
-use crate::patches::Patches;
-use crate::validity::Validity;
-use crate::{ArrayDType, ArrayLen, Canonical, IntoCanonical};
+use crate::SparseArray;
 
 impl IntoCanonical for SparseArray {
     fn into_canonical(self) -> VortexResult<Canonical> {
@@ -59,7 +57,7 @@ fn canonicalize_sparse_bools(patches: Patches, fill_value: &Scalar) -> VortexRes
 }
 
 fn canonicalize_sparse_primitives<
-    T: NativePType + for<'a> TryFrom<&'a Scalar, Error = VortexError> + ArrowNativeType,
+    T: NativePType + for<'a> TryFrom<&'a Scalar, Error = VortexError>,
 >(
     patches: Patches,
     fill_value: &Scalar,
@@ -84,17 +82,16 @@ fn canonicalize_sparse_primitives<
 
 #[cfg(test)]
 mod test {
-    use arrow_buffer::BooleanBufferBuilder;
     use rstest::rstest;
+    use vortex_array::array::{BoolArray, BooleanBufferBuilder, PrimitiveArray};
+    use vortex_array::validity::Validity;
+    use vortex_array::{ArrayDType, IntoArrayData, IntoCanonical};
     use vortex_buffer::buffer;
     use vortex_dtype::{DType, Nullability, PType};
     use vortex_error::VortexExpect;
     use vortex_scalar::Scalar;
 
-    use crate::array::sparse::SparseArray;
-    use crate::array::{BoolArray, PrimitiveArray};
-    use crate::validity::Validity;
-    use crate::{ArrayDType, IntoArrayData, IntoCanonical};
+    use crate::SparseArray;
 
     #[rstest]
     #[case(Some(true))]

…rtex-array/src/array/sparse/canonical.rs encodings/sparse/src/canonical.rsvortex-array/src/array/sparse/canonical.rs renamed to encodings/sparse/src/canonical.rs vortex-array/src/array/sparse/canonical.rs renamed to encodings/sparse/src/canonical.rs

@@ -1,9 +1,10 @@
+use vortex_array::array::ConstantArray;
+use vortex_array::compute::{binary_numeric, BinaryNumericFn};
+use vortex_array::{ArrayData, ArrayLen, IntoArrayData};
 use vortex_error::{vortex_err, VortexResult};
 use vortex_scalar::BinaryNumericOperator;
 
-use crate::array::{ConstantArray, SparseArray, SparseEncoding};
-use crate::compute::{binary_numeric, BinaryNumericFn};
-use crate::{ArrayData, ArrayLen as _, IntoArrayData};
+use crate::{SparseArray, SparseEncoding};
 
 impl BinaryNumericFn<SparseArray> for SparseEncoding {
     fn binary_numeric(