More vector scalars

Signed-off-by: Nicholas Gates <[email protected]>

Author: gatesn

Cargo.lock

@@ -3,16 +3,16 @@
 
 use std::fmt::Formatter;
 
-use prost::Message;
-use vortex_dtype::{DType, match_each_float_ptype};
-use vortex_error::{VortexResult, vortex_bail, vortex_err};
-use vortex_proto::expr as pb;
-use vortex_scalar::Scalar;
-
 use crate::arrays::ConstantArray;
 use crate::expr::{ChildName, ExprId, Expression, ExpressionView, StatsCatalog, VTable, VTableExt};
 use crate::stats::Stat;
 use crate::{Array, ArrayRef, IntoArray};
+use prost::Message;
+use vortex_dtype::{match_each_float_ptype, DType};
+use vortex_error::{vortex_bail, vortex_err, VortexResult};
+use vortex_proto::expr as pb;
+use vortex_scalar::Scalar;
+use vortex_vector::Vector;
 
 /// Expression that represents a literal scalar value.
 pub struct Literal;
@@ -73,6 +73,14 @@ impl VTable for Literal {
         Ok(ConstantArray::new(expr.data().clone(), scope.len()).into_array())
     }
 
+    fn execute(
+        &self,
+        _expr: &ExpressionView<Self>,
+        _vector: &Vector,
+        _dtype: &DType,
+    ) -> VortexResult<Vector> {
+    }
+
     fn stat_expression(
         &self,
         expr: &ExpressionView<Self>,

vortex-array/src/expr/exprs/literal.rs

@@ -29,6 +29,7 @@ vortex-dtype = { workspace = true, features = ["arrow"] }
 vortex-error = { workspace = true }
 vortex-proto = { workspace = true, features = ["scalar"] }
 vortex-utils = { workspace = true }
+vortex-vector = { workspace = true }
 
 [dev-dependencies]
 rstest = { workspace = true }

vortex-scalar/Cargo.toml

@@ -28,6 +28,7 @@ mod struct_;
 #[cfg(test)]
 mod tests;
 mod utf8;
+mod vectors;
 
 pub use binary::*;
 pub use bool::*;

vortex-scalar/src/lib.rs

@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+//! Conversion logic from this "legacy" scalar crate to Vortex Vector scalars.
+
+use crate::Scalar;
+use vortex_dtype::{
+    match_each_decimal_value_type, match_each_native_ptype, DType, DecimalType, PrecisionScale,
+};
+use vortex_error::VortexExpect;
+use vortex_vector::binaryview::{BinaryViewScalar, StringScalar};
+use vortex_vector::bool::BoolScalar;
+use vortex_vector::decimal::DScalar;
+use vortex_vector::null::NullScalar;
+use vortex_vector::primitive::PScalar;
+
+impl Scalar {
+    pub fn into_vector(self) -> vortex_vector::Scalar {
+        match self.dtype() {
+            DType::Null => NullScalar.into(),
+            DType::Bool(_) => BoolScalar::new(self.as_bool().value()).into(),
+            DType::Primitive(ptype, _) => {
+                match_each_native_ptype!(ptype, |T| {
+                    PScalar::new(self.as_primitive().typed_value()).into()
+                })
+            }
+            DType::Decimal(dec_dtype, _) => {
+                let dscalar = self.as_decimal();
+                let dec_type = DecimalType::smallest_decimal_value_type(dec_dtype);
+                match_each_decimal_value_type!(dec_type, |D| {
+                    let ps = PrecisionScale::<D>::new(dec_dtype.precision(), dec_dtype.scale());
+                    DScalar::maybe_new(
+                        ps,
+                        dscalar
+                            .decimal_value()
+                            .map(|d| d.cast::<D>().vortex_expect("Failed to cast decimal value")),
+                    )
+                    .vortex_expect("Failed to create decimal scalar")
+                    .into()
+                })
+            }
+            DType::Utf8(_) => StringScalar::new(self.as_utf8().value()).into(),
+            DType::Binary(_) => BinaryViewScalar::new(self.as_binary().value()).into(),
+            DType::List(_, _) => {}
+            DType::FixedSizeList(_, _, _) => {}
+            DType::Struct(_, _) => {}
+            DType::Extension(_) => {}
+        }
+    }
+}

vortex-scalar/src/vectors.rs

@@ -10,8 +10,9 @@ use crate::{Scalar, ScalarOps, VectorMutOps};
 #[derive(Debug)]
 pub struct BinaryViewScalar<T: BinaryViewType>(Option<T::Scalar>);
 
-impl<T: BinaryViewType> From<Option<T::Scalar>> for BinaryViewScalar<T> {
-    fn from(value: Option<T::Scalar>) -> Self {
+impl<T: BinaryViewType> BinaryViewScalar<T> {
+    /// Creates a new binary view scalar with the given value.
+    pub fn new(value: Option<T::Scalar>) -> Self {
         Self(value)
     }
 }

vortex-vector/src/binaryview/scalar.rs

@@ -8,13 +8,13 @@ use std::ops::RangeBounds;
 use std::sync::Arc;
 
 use vortex_buffer::{Alignment, Buffer, ByteBuffer};
-use vortex_error::{VortexExpect, VortexResult, vortex_ensure};
+use vortex_error::{vortex_ensure, VortexExpect, VortexResult};
 use vortex_mask::Mask;
 
 use crate::binaryview::vector_mut::BinaryViewVectorMut;
-use crate::binaryview::view::{BinaryView, validate_views};
+use crate::binaryview::view::{validate_views, BinaryView};
 use crate::binaryview::{BinaryViewScalar, BinaryViewType};
-use crate::{Scalar, VectorOps};
+use crate::VectorOps;
 
 /// A variable-length binary vector.
 ///
@@ -193,6 +193,7 @@ impl<T: BinaryViewType> BinaryViewVector<T> {
 
 impl<T: BinaryViewType> VectorOps for BinaryViewVector<T> {
     type Mutable = BinaryViewVectorMut<T>;
+    type Scalar = BinaryViewScalar<T>;
 
     fn len(&self) -> usize {
         self.views.len()
@@ -202,9 +203,9 @@ impl<T: BinaryViewType> VectorOps for BinaryViewVector<T> {
         &self.validity
     }
 
-    fn scalar_at(&self, index: usize) -> Scalar {
+    fn scalar_at(&self, index: usize) -> BinaryViewScalar<T> {
         assert!(index < self.len());
-        BinaryViewScalar::<T>::from(self.get(index)).into()
+        BinaryViewScalar::<T>::new(self.get(index))
     }
 
     fn slice(&self, _range: impl RangeBounds<usize> + Clone + Debug) -> Self {
@@ -281,7 +282,7 @@ impl<T: BinaryViewType> VectorOps for BinaryViewVector<T> {
 mod tests {
     use std::sync::Arc;
 
-    use vortex_buffer::{ByteBuffer, buffer};
+    use vortex_buffer::{buffer, ByteBuffer};
     use vortex_mask::Mask;
 
     use crate::binaryview::view::BinaryView;

vortex-vector/src/binaryview/vector.rs

@@ -6,12 +6,12 @@
 use std::sync::Arc;
 
 use vortex_buffer::{BufferMut, ByteBuffer, ByteBufferMut};
-use vortex_error::{VortexExpect, VortexResult, vortex_ensure};
+use vortex_error::{vortex_ensure, VortexExpect, VortexResult};
 use vortex_mask::MaskMut;
 
-use crate::binaryview::BinaryViewType;
 use crate::binaryview::vector::BinaryViewVector;
-use crate::binaryview::view::{BinaryView, validate_views};
+use crate::binaryview::view::{validate_views, BinaryView};
+use crate::binaryview::{BinaryViewScalar, BinaryViewType};
 use crate::{VectorMutOps, VectorOps};
 
 // Default capacity for new string data buffers of 2MiB.
@@ -264,6 +264,20 @@ impl<T: BinaryViewType> VectorMutOps for BinaryViewVectorMut<T> {
         self.validity.append_n(false, n);
     }
 
+    fn append_zeros(&mut self, n: usize) {
+        self.views.push_n(BinaryView::empty_view(), n);
+        self.validity.append_n(true, n);
+    }
+
+    fn append_scalars(&mut self, scalar: &BinaryViewScalar<T>, n: usize) {
+        match scalar.value() {
+            None => self.append_nulls(n),
+            Some(v) => {
+                self.append_owned_values(v.clone(), n);
+            }
+        }
+    }
+
     fn freeze(mut self) -> BinaryViewVector<T> {
         // Freeze all components, close any in-progress views
         self.flush_open_buffer();
@@ -296,7 +310,7 @@ mod tests {
     use std::ops::Deref;
     use std::sync::Arc;
 
-    use vortex_buffer::{ByteBuffer, buffer, buffer_mut};
+    use vortex_buffer::{buffer, buffer_mut, ByteBuffer};
     use vortex_mask::{Mask, MaskMut};
 
     use crate::binaryview::view::BinaryView;

vortex-vector/src/binaryview/vector_mut.rs

@@ -8,10 +8,16 @@ use crate::{Scalar, ScalarOps, VectorMut, VectorMutOps};
 #[derive(Debug)]
 pub struct BoolScalar(Option<bool>);
 
-impl From<Option<bool>> for BoolScalar {
-    fn from(value: Option<bool>) -> Self {
+impl BoolScalar {
+    /// Creates a new bool scalar with the given value.
+    pub fn new(value: Option<bool>) -> Self {
         Self(value)
     }
+
+    /// Returns the value of the bool scalar, or `None` if the scalar is null.
+    pub fn value(&self) -> Option<bool> {
+        self.0
+    }
 }
 
 impl ScalarOps for BoolScalar {

vortex-vector/src/bool/scalar.rs

@@ -7,11 +7,11 @@ use std::fmt::Debug;
 use std::ops::RangeBounds;
 
 use vortex_buffer::BitBuffer;
-use vortex_error::{VortexExpect, VortexResult, vortex_ensure};
+use vortex_error::{vortex_ensure, VortexExpect, VortexResult};
 use vortex_mask::Mask;
 
-use crate::bool::BoolVectorMut;
-use crate::{Scalar, VectorOps};
+use crate::bool::{BoolScalar, BoolVectorMut};
+use crate::VectorOps;
 
 /// An immutable vector of boolean values.
 ///
@@ -74,6 +74,7 @@ impl BoolVector {
 
 impl VectorOps for BoolVector {
     type Mutable = BoolVectorMut;
+    type Scalar = BoolScalar;
 
     fn len(&self) -> usize {
         debug_assert!(self.validity.len() == self.bits.len());
@@ -84,13 +85,13 @@ impl VectorOps for BoolVector {
         &self.validity
     }
 
-    fn scalar_at(&self, index: usize) -> Scalar {
+    fn scalar_at(&self, index: usize) -> BoolScalar {
         assert!(index < self.len());
 
         let is_valid = self.validity.value(index);
         let value = is_valid.then(|| self.bits.value(index));
 
-        Scalar::Bool(value.into())
+        BoolScalar::new(value)
     }
 
     fn slice(&self, range: impl RangeBounds<usize> + Clone + Debug) -> Self {