Merge remote-tracking branch 'origin/develop' into ji/scalar-fn-binary-2

# Conflicts:
#	vortex-vector/src/binaryview/scalar.rs
#	vortex-vector/src/bool/scalar.rs
#	vortex-vector/src/datum.rs
#	vortex-vector/src/decimal/scalar.rs
#	vortex-vector/src/fixed_size_list/scalar.rs
#	vortex-vector/src/lib.rs
#	vortex-vector/src/listview/scalar.rs
#	vortex-vector/src/null/scalar.rs
#	vortex-vector/src/primitive/scalar.rs
#	vortex-vector/src/scalar.rs
#	vortex-vector/src/struct_/scalar.rs

Author: joseph-isaacs

.github/actions/setup-rust/action.yml

@@ -15,6 +15,10 @@ inputs:
   targets:
     description: "optional targets override (e.g. wasm32-unknown-unknown)"
     required: false
+  cache-suffix:
+    description: "optional suffix for cache key to isolate builds with different RUSTFLAGS (e.g. 'sanitizer')"
+    required: false
+    default: ""
 
 runs:
   using: "composite"
@@ -48,7 +52,7 @@ runs:
       uses: Swatinem/rust-cache@v2
       with:
         save-if: ${{ github.ref_name == 'develop' }}
-        shared-key: "rust-cache-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.toolchain-config.outputs.toolchain }}-${{ steps.toolchain-config.outputs.targets }}"
+        shared-key: "rust-cache-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.toolchain-config.outputs.toolchain }}-${{ steps.toolchain-config.outputs.targets }}${{ inputs.cache-suffix && format('-{0}', inputs.cache-suffix) || '' }}"
 
     - name: Rust Compile Cache
       uses: mozilla-actions/[email protected]

.github/runs-on.yml

@@ -0,0 +1,11 @@
+images:
+  vortex-ci-amd64:
+    platform: "linux"
+    arch: "x64"
+    name: "vortex-ci-*"
+    owner: "375504701696"
+  vortex-ci-arm64:
+    platform: "linux"
+    arch: "arm64"
+    name: "vortex-ci-*"
+    owner: "375504701696"

.github/workflows/ci.yml

@@ -421,6 +421,7 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           toolchain: nightly
           components: "rust-src, rustfmt, clippy, llvm-tools-preview"
+          cache-suffix: "sanitizer"
       - name: Install build dependencies
         run: |
           sudo apt-get update

vortex-array/src/arrays/filter/array.rs

@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+use vortex_mask::Mask;
+
+use crate::ArrayRef;
+use crate::stats::ArrayStats;
+
+#[derive(Clone, Debug)]
+pub struct FilterArray {
+    pub(super) child: ArrayRef,
+    pub(super) mask: Mask,
+    pub(super) stats: ArrayStats,
+}

vortex-array/src/arrays/filter/mod.rs

@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+mod array;
+mod vtable;
+
+pub use vtable::*;

vortex-array/src/arrays/filter/vtable.rs

@@ -0,0 +1,119 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+use std::hash::Hasher;
+
+use vortex_buffer::BufferHandle;
+use vortex_dtype::DType;
+use vortex_error::VortexResult;
+use vortex_error::vortex_bail;
+use vortex_mask::Mask;
+use vortex_vector::Vector;
+
+use crate::Array;
+use crate::ArrayBufferVisitor;
+use crate::ArrayChildVisitor;
+use crate::ArrayEq;
+use crate::ArrayHash;
+use crate::Precision;
+use crate::arrays::filter::array::FilterArray;
+use crate::execution::ExecutionCtx;
+use crate::serde::ArrayChildren;
+use crate::stats::StatsSetRef;
+use crate::vtable;
+use crate::vtable::ArrayId;
+use crate::vtable::ArrayVTable;
+use crate::vtable::ArrayVTableExt;
+use crate::vtable::BaseArrayVTable;
+use crate::vtable::NotSupported;
+use crate::vtable::VTable;
+use crate::vtable::VisitorVTable;
+
+vtable!(Filter);
+
+#[derive(Debug)]
+pub struct FilterVTable;
+
+impl VTable for FilterVTable {
+    type Array = FilterArray;
+    type Metadata = Mask;
+    type ArrayVTable = Self;
+    type CanonicalVTable = NotSupported;
+    type OperationsVTable = NotSupported;
+    type ValidityVTable = NotSupported;
+    type VisitorVTable = Self;
+    type ComputeVTable = NotSupported;
+    type EncodeVTable = NotSupported;
+
+    fn id(&self) -> ArrayId {
+        ArrayId::from("vortex.filter")
+    }
+
+    fn encoding(_array: &Self::Array) -> ArrayVTable {
+        FilterVTable.as_vtable()
+    }
+
+    fn metadata(array: &Self::Array) -> VortexResult<Self::Metadata> {
+        Ok(array.mask.clone())
+    }
+
+    fn serialize(_metadata: Self::Metadata) -> VortexResult<Option<Vec<u8>>> {
+        Ok(None)
+    }
+
+    fn deserialize(_bytes: &[u8]) -> VortexResult<Self::Metadata> {
+        vortex_bail!("Filter array is not serializable")
+    }
+
+    fn build(
+        &self,
+        dtype: &DType,
+        len: usize,
+        metadata: &Self::Metadata,
+        _buffers: &[BufferHandle],
+        children: &dyn ArrayChildren,
+    ) -> VortexResult<Self::Array> {
+        let child = children.get(0, dtype, len)?;
+        Ok(FilterArray {
+            child,
+            mask: metadata.clone(),
+            stats: Default::default(),
+        })
+    }
+
+    fn batch_execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult<Vector> {
+        let child = array.child.batch_execute(ctx)?;
+        Ok(vortex_compute::filter::Filter::filter(&child, &array.mask))
+    }
+}
+
+impl BaseArrayVTable<FilterVTable> for FilterVTable {
+    fn len(array: &FilterArray) -> usize {
+        array.mask.true_count()
+    }
+
+    fn dtype(array: &FilterArray) -> &DType {
+        array.child.dtype()
+    }
+
+    fn stats(array: &FilterArray) -> StatsSetRef<'_> {
+        array.stats.to_ref(array.as_ref())
+    }
+
+    fn array_hash<H: Hasher>(array: &FilterArray, state: &mut H, precision: Precision) {
+        array.child.array_hash(state, precision);
+        array.mask.array_hash(state, precision);
+    }
+
+    fn array_eq(array: &FilterArray, other: &FilterArray, precision: Precision) -> bool {
+        array.child.array_eq(&other.child, precision) && array.mask.array_eq(&other.mask, precision)
+    }
+}
+
+impl VisitorVTable<FilterVTable> for FilterVTable {
+    fn visit_buffers(_array: &FilterArray, _visitor: &mut dyn ArrayBufferVisitor) {}
+
+    fn visit_children(array: &FilterArray, visitor: &mut dyn ArrayChildVisitor) {
+        visitor.visit_child("child", &array.child);
+    }
+}

vortex-array/src/arrays/masked/vtable/mod.rs

@@ -7,11 +7,11 @@ mod operations;
 mod validity;
 
 use vortex_buffer::BufferHandle;
-use vortex_compute::mask::MaskValidity;
 use vortex_dtype::DType;
 use vortex_error::VortexResult;
 use vortex_error::vortex_bail;
 use vortex_vector::Vector;
+use vortex_vector::VectorOps;
 
 use crate::ArrayBufferVisitor;
 use crate::ArrayChildVisitor;
@@ -106,8 +106,9 @@ impl VTable for MaskedVTable {
     }
 
     fn batch_execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult<Vector> {
-        let vector = array.child().batch_execute(ctx)?;
-        Ok(MaskValidity::mask_validity(vector, &array.validity_mask()))
+        let mut vector = array.child().batch_execute(ctx)?;
+        vector.mask_validity(&array.validity_mask());
+        Ok(vector)
     }
 }
 

vortex-array/src/arrays/mod.rs

@@ -22,6 +22,7 @@ mod datetime;
 mod decimal;
 mod dict;
 mod extension;
+mod filter;
 mod fixed_size_list;
 mod list;
 mod listview;
@@ -35,6 +36,7 @@ mod varbinview;
 
 #[cfg(feature = "arbitrary")]
 pub mod arbitrary;
+// pub mod pipeline;
 // TODO(connor): Export exact types, not glob.
 
 pub use bool::*;
@@ -44,6 +46,7 @@ pub use datetime::*;
 pub use decimal::*;
 pub use dict::*;
 pub use extension::*;
+pub use filter::*;
 pub use fixed_size_list::*;
 pub use list::*;
 pub use listview::*;

vortex-array/src/arrays/scalar_fn/array.rs

@@ -2,11 +2,16 @@
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
 use vortex_dtype::DType;
+use vortex_error::VortexResult;
+use vortex_error::vortex_ensure;
 
+use crate::Array;
 use crate::ArrayRef;
+use crate::arrays::ScalarFnVTable;
 use crate::expr::functions::scalar::ScalarFn;
 use crate::stats::ArrayStats;
 use crate::vtable::ArrayVTable;
+use crate::vtable::ArrayVTableExt;
 
 #[derive(Clone, Debug)]
 pub struct ScalarFnArray {
@@ -18,3 +23,25 @@ pub struct ScalarFnArray {
     pub(super) children: Vec<ArrayRef>,
     pub(super) stats: ArrayStats,
 }
+
+impl ScalarFnArray {
+    /// Create a new ScalarFnArray from a scalar function and its children.
+    pub fn try_new(scalar_fn: ScalarFn, children: Vec<ArrayRef>, len: usize) -> VortexResult<Self> {
+        let arg_dtypes: Vec<_> = children.iter().map(|c| c.dtype().clone()).collect();
+        let dtype = scalar_fn.return_dtype(&arg_dtypes)?;
+
+        vortex_ensure!(
+            children.iter().all(|c| c.len() == len),
+            "ScalarFnArray must have children equal to the array length"
+        );
+
+        Ok(Self {
+            vtable: ScalarFnVTable::new(scalar_fn.vtable().clone()).into_vtable(),
+            scalar_fn,
+            dtype,
+            len,
+            children,
+            stats: Default::default(),
+        })
+    }
+}

vortex-array/src/arrays/scalar_fn/mod.rs

@@ -5,4 +5,5 @@ mod array;
 mod metadata;
 mod vtable;
 
+pub use array::*;
 pub use vtable::*;