11// SPDX-License-Identifier: Apache-2.0
22// SPDX-FileCopyrightText: Copyright the Vortex contributors
33
4- //! Vortex crate containing vectorized operator processing.
5- //!
6- //! This module contains experiments into pipelined data processing within Vortex.
7- //!
8- //! Arrays (and eventually Layouts) will be convertible into a [`Kernel`] that can then be
9- //! exported into a [`ViewMut`] one chunk of [`N`] elements at a time. This allows us to keep
10- //! compute largely within the L1 cache, as well as to write out canonical data into externally
11- //! provided buffers.
12- //!
13- //! Each chunk is represented in a canonical physical form, as determined by the logical
14- //! [`vortex_dtype::DType`] of the array. This provides a predicate base on which to perform
15- //! compute. Unlike DuckDB and other vectorized systems, we force a single canonical representation
16- //! instead of supporting multiple encodings because compute push-down is applied a priori to the
17- //! logical representation.
18- //!
19- //! It is a work-in-progress and is not yet used in production.
20-
214pub mod bit_view;
225pub mod source_driver;
236
@@ -38,8 +21,11 @@ pub const N_BYTES: usize = N / 8;
3821/// Number of usize words needed to store N bits
3922pub const N_WORDS : usize = N / usize:: BITS as usize ;
4023
41- /// Returned by an array to indicate that it can be executed in a pipelined fashion.
42- pub trait PipelinedOperator : Array {
24+ /// Indicates that an array supports acting as a transformation node in a pipelined execution.
25+ ///
26+ /// That is, it has one or more child arrays for which each input element produces a single output
27+ /// element. See [`PipelineSource`] for nodes that have zero pipelined children.
28+ pub trait PipelineTransform : Deref < Target = dyn Array > {
4329 // Whether this operator works by mutating its first child in-place.
4430 //
4531 // If `true`, the operator is invoked with the first child's input data passed via the
@@ -56,36 +42,39 @@ pub trait PipelinedOperator: Array {
5642/// computed before pipelined execution begins.
5743fn is_pipelined_child ( & self , child_idx : usize ) -> bool ;
5844
59- /// Bind the operator into a [`Kernel `] for pipelined execution.
45+ /// Bind the operator into a [`TransformKernel `] for pipelined execution.
6046///
6147/// The provided [`BindContext`] can be used to obtain vector IDs for pipelined children and
6248/// batch IDs for batch children. Each child can only be bound once.
63- fn bind ( & self , ctx : & mut dyn BindContext ) -> VortexResult < Box < dyn OperatorKernel > > ;
49+ fn bind ( & self , ctx : & mut dyn BindContext ) -> VortexResult < Box < dyn TransformKernel > > ;
6450}
6551
66- pub trait PipelinedSource : Deref < Target = dyn Array > {
67- /// Bind the operator into a [`Kernel`] for pipelined execution.
52+ /// Indicates that an array supports acting as a source node in a pipelined execution.
53+ pub trait PipelineSource : Deref < Target = dyn Array > {
54+ /// Bind the operator into a [`SourceKernel`] for pipelined execution.
6855///
6956/// The provided [`BindContext`] can be used to obtain vector IDs for pipelined children and
7057/// batch IDs for batch children. Each child can only be bound once.
71- fn bind_source ( & self , ctx : & mut dyn BindContext ) -> VortexResult < Box < dyn SourceKernel > > ;
58+ fn bind ( & self , ctx : & mut dyn BindContext ) -> VortexResult < Box < dyn SourceKernel > > ;
7259}
7360
7461/// The context used when binding an operator for execution.
7562pub trait BindContext {
7663 /// Returns a [`VectorId`] that can be passed to the [`KernelContext`] within the body of
77- /// the [`Kernel`] to access the given child as a pipelined input vector.
64+ /// the kernel to access the given child as a pipelined input vector.
7865///
7966/// # Panics
8067///
81- /// If the child index requested here was not listed in [`Pipelined::pipelined_children`].
68+ /// If the child index requested here was not marked as a pipelined child in
69+ /// [`PipelineTransform::is_pipelined_child`].
8270fn pipelined_input ( & self , child_idx : usize ) -> VectorId ;
8371
8472 /// Returns the batch input vector for the given child.
8573///
8674/// # Panics
8775///
88- /// If the child index requested here was listed in [`Pipelined::pipelined_children`].
76+ /// If the child index requested here was marked as a pipelined child in
77+ /// [`PipelineTransform::is_pipelined_child`].
8978fn batch_input ( & self , child_idx : usize ) -> Vector ;
9079}
9180
@@ -115,7 +104,12 @@ pub trait SourceKernel: Send {
115104/// For example, if `n` is 3, then the kernel should skip over `3 * N` elements of input data.
116105fn skip ( & mut self , n : usize ) ;
117106
118- /// Attempts to perform a single step of the operator, writing data to the output vector.
107+ /// Attempts to perform a single step of the operator, appending data to the output vector.
108+ ///
109+ /// The provided selection mask indicates which elements of the current chunk should be
110+ /// appended to the output vector.
111+ ///
112+ /// The provided output vector is guaranteed to have at least `N` elements of capacity.
119113fn step (
120114 & mut self ,
121115 ctx : & KernelContext ,
@@ -124,12 +118,13 @@ pub trait SourceKernel: Send {
124118 ) -> VortexResult < ( ) > ;
125119}
126120
127- pub trait OperatorKernel : Send {
128- /// Attempts to perform a single step of the operator, writing data to the output vector.
121+ pub trait TransformKernel : Send {
122+ /// Attempts to perform a single step of the operator, appending data to the output vector.
123+ ///
124+ /// The input vectors can be accessed via the provided `KernelContext`.
129125///
130- /// The output vector has length equal to the number of valid elements in the input vectors.
131- /// This number of values should be written to the output vector.
132- fn step ( & self , ctx : & KernelContext , out : & mut VectorMut ) -> VortexResult < ( ) > ;
126+ /// The provided output vector is guaranteed to have at least `N` elements of capacity.
127+ fn step ( & mut self , ctx : & KernelContext , out : & mut VectorMut ) -> VortexResult < ( ) > ;
133128}
134129
135130/// Context passed to kernels during execution, providing access to vectors.
0 commit comments