Skip to content

Commit 6d7eea9

Browse files
gatesngatesn
authored
Use execution rather than evaluation from FlatLayout (#5638)
This is a WIP PR to flesh out the code path that uses vector execution rather than array compute functions. --------- Signed-off-by: Nicholas Gates <[email protected]>
1 parent 83a1570 commit 6d7eea9

File tree

97 files changed

+2203
-792
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

97 files changed

+2203
-792
lines changed

‎.github/workflows/bench-pr.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ name: PR Benchmarks
55
on:
66
pull_request:
77
types: [labeled, synchronize]
8-
branches: ["develop"]
98
workflow_dispatch: { }
109

1110
permissions:

‎.github/workflows/sql-pr.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ name: PR SQL Benchmarks
55
on:
66
pull_request:
77
types: [labeled, synchronize]
8-
branches: ["develop"]
98
workflow_dispatch: { }
109

1110
permissions:

‎Cargo.lock‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎encodings/alp/src/alp/array.rs‎

Lines changed: 37 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,9 @@ use vortex_array::DeserializeMetadata;
1515
use vortex_array::Precision;
1616
use vortex_array::ProstMetadata;
1717
use vortex_array::SerializeMetadata;
18-
use vortex_array::execution::ExecutionCtx;
18+
use vortex_array::kernel::BindCtx;
19+
use vortex_array::kernel::KernelRef;
20+
use vortex_array::kernel::kernel;
1921
use vortex_array::patches::Patches;
2022
use vortex_array::patches::PatchesMetadata;
2123
use vortex_array::serde::ArrayChildren;
@@ -41,7 +43,6 @@ use vortex_error::VortexExpect;
4143
use vortex_error::VortexResult;
4244
use vortex_error::vortex_bail;
4345
use vortex_error::vortex_ensure;
44-
use vortex_vector::Vector;
4546

4647
use crate::ALPFloat;
4748
use crate::alp::Exponents;
@@ -140,17 +141,16 @@ impl VTable for ALPVTable {
140141
)
141142
}
142143

143-
fn batch_execute(array: &ALPArray, ctx: &mut ExecutionCtx) -> VortexResult<Vector> {
144-
let encoded_vector = array.encoded().batch_execute(ctx)?;
145-
146-
let patches_vectors = if let Some(patches) = array.patches() {
144+
fn bind_kernel(array: &ALPArray, ctx: &mut BindCtx) -> VortexResult<KernelRef> {
145+
let encoded = array.encoded().bind_kernel(ctx)?;
146+
let patches_kernels = if let Some(patches) = array.patches() {
147147
Some((
148-
patches.indices().batch_execute(ctx)?,
149-
patches.values().batch_execute(ctx)?,
148+
patches.indices().bind_kernel(ctx)?,
149+
patches.values().bind_kernel(ctx)?,
150150
patches
151151
.chunk_offsets()
152152
.as_ref()
153-
.map(|co| co.batch_execute(ctx))
153+
.map(|co| co.bind_kernel(ctx))
154154
.transpose()?,
155155
))
156156
} else {
@@ -161,7 +161,24 @@ impl VTable for ALPVTable {
161161
let exponents = array.exponents();
162162

163163
match_each_alp_float_ptype!(array.dtype().as_ptype(), |T| {
164-
decompress_into_vector::<T>(encoded_vector, exponents, patches_vectors, patches_offset)
164+
Ok(kernel(move || {
165+
let encoded_vector = encoded.execute()?;
166+
let patches_vectors = match patches_kernels {
167+
Some((idx_kernel, val_kernel, co_kernel)) => Some((
168+
idx_kernel.execute()?,
169+
val_kernel.execute()?,
170+
co_kernel.map(|k| k.execute()).transpose()?,
171+
)),
172+
None => None,
173+
};
174+
175+
decompress_into_vector::<T>(
176+
encoded_vector,
177+
exponents,
178+
patches_vectors,
179+
patches_offset,
180+
)
181+
}))
165182
})
166183
}
167184
}
@@ -456,15 +473,18 @@ mod tests {
456473
use std::sync::LazyLock;
457474

458475
use rstest::rstest;
476+
use vortex_array::VectorExecutor;
459477
use vortex_array::arrays::PrimitiveArray;
478+
use vortex_array::session::ArraySession;
460479
use vortex_array::vtable::ValidityHelper;
461480
use vortex_dtype::PTypeDowncast;
462481
use vortex_session::VortexSession;
463482
use vortex_vector::VectorOps;
464483

465484
use super::*;
466485

467-
static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
486+
static SESSION: LazyLock<VortexSession> =
487+
LazyLock::new(|| VortexSession::empty().with::<ArraySession>());
468488

469489
#[rstest]
470490
#[case(0)]
@@ -480,7 +500,7 @@ mod tests {
480500
let values = PrimitiveArray::from_iter((0..size).map(|i| i as f32));
481501
let encoded = alp_encode(&values, None).unwrap();
482502

483-
let result_vector = encoded.to_array().execute(&SESSION).unwrap();
503+
let result_vector = encoded.to_array().execute_vector(&SESSION).unwrap();
484504
// Compare against the traditional array-based decompress path
485505
let expected = decompress_into_array(encoded);
486506

@@ -504,7 +524,7 @@ mod tests {
504524
let values = PrimitiveArray::from_iter((0..size).map(|i| i as f64));
505525
let encoded = alp_encode(&values, None).unwrap();
506526

507-
let result_vector = encoded.to_array().execute(&SESSION).unwrap();
527+
let result_vector = encoded.to_array().execute_vector(&SESSION).unwrap();
508528
// Compare against the traditional array-based decompress path
509529
let expected = decompress_into_array(encoded);
510530

@@ -534,7 +554,7 @@ mod tests {
534554
let encoded = alp_encode(&array, None).unwrap();
535555
assert!(encoded.patches().unwrap().array_len() > 0);
536556

537-
let result_vector = encoded.to_array().execute(&SESSION).unwrap();
557+
let result_vector = encoded.to_array().execute_vector(&SESSION).unwrap();
538558
// Compare against the traditional array-based decompress path
539559
let expected = decompress_into_array(encoded);
540560

@@ -562,7 +582,7 @@ mod tests {
562582
let array = PrimitiveArray::from_option_iter(values);
563583
let encoded = alp_encode(&array, None).unwrap();
564584

565-
let result_vector = encoded.to_array().execute(&SESSION).unwrap();
585+
let result_vector = encoded.to_array().execute_vector(&SESSION).unwrap();
566586
// Compare against the traditional array-based decompress path
567587
let expected = decompress_into_array(encoded);
568588

@@ -601,7 +621,7 @@ mod tests {
601621
let encoded = alp_encode(&array, None).unwrap();
602622
assert!(encoded.patches().unwrap().array_len() > 0);
603623

604-
let result_vector = encoded.to_array().execute(&SESSION).unwrap();
624+
let result_vector = encoded.to_array().execute_vector(&SESSION).unwrap();
605625
// Compare against the traditional array-based decompress path
606626
let expected = decompress_into_array(encoded);
607627

@@ -643,7 +663,7 @@ mod tests {
643663
let slice_len = slice_end - slice_start;
644664
let sliced_encoded = encoded.slice(slice_start..slice_end);
645665

646-
let result_vector = sliced_encoded.execute(&SESSION).unwrap();
666+
let result_vector = sliced_encoded.execute_vector_optimized(&SESSION).unwrap();
647667
let result_primitive = result_vector.into_primitive().into_f64();
648668

649669
for idx in 0..slice_len {

‎encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,7 @@ mod tests {
204204
use std::sync::LazyLock;
205205

206206
use vortex_array::IntoArray;
207+
use vortex_array::VectorExecutor;
207208
use vortex_array::assert_arrays_eq;
208209
use vortex_array::validity::Validity;
209210
use vortex_buffer::Buffer;
@@ -536,7 +537,7 @@ mod tests {
536537
let unpacked_array = unpack_array(&bitpacked);
537538

538539
// Method 3: Using the execute() method (this is what would be used in production).
539-
let executed = bitpacked.into_array().execute(&SESSION).unwrap();
540+
let executed = bitpacked.into_array().execute_vector(&SESSION).unwrap();
540541

541542
// All three should produce the same length.
542543
assert_eq!(vector_result.len(), array.len(), "vector length mismatch");
@@ -556,7 +557,10 @@ mod tests {
556557

557558
// Verify that the execute() method works correctly by comparing with unpack_array.
558559
// We convert unpack_array result to a vector using execute() to compare.
559-
let unpacked_executed = unpacked_array.into_array().execute(&SESSION).unwrap();
560+
let unpacked_executed = unpacked_array
561+
.into_array()
562+
.execute_vector(&SESSION)
563+
.unwrap();
560564
match (&executed, &unpacked_executed) {
561565
(Vector::Primitive(exec_pv), Vector::Primitive(unpack_pv)) => {
562566
assert_eq!(
@@ -593,7 +597,7 @@ mod tests {
593597
let sliced_bp = sliced.as_::<BitPackedVTable>();
594598
let vector_result = unpack_to_primitive_vector(sliced_bp);
595599
let unpacked_array = unpack_array(sliced_bp);
596-
let executed = sliced.execute(&SESSION).unwrap();
600+
let executed = sliced.execute_vector(&SESSION).unwrap();
597601

598602
assert_eq!(
599603
vector_result.len(),

‎encodings/fastlanes/src/bitpacking/vtable/mod.rs‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@
44
use vortex_array::DeserializeMetadata;
55
use vortex_array::ProstMetadata;
66
use vortex_array::SerializeMetadata;
7-
use vortex_array::execution::ExecutionCtx;
7+
use vortex_array::kernel::BindCtx;
8+
use vortex_array::kernel::KernelRef;
9+
use vortex_array::kernel::kernel;
810
use vortex_array::patches::Patches;
911
use vortex_array::patches::PatchesMetadata;
1012
use vortex_array::serde::ArrayChildren;
@@ -23,7 +25,6 @@ use vortex_error::VortexError;
2325
use vortex_error::VortexResult;
2426
use vortex_error::vortex_bail;
2527
use vortex_error::vortex_err;
26-
use vortex_vector::Vector;
2728
use vortex_vector::VectorMutOps;
2829

2930
use crate::BitPackedArray;
@@ -172,8 +173,11 @@ impl VTable for BitPackedVTable {
172173
)
173174
}
174175

175-
fn batch_execute(array: &BitPackedArray, _ctx: &mut ExecutionCtx) -> VortexResult<Vector> {
176-
Ok(unpack_to_primitive_vector(array).freeze().into())
176+
fn bind_kernel(array: &BitPackedArray, _ctx: &mut BindCtx) -> VortexResult<KernelRef> {
177+
let array = array.clone();
178+
Ok(kernel(move || {
179+
Ok(unpack_to_primitive_vector(&array).freeze().into())
180+
}))
177181
}
178182
}
179183

‎encodings/sequence/src/array.rs‎

Lines changed: 21 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,9 @@ use vortex_array::Precision;
1515
use vortex_array::ProstMetadata;
1616
use vortex_array::SerializeMetadata;
1717
use vortex_array::arrays::PrimitiveArray;
18-
use vortex_array::execution::ExecutionCtx;
18+
use vortex_array::kernel::BindCtx;
19+
use vortex_array::kernel::KernelRef;
20+
use vortex_array::kernel::kernel;
1921
use vortex_array::serde::ArrayChildren;
2022
use vortex_array::stats::ArrayStats;
2123
use vortex_array::stats::StatsSetRef;
@@ -48,7 +50,6 @@ use vortex_mask::Mask;
4850
use vortex_scalar::PValue;
4951
use vortex_scalar::Scalar;
5052
use vortex_scalar::ScalarValue;
51-
use vortex_vector::Vector;
5253
use vortex_vector::primitive::PVector;
5354

5455
vtable!(Sequence);
@@ -268,23 +269,28 @@ impl VTable for SequenceVTable {
268269
))
269270
}
270271

271-
fn batch_execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult<Vector> {
272+
fn bind_kernel(array: &Self::Array, _ctx: &mut BindCtx) -> VortexResult<KernelRef> {
273+
let array = array.clone();
274+
272275
Ok(match_each_native_ptype!(array.ptype(), |P| {
273276
let base = array.base().cast::<P>();
274277
let multiplier = array.multiplier().cast::<P>();
275278

276-
let values = if multiplier == <P>::one() {
277-
BufferMut::from_iter(
278-
(0..array.len()).map(|i| base + <P>::from_usize(i).vortex_expect("must fit")),
279-
)
280-
} else {
281-
BufferMut::from_iter(
282-
(0..array.len())
283-
.map(|i| base + <P>::from_usize(i).vortex_expect("must fit") * multiplier),
284-
)
285-
};
286-
287-
PVector::<P>::new(values.freeze(), Mask::new_true(array.len())).into()
279+
kernel(move || {
280+
let values =
281+
if multiplier == <P>::one() {
282+
BufferMut::from_iter(
283+
(0..array.len())
284+
.map(|i| base + <P>::from_usize(i).vortex_expect("must fit")),
285+
)
286+
} else {
287+
BufferMut::from_iter((0..array.len()).map(|i| {
288+
base + <P>::from_usize(i).vortex_expect("must fit") * multiplier
289+
}))
290+
};
291+
292+
Ok(PVector::<P>::new(values.freeze(), Mask::new_true(array.len())).into())
293+
})
288294
}))
289295
}
290296
}

0 commit comments

Comments
 (0)