Skip to content

Commit 8bdc860

Browse files
gatesngatesn
authored
Merge branch 'develop' into operators-bench
2 parents 60de35a + 0562416 commit 8bdc860

File tree

41 files changed

+458
-861
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+458
-861
lines changed

‎.github/workflows/bench.yml‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -178,20 +178,4 @@ jobs:
178178
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
179179
"scale_factor": "--scale-factor 100"
180180
},
181-
{
182-
"id": "gharchive-nvme",
183-
"subcommand": "gharchive",
184-
"name": "GitHub Archive (NVMe)",
185-
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
186-
"scale_factor": "--scale-factor 100"
187-
},
188-
{
189-
"id": "gharchive-s3",
190-
"subcommand": "gharchive",
191-
"name": "GitHub Archive (S3)",
192-
"local_dir": "bench-vortex/data/gharchive",
193-
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/gharchive/",
194-
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
195-
"scale_factor": "--scale-factor 100"
196-
},
197181
]

‎.github/workflows/sql-benchmarks.yml‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -88,22 +88,6 @@ on:
8888
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
8989
"scale_factor": "--scale-factor 100"
9090
},
91-
{
92-
"id": "gharchive-nvme",
93-
"subcommand": "gharchive",
94-
"name": "GitHub Archive (NVMe)",
95-
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
96-
"scale_factor": "--scale-factor 100"
97-
},
98-
{
99-
"id": "gharchive-s3",
100-
"subcommand": "gharchive",
101-
"name": "GitHub Archive (S3)",
102-
"local_dir": "bench-vortex/data/gharchive",
103-
"remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/gharchive/",
104-
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
105-
"scale_factor": "--scale-factor 100"
106-
},
10791
]
10892
10993
jobs:

‎encodings/alp/src/alp/array.rs‎

Lines changed: 9 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,10 @@ use vortex_array::ArrayHash;
1212
use vortex_array::ArrayRef;
1313
use vortex_array::Canonical;
1414
use vortex_array::DeserializeMetadata;
15+
use vortex_array::ExecutionCtx;
1516
use vortex_array::Precision;
1617
use vortex_array::ProstMetadata;
1718
use vortex_array::SerializeMetadata;
18-
use vortex_array::kernel::BindCtx;
19-
use vortex_array::kernel::KernelRef;
20-
use vortex_array::kernel::kernel;
2119
use vortex_array::patches::Patches;
2220
use vortex_array::patches::PatchesMetadata;
2321
use vortex_array::serde::ArrayChildren;
@@ -44,6 +42,7 @@ use vortex_error::VortexResult;
4442
use vortex_error::vortex_bail;
4543
use vortex_error::vortex_ensure;
4644
use vortex_error::vortex_err;
45+
use vortex_vector::Vector;
4746

4847
use crate::ALPFloat;
4948
use crate::alp::Exponents;
@@ -187,16 +186,16 @@ impl VTable for ALPVTable {
187186
Ok(())
188187
}
189188

190-
fn bind_kernel(array: &ALPArray, ctx: &mut BindCtx) -> VortexResult<KernelRef> {
191-
let encoded = array.encoded().bind_kernel(ctx)?;
192-
let patches_kernels = if let Some(patches) = array.patches() {
189+
fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult<Vector> {
190+
let encoded = array.encoded().execute(ctx)?;
191+
let patches = if let Some(patches) = array.patches() {
193192
Some((
194-
patches.indices().bind_kernel(ctx)?,
195-
patches.values().bind_kernel(ctx)?,
193+
patches.indices().execute(ctx)?,
194+
patches.values().execute(ctx)?,
196195
patches
197196
.chunk_offsets()
198197
.as_ref()
199-
.map(|co| co.bind_kernel(ctx))
198+
.map(|co| co.execute(ctx))
200199
.transpose()?,
201200
))
202201
} else {
@@ -207,24 +206,7 @@ impl VTable for ALPVTable {
207206
let exponents = array.exponents();
208207

209208
match_each_alp_float_ptype!(array.dtype().as_ptype(), |T| {
210-
Ok(kernel(move || {
211-
let encoded_vector = encoded.execute()?;
212-
let patches_vectors = match patches_kernels {
213-
Some((idx_kernel, val_kernel, co_kernel)) => Some((
214-
idx_kernel.execute()?,
215-
val_kernel.execute()?,
216-
co_kernel.map(|k| k.execute()).transpose()?,
217-
)),
218-
None => None,
219-
};
220-
221-
decompress_into_vector::<T>(
222-
encoded_vector,
223-
exponents,
224-
patches_vectors,
225-
patches_offset,
226-
)
227-
}))
209+
decompress_into_vector::<T>(encoded, exponents, patches, patches_offset)
228210
})
229211
}
230212
}

‎encodings/fastlanes/src/bitpacking/vtable/mod.rs‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,9 @@
33

44
use vortex_array::ArrayRef;
55
use vortex_array::DeserializeMetadata;
6+
use vortex_array::ExecutionCtx;
67
use vortex_array::ProstMetadata;
78
use vortex_array::SerializeMetadata;
8-
use vortex_array::kernel::BindCtx;
9-
use vortex_array::kernel::KernelRef;
10-
use vortex_array::kernel::kernel;
119
use vortex_array::patches::Patches;
1210
use vortex_array::patches::PatchesMetadata;
1311
use vortex_array::serde::ArrayChildren;
@@ -27,6 +25,7 @@ use vortex_error::VortexResult;
2725
use vortex_error::vortex_bail;
2826
use vortex_error::vortex_ensure;
2927
use vortex_error::vortex_err;
28+
use vortex_vector::Vector;
3029
use vortex_vector::VectorMutOps;
3130

3231
use crate::BitPackedArray;
@@ -244,11 +243,8 @@ impl VTable for BitPackedVTable {
244243
)
245244
}
246245

247-
fn bind_kernel(array: &BitPackedArray, _ctx: &mut BindCtx) -> VortexResult<KernelRef> {
248-
let array = array.clone();
249-
Ok(kernel(move || {
250-
Ok(unpack_to_primitive_vector(&array).freeze().into())
251-
}))
246+
fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult<Vector> {
247+
Ok(unpack_to_primitive_vector(array).freeze().into())
252248
}
253249
}
254250

‎encodings/fastlanes/src/delta/vtable/kernel.rs‎

Lines changed: 0 additions & 103 deletions
This file was deleted.

‎encodings/fastlanes/src/delta/vtable/mod.rs‎

Lines changed: 56 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

4+
use fastlanes::Delta;
45
use fastlanes::FastLanes;
6+
use fastlanes::Transpose;
7+
use num_traits::WrappingAdd;
58
use prost::Message;
69
use vortex_array::ArrayRef;
10+
use vortex_array::ExecutionCtx;
711
use vortex_array::ProstMetadata;
8-
use vortex_array::kernel::BindCtx;
9-
use vortex_array::kernel::KernelRef;
1012
use vortex_array::serde::ArrayChildren;
1113
use vortex_array::vtable;
1214
use vortex_array::vtable::ArrayId;
@@ -17,18 +19,24 @@ use vortex_array::vtable::VTable;
1719
use vortex_array::vtable::ValidityVTableFromChildSliceHelper;
1820
use vortex_buffer::BufferHandle;
1921
use vortex_dtype::DType;
22+
use vortex_dtype::NativePType;
2023
use vortex_dtype::PType;
24+
use vortex_dtype::PTypeDowncastExt;
2125
use vortex_dtype::match_each_unsigned_integer_ptype;
2226
use vortex_error::VortexResult;
2327
use vortex_error::vortex_ensure;
2428
use vortex_error::vortex_err;
29+
use vortex_error::vortex_panic;
30+
use vortex_mask::Mask;
31+
use vortex_vector::Vector;
32+
use vortex_vector::primitive::PVector;
33+
use vortex_vector::primitive::PrimitiveVector;
2534

26-
use self::kernel::DeltaKernel;
2735
use crate::DeltaArray;
36+
use crate::delta::array::delta_decompress::decompress_primitive;
2837

2938
mod array;
3039
mod canonical;
31-
mod kernel;
3240
mod operations;
3341
mod validity;
3442
mod visitor;
@@ -122,25 +130,58 @@ impl VTable for DeltaVTable {
122130
DeltaArray::try_new(bases, deltas, metadata.0.offset as usize, len)
123131
}
124132

125-
fn bind_kernel(array: &DeltaArray, ctx: &mut BindCtx) -> VortexResult<KernelRef> {
126-
let bases_kernel = array.bases().bind_kernel(ctx)?;
127-
let deltas_kernel = array.deltas().bind_kernel(ctx)?;
133+
fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult<Vector> {
134+
let bases = array.bases().execute(ctx)?.into_primitive();
135+
let deltas = array.deltas().execute(ctx)?.into_primitive();
128136

129137
let start = array.offset();
130138
let end = start + array.len();
131-
132139
let validity = array.deltas().validity_mask().slice(start..end);
133140

134-
Ok(Box::new(DeltaKernel {
135-
bases_kernel,
136-
deltas_kernel,
137-
start,
138-
end,
139-
validity,
140-
}))
141+
Ok(match bases {
142+
PrimitiveVector::U8(pv) => {
143+
decompress::<u8, { u8::LANES }>(&pv, &deltas, start, end, validity)
144+
}
145+
PrimitiveVector::U16(pv) => {
146+
decompress::<u16, { u16::LANES }>(&pv, &deltas, start, end, validity)
147+
}
148+
PrimitiveVector::U32(pv) => {
149+
decompress::<u32, { u32::LANES }>(&pv, &deltas, start, end, validity)
150+
}
151+
PrimitiveVector::U64(pv) => {
152+
decompress::<u64, { u64::LANES }>(&pv, &deltas, start, end, validity)
153+
}
154+
PrimitiveVector::I8(_)
155+
| PrimitiveVector::I16(_)
156+
| PrimitiveVector::I32(_)
157+
| PrimitiveVector::I64(_)
158+
| PrimitiveVector::F16(_)
159+
| PrimitiveVector::F32(_)
160+
| PrimitiveVector::F64(_) => {
161+
vortex_panic!("Tried to match a non-unsigned vector in an unsigned match statement")
162+
}
163+
})
141164
}
142165
}
143166

167+
/// Decompresses delta-encoded data for a specific primitive type.
168+
fn decompress<T, const LANES: usize>(
169+
bases: &PVector<T>,
170+
deltas: &PrimitiveVector,
171+
start: usize,
172+
end: usize,
173+
validity: Mask,
174+
) -> Vector
175+
where
176+
T: NativePType + Delta + Transpose + WrappingAdd,
177+
{
178+
let buffer = decompress_primitive::<T, LANES>(bases.as_ref(), deltas.downcast::<T>().as_ref());
179+
let buffer = buffer.slice(start..end);
180+
181+
// SAFETY: We slice the buffer and the validity by the same range.
182+
unsafe { PVector::<T>::new_unchecked(buffer, validity) }.into()
183+
}
184+
144185
#[derive(Debug)]
145186
pub struct DeltaVTable;
146187

0 commit comments

Comments
 (0)