Skip to content

Commit d8cab4c

Browse files
joseph-isaacsjoseph-isaacs
authored
feat[gpu]: use BufferHandle to the array vtable build (#5565)
We want to have array which are backed by either CPU or GPU/device buffers. The first stage is to create that enum and change the build array method. Then add this to the segment source and array construction. Then we can go through each array and support non-cpu buffers. There is an open question if we want to have this logical-scan-plann have buffers or BufferIDs --------- Signed-off-by: Joe Isaacs <[email protected]>
1 parent 60d7520 commit d8cab4c

File tree

41 files changed

+188
-95
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+188
-95
lines changed

‎encodings/alp/src/alp/array.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ use vortex_array::vtable::VTable;
3232
use vortex_array::vtable::ValidityChild;
3333
use vortex_array::vtable::ValidityVTableFromChild;
3434
use vortex_array::vtable::VisitorVTable;
35-
use vortex_buffer::ByteBuffer;
35+
use vortex_buffer::BufferHandle;
3636
use vortex_dtype::DType;
3737
use vortex_dtype::PType;
3838
use vortex_error::VortexError;
@@ -97,7 +97,7 @@ impl VTable for ALPVTable {
9797
dtype: &DType,
9898
len: usize,
9999
metadata: &Self::Metadata,
100-
_buffers: &[ByteBuffer],
100+
_buffers: &[BufferHandle],
101101
children: &dyn ArrayChildren,
102102
) -> VortexResult<ALPArray> {
103103
let encoded_ptype = match &dtype {

‎encodings/alp/src/alp_rd/array.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ use vortex_array::vtable::ValidityChild;
3737
use vortex_array::vtable::ValidityVTableFromChild;
3838
use vortex_array::vtable::VisitorVTable;
3939
use vortex_buffer::Buffer;
40-
use vortex_buffer::ByteBuffer;
40+
use vortex_buffer::BufferHandle;
4141
use vortex_dtype::DType;
4242
use vortex_dtype::Nullability;
4343
use vortex_dtype::PType;
@@ -122,7 +122,7 @@ impl VTable for ALPRDVTable {
122122
dtype: &DType,
123123
len: usize,
124124
metadata: &Self::Metadata,
125-
_buffers: &[ByteBuffer],
125+
_buffers: &[BufferHandle],
126126
children: &dyn ArrayChildren,
127127
) -> VortexResult<ALPRDArray> {
128128
if children.len() < 2 {

‎encodings/bytebool/src/array.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ use vortex_array::vtable::ValidityHelper;
3232
use vortex_array::vtable::ValidityVTableFromValidityHelper;
3333
use vortex_array::vtable::VisitorVTable;
3434
use vortex_buffer::BitBuffer;
35+
use vortex_buffer::BufferHandle;
3536
use vortex_buffer::ByteBuffer;
3637
use vortex_dtype::DType;
3738
use vortex_error::VortexResult;
@@ -80,7 +81,7 @@ impl VTable for ByteBoolVTable {
8081
dtype: &DType,
8182
len: usize,
8283
_metadata: &Self::Metadata,
83-
buffers: &[ByteBuffer],
84+
buffers: &[BufferHandle],
8485
children: &dyn ArrayChildren,
8586
) -> VortexResult<ByteBoolArray> {
8687
let validity = if children.is_empty() {
@@ -95,7 +96,7 @@ impl VTable for ByteBoolVTable {
9596
if buffers.len() != 1 {
9697
vortex_bail!("Expected 1 buffer, got {}", buffers.len());
9798
}
98-
let buffer = buffers[0].clone();
99+
let buffer = buffers[0].clone().try_to_bytes()?;
99100

100101
Ok(ByteBoolArray::new(buffer, validity))
101102
}

‎encodings/datetime-parts/src/array.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ use vortex_array::vtable::VTable;
3030
use vortex_array::vtable::ValidityChild;
3131
use vortex_array::vtable::ValidityVTableFromChild;
3232
use vortex_array::vtable::VisitorVTable;
33-
use vortex_buffer::ByteBuffer;
33+
use vortex_buffer::BufferHandle;
3434
use vortex_dtype::DType;
3535
use vortex_dtype::Nullability;
3636
use vortex_dtype::PType;
@@ -115,7 +115,7 @@ impl VTable for DateTimePartsVTable {
115115
dtype: &DType,
116116
len: usize,
117117
metadata: &Self::Metadata,
118-
_buffers: &[ByteBuffer],
118+
_buffers: &[BufferHandle],
119119
children: &dyn ArrayChildren,
120120
) -> VortexResult<DateTimePartsArray> {
121121
if children.len() != 3 {

‎encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ use vortex_array::vtable::ValidityChild;
3636
use vortex_array::vtable::ValidityHelper;
3737
use vortex_array::vtable::ValidityVTableFromChild;
3838
use vortex_array::vtable::VisitorVTable;
39-
use vortex_buffer::ByteBuffer;
39+
use vortex_buffer::BufferHandle;
4040
use vortex_dtype::DType;
4141
use vortex_dtype::DecimalDType;
4242
use vortex_dtype::PType;
@@ -99,7 +99,7 @@ impl VTable for DecimalBytePartsVTable {
9999
dtype: &DType,
100100
len: usize,
101101
metadata: &Self::Metadata,
102-
_buffers: &[ByteBuffer],
102+
_buffers: &[BufferHandle],
103103
children: &dyn ArrayChildren,
104104
) -> VortexResult<DecimalBytePartsArray> {
105105
let Some(decimal_dtype) = dtype.as_decimal_opt() else {

‎encodings/fastlanes/src/bitpacking/vtable/mod.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ use vortex_array::vtable::ArrayVTableExt;
1616
use vortex_array::vtable::NotSupported;
1717
use vortex_array::vtable::VTable;
1818
use vortex_array::vtable::ValidityVTableFromValidityHelper;
19-
use vortex_buffer::ByteBuffer;
19+
use vortex_buffer::BufferHandle;
2020
use vortex_dtype::DType;
2121
use vortex_dtype::PType;
2222
use vortex_error::VortexError;
@@ -100,13 +100,13 @@ impl VTable for BitPackedVTable {
100100
dtype: &DType,
101101
len: usize,
102102
metadata: &Self::Metadata,
103-
buffers: &[ByteBuffer],
103+
buffers: &[BufferHandle],
104104
children: &dyn ArrayChildren,
105105
) -> VortexResult<BitPackedArray> {
106106
if buffers.len() != 1 {
107107
vortex_bail!("Expected 1 buffer, got {}", buffers.len());
108108
}
109-
let packed = buffers[0].clone();
109+
let packed = buffers[0].clone().try_to_bytes()?;
110110

111111
let load_validity = |child_idx: usize| {
112112
if children.len() == child_idx {

‎encodings/fastlanes/src/delta/vtable/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ use vortex_array::vtable::ArrayVTableExt;
1212
use vortex_array::vtable::NotSupported;
1313
use vortex_array::vtable::VTable;
1414
use vortex_array::vtable::ValidityVTableFromChildSliceHelper;
15-
use vortex_buffer::ByteBuffer;
15+
use vortex_buffer::BufferHandle;
1616
use vortex_dtype::DType;
1717
use vortex_dtype::PType;
1818
use vortex_dtype::match_each_unsigned_integer_ptype;
@@ -80,7 +80,7 @@ impl VTable for DeltaVTable {
8080
dtype: &DType,
8181
len: usize,
8282
metadata: &Self::Metadata,
83-
_buffers: &[ByteBuffer],
83+
_buffers: &[BufferHandle],
8484
children: &dyn ArrayChildren,
8585
) -> VortexResult<DeltaArray> {
8686
assert_eq!(children.len(), 2);

‎encodings/fastlanes/src/for/vtable/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ use vortex_array::vtable::ArrayVTableExt;
1414
use vortex_array::vtable::NotSupported;
1515
use vortex_array::vtable::VTable;
1616
use vortex_array::vtable::ValidityVTableFromChild;
17-
use vortex_buffer::ByteBuffer;
17+
use vortex_buffer::BufferHandle;
1818
use vortex_dtype::DType;
1919
use vortex_error::VortexResult;
2020
use vortex_error::vortex_bail;
@@ -74,7 +74,7 @@ impl VTable for FoRVTable {
7474
dtype: &DType,
7575
len: usize,
7676
metadata: &Self::Metadata,
77-
_buffers: &[ByteBuffer],
77+
_buffers: &[BufferHandle],
7878
children: &dyn ArrayChildren,
7979
) -> VortexResult<FoRArray> {
8080
if children.len() != 1 {

‎encodings/fastlanes/src/rle/vtable/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ use vortex_array::vtable::ArrayVTableExt;
1111
use vortex_array::vtable::NotSupported;
1212
use vortex_array::vtable::VTable;
1313
use vortex_array::vtable::ValidityVTableFromChildSliceHelper;
14-
use vortex_buffer::ByteBuffer;
14+
use vortex_buffer::BufferHandle;
1515
use vortex_dtype::DType;
1616
use vortex_dtype::Nullability;
1717
use vortex_dtype::PType;
@@ -90,7 +90,7 @@ impl VTable for RLEVTable {
9090
dtype: &DType,
9191
len: usize,
9292
metadata: &Self::Metadata,
93-
_buffers: &[ByteBuffer],
93+
_buffers: &[BufferHandle],
9494
children: &dyn ArrayChildren,
9595
) -> VortexResult<RLEArray> {
9696
let metadata = &metadata.0;

‎encodings/fsst/src/array.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ use vortex_array::vtable::ValidityChild;
3838
use vortex_array::vtable::ValidityVTableFromChild;
3939
use vortex_array::vtable::VisitorVTable;
4040
use vortex_buffer::Buffer;
41-
use vortex_buffer::ByteBuffer;
41+
use vortex_buffer::BufferHandle;
4242
use vortex_dtype::DType;
4343
use vortex_dtype::Nullability;
4444
use vortex_dtype::PType;
@@ -108,14 +108,14 @@ impl VTable for FSSTVTable {
108108
dtype: &DType,
109109
len: usize,
110110
metadata: &Self::Metadata,
111-
buffers: &[ByteBuffer],
111+
buffers: &[BufferHandle],
112112
children: &dyn ArrayChildren,
113113
) -> VortexResult<FSSTArray> {
114114
if buffers.len() != 2 {
115115
vortex_bail!(InvalidArgument: "Expected 2 buffers, got {}", buffers.len());
116116
}
117-
let symbols = Buffer::<Symbol>::from_byte_buffer(buffers[0].clone());
118-
let symbol_lengths = Buffer::<u8>::from_byte_buffer(buffers[1].clone());
117+
let symbols = Buffer::<Symbol>::from_byte_buffer(buffers[0].clone().try_to_bytes()?);
118+
let symbol_lengths = Buffer::<u8>::from_byte_buffer(buffers[1].clone().try_to_bytes()?);
119119

120120
if children.len() != 2 {
121121
vortex_bail!(InvalidArgument: "Expected 2 children, got {}", children.len());

0 commit comments

Comments
 (0)