Skip to content

Commit 1d20f04

Browse files
joseph-isaacsjoseph-isaacs
committed
wip
Signed-off-by: Joe Isaacs <[email protected]>
1 parent 5a77496 commit 1d20f04

File tree

70 files changed

+2449
-2298
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

70 files changed

+2449
-2298
lines changed

‎encodings/alp/src/alp/array.rs‎

Lines changed: 110 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,25 +4,30 @@
44
use std::fmt::Debug;
55
use std::hash::Hash;
66

7-
use vortex_array::patches::Patches;
7+
use vortex_array::patches::{Patches, PatchesMetadata};
8+
use vortex_array::serde::ArrayChildren;
89
use vortex_array::stats::{ArrayStats, StatsSetRef};
910
use vortex_array::vtable::{
10-
ArrayVTable, CanonicalVTable, NotSupported, VTable, ValidityChild, ValidityVTableFromChild,
11+
ArrayVTable, CanonicalVTable, EncodeVTable, NotSupported, VTable, ValidityChild,
12+
ValidityVTableFromChild, VisitorVTable,
1113
};
1214
use vortex_array::{
13-
Array, ArrayEq, ArrayHash, ArrayRef, Canonical, EncodingId, EncodingRef, Precision, vtable,
15+
Array, ArrayBufferVisitor, ArrayChildVisitor, ArrayEq, ArrayHash, ArrayRef, Canonical,
16+
DeserializeMetadata, EncodingId, EncodingRef, Precision, ProstMetadata, vtable,
1417
};
18+
use vortex_buffer::ByteBuffer;
1519
use vortex_dtype::{DType, PType};
16-
use vortex_error::{VortexExpect, VortexResult, vortex_ensure};
20+
use vortex_error::{VortexError, VortexExpect, VortexResult, vortex_bail, vortex_ensure};
1721

1822
use crate::ALPFloat;
19-
use crate::alp::{Exponents, decompress};
23+
use crate::alp::{Exponents, alp_encode, decompress};
2024

2125
vtable!(ALP);
2226

2327
impl VTable for ALPVTable {
2428
type Array = ALPArray;
2529
type Encoding = ALPEncoding;
30+
type Metadata = ProstMetadata<ALPMetadata>;
2631

2732
type ArrayVTable = Self;
2833
type CanonicalVTable = Self;
@@ -31,7 +36,6 @@ impl VTable for ALPVTable {
3136
type VisitorVTable = Self;
3237
type ComputeVTable = NotSupported;
3338
type EncodeVTable = Self;
34-
type SerdeVTable = Self;
3539
type OperatorVTable = NotSupported;
3640

3741
fn id(_encoding: &Self::Encoding) -> EncodingId {
@@ -41,6 +45,71 @@ impl VTable for ALPVTable {
4145
fn encoding(_array: &Self::Array) -> EncodingRef {
4246
EncodingRef::new_ref(ALPEncoding.as_ref())
4347
}
48+
49+
fn metadata(array: &ALPArray) -> VortexResult<Self::Metadata> {
50+
let exponents = array.exponents();
51+
Ok(ProstMetadata(ALPMetadata {
52+
exp_e: exponents.e as u32,
53+
exp_f: exponents.f as u32,
54+
patches: array
55+
.patches()
56+
.map(|p| p.to_metadata(array.len(), array.dtype()))
57+
.transpose()?,
58+
}))
59+
}
60+
61+
fn serialize(metadata: Self::Metadata) -> VortexResult<Option<Vec<u8>>> {
62+
metadata.serialize()
63+
}
64+
65+
fn deserialize(buffer: &[u8]) -> VortexResult<Self::Metadata> {
66+
ProstMetadata::<ALPMetadata>::deserialize(buffer)
67+
}
68+
69+
fn build(
70+
_encoding: &ALPEncoding,
71+
dtype: &DType,
72+
len: usize,
73+
metadata: &Self::Metadata,
74+
_buffers: &[ByteBuffer],
75+
children: &dyn ArrayChildren,
76+
) -> VortexResult<ALPArray> {
77+
let encoded_ptype = match &dtype {
78+
DType::Primitive(PType::F32, n) => DType::Primitive(PType::I32, *n),
79+
DType::Primitive(PType::F64, n) => DType::Primitive(PType::I64, *n),
80+
d => vortex_bail!(MismatchedTypes: "f32 or f64", d),
81+
};
82+
let encoded = children.get(0, &encoded_ptype, len)?;
83+
84+
let patches = metadata
85+
.patches
86+
.map(|p| {
87+
let indices = children.get(1, &p.indices_dtype(), p.len())?;
88+
let values = children.get(2, dtype, p.len())?;
89+
let chunk_offsets = p
90+
.chunk_offsets_dtype()
91+
.map(|dtype| children.get(3, &dtype, usize::try_from(p.chunk_offsets_len())?))
92+
.transpose()?;
93+
94+
Ok::<_, VortexError>(Patches::new(
95+
len,
96+
p.offset(),
97+
indices,
98+
values,
99+
chunk_offsets,
100+
))
101+
})
102+
.transpose()?;
103+
104+
ALPArray::try_new(
105+
encoded,
106+
Exponents {
107+
e: u8::try_from(metadata.exp_e)?,
108+
f: u8::try_from(metadata.exp_f)?,
109+
},
110+
patches,
111+
)
112+
}
44113
}
45114

46115
#[derive(Clone, Debug)]
@@ -55,6 +124,16 @@ pub struct ALPArray {
55124
#[derive(Clone, Debug)]
56125
pub struct ALPEncoding;
57126

127+
#[derive(Clone, prost::Message)]
128+
pub struct ALPMetadata {
129+
#[prost(uint32, tag = "1")]
130+
pub(crate) exp_e: u32,
131+
#[prost(uint32, tag = "2")]
132+
pub(crate) exp_f: u32,
133+
#[prost(message, optional, tag = "3")]
134+
pub(crate) patches: Option<PatchesMetadata>,
135+
}
136+
58137
impl ALPArray {
59138
fn validate(
60139
encoded: &dyn Array,
@@ -285,3 +364,28 @@ impl CanonicalVTable<ALPVTable> for ALPVTable {
285364
Canonical::Primitive(decompress(array.clone()))
286365
}
287366
}
367+
368+
impl EncodeVTable<ALPVTable> for ALPVTable {
369+
fn encode(
370+
_encoding: &ALPEncoding,
371+
canonical: &Canonical,
372+
like: Option<&ALPArray>,
373+
) -> VortexResult<Option<ALPArray>> {
374+
let parray = canonical.clone().into_primitive();
375+
let exponents = like.map(|a| a.exponents());
376+
let alp = alp_encode(&parray, exponents)?;
377+
378+
Ok(Some(alp))
379+
}
380+
}
381+
382+
impl VisitorVTable<ALPVTable> for ALPVTable {
383+
fn visit_buffers(_array: &ALPArray, _visitor: &mut dyn ArrayBufferVisitor) {}
384+
385+
fn visit_children(array: &ALPArray, visitor: &mut dyn ArrayChildVisitor) {
386+
visitor.visit_child("encoded", array.encoded());
387+
if let Some(patches) = array.patches() {
388+
visitor.visit_patches(patches);
389+
}
390+
}
391+
}

‎encodings/alp/src/alp/serde.rs‎

Lines changed: 1 addition & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -1,128 +1,14 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

4-
use vortex_array::patches::{Patches, PatchesMetadata};
5-
use vortex_array::serde::ArrayChildren;
6-
use vortex_array::vtable::{EncodeVTable, SerdeVTable, VisitorVTable};
7-
use vortex_array::{
8-
ArrayBufferVisitor, ArrayChildVisitor, Canonical, DeserializeMetadata, ProstMetadata,
9-
};
10-
use vortex_buffer::ByteBuffer;
11-
use vortex_dtype::{DType, PType};
12-
use vortex_error::{VortexError, VortexResult, vortex_bail};
13-
14-
use super::{ALPEncoding, alp_encode};
15-
use crate::{ALPArray, ALPVTable, Exponents};
16-
17-
#[derive(Clone, prost::Message)]
18-
pub struct ALPMetadata {
19-
#[prost(uint32, tag = "1")]
20-
exp_e: u32,
21-
#[prost(uint32, tag = "2")]
22-
exp_f: u32,
23-
#[prost(message, optional, tag = "3")]
24-
patches: Option<PatchesMetadata>,
25-
}
26-
27-
impl SerdeVTable<ALPVTable> for ALPVTable {
28-
type Metadata = ProstMetadata<ALPMetadata>;
29-
30-
fn metadata(array: &ALPArray) -> VortexResult<Option<Self::Metadata>> {
31-
let exponents = array.exponents();
32-
Ok(Some(ProstMetadata(ALPMetadata {
33-
exp_e: exponents.e as u32,
34-
exp_f: exponents.f as u32,
35-
patches: array
36-
.patches()
37-
.map(|p| p.to_metadata(array.len(), array.dtype()))
38-
.transpose()?,
39-
})))
40-
}
41-
42-
/// Deserialize an ALPArray from its components.
43-
///
44-
/// Note that the layout depends on whether patches and chunk_offsets are present:
45-
/// - No patches: `[encoded]`
46-
/// - With patches: `[encoded, patch_indices, patch_values, chunk_offsets?]`
47-
fn build(
48-
_encoding: &ALPEncoding,
49-
dtype: &DType,
50-
len: usize,
51-
metadata: &<Self::Metadata as DeserializeMetadata>::Output,
52-
_buffers: &[ByteBuffer],
53-
children: &dyn ArrayChildren,
54-
) -> VortexResult<ALPArray> {
55-
let encoded_ptype = match &dtype {
56-
DType::Primitive(PType::F32, n) => DType::Primitive(PType::I32, *n),
57-
DType::Primitive(PType::F64, n) => DType::Primitive(PType::I64, *n),
58-
d => vortex_bail!(MismatchedTypes: "f32 or f64", d),
59-
};
60-
let encoded = children.get(0, &encoded_ptype, len)?;
61-
62-
let patches = metadata
63-
.patches
64-
.map(|p| {
65-
let indices = children.get(1, &p.indices_dtype(), p.len())?;
66-
let values = children.get(2, dtype, p.len())?;
67-
let chunk_offsets = p
68-
.chunk_offsets_dtype()
69-
.map(|dtype| children.get(3, &dtype, usize::try_from(p.chunk_offsets_len())?))
70-
.transpose()?;
71-
72-
Ok::<_, VortexError>(Patches::new(
73-
len,
74-
p.offset(),
75-
indices,
76-
values,
77-
chunk_offsets,
78-
))
79-
})
80-
.transpose()?;
81-
82-
ALPArray::try_new(
83-
encoded,
84-
Exponents {
85-
e: u8::try_from(metadata.exp_e)?,
86-
f: u8::try_from(metadata.exp_f)?,
87-
},
88-
patches,
89-
)
90-
}
91-
}
92-
93-
impl EncodeVTable<ALPVTable> for ALPVTable {
94-
fn encode(
95-
_encoding: &ALPEncoding,
96-
canonical: &Canonical,
97-
like: Option<&ALPArray>,
98-
) -> VortexResult<Option<ALPArray>> {
99-
let parray = canonical.clone().into_primitive();
100-
let exponents = like.map(|a| a.exponents());
101-
let alp = alp_encode(&parray, exponents)?;
102-
103-
Ok(Some(alp))
104-
}
105-
}
106-
107-
impl VisitorVTable<ALPVTable> for ALPVTable {
108-
fn visit_buffers(_array: &ALPArray, _visitor: &mut dyn ArrayBufferVisitor) {}
109-
110-
fn visit_children(array: &ALPArray, visitor: &mut dyn ArrayChildVisitor) {
111-
visitor.visit_child("encoded", array.encoded());
112-
if let Some(patches) = array.patches() {
113-
visitor.visit_patches(patches);
114-
}
115-
}
116-
}
117-
1184
#[cfg(test)]
1195
mod tests {
1206
use vortex_array::ProstMetadata;
1217
use vortex_array::patches::PatchesMetadata;
1228
use vortex_array::test_harness::check_metadata;
1239
use vortex_dtype::PType;
12410

125-
use crate::alp::serde::ALPMetadata;
11+
use crate::alp::array::ALPMetadata;
12612

12713
#[cfg_attr(miri, ignore)]
12814
#[test]

0 commit comments

Comments
 (0)