Add serde feature to buffers and expose it externally (#4709)

This is useful for persisting buffers easily, or sending them over the
network.

Crates like `foyer` require it for their persistent feature.

Signed-off-by: Adam Gutglick <[email protected]>

Author: AdamGS

Cargo.lock

@@ -20,6 +20,7 @@ all-features = true
 arrow = ["dep:arrow-buffer"]
 memmap2 = ["dep:memmap2"]
 warn-copy = ["dep:log"]
+serde = ["dep:serde"]
 
 [dependencies]
 arrow-buffer = { workspace = true, optional = true }
@@ -28,6 +29,7 @@ itertools = { workspace = true }
 log = { workspace = true, optional = true }
 memmap2 = { workspace = true, optional = true }
 num-traits = { workspace = true }
+serde = { workspace = true, optional = true }
 simdutf8 = { workspace = true }
 vortex-error = { workspace = true }
 

vortex-buffer/Cargo.toml

@@ -63,6 +63,8 @@ mod debug;
 mod macros;
 #[cfg(feature = "memmap2")]
 mod memmap2;
+#[cfg(feature = "serde")]
+mod serde;
 mod string;
 mod trusted_len;
 

vortex-buffer/src/lib.rs

@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+use std::marker::PhantomData;
+
+use serde::de::Visitor;
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
+
+use crate::{Alignment, Buffer, BufferMut, ByteBuffer};
+
+impl<T> Serialize for Buffer<T>
+where
+    T: Serialize,
+{
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_bytes(self.inner().as_ref())
+    }
+}
+
+struct BufferVisitor<T> {
+    _marker: PhantomData<T>,
+}
+
+impl<T> Default for BufferVisitor<T> {
+    fn default() -> Self {
+        Self {
+            _marker: PhantomData,
+        }
+    }
+}
+
+impl<'de, T> Visitor<'de> for BufferVisitor<T>
+where
+    T: Deserialize<'de>,
+{
+    type Value = Buffer<T>;
+
+    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(formatter, "byte buffer")
+    }
+
+    fn visit_byte_buf<E>(self, v: Vec<u8>) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        let bytes = ByteBuffer::copy_from_aligned(v, Alignment::of::<T>());
+        Ok(Buffer::from_byte_buffer(bytes))
+    }
+
+    fn visit_borrowed_bytes<E>(self, v: &'de [u8]) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        let bytes = ByteBuffer::copy_from_aligned(v, Alignment::of::<T>());
+        Ok(Buffer::from_byte_buffer(bytes))
+    }
+
+    fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        let bytes = ByteBuffer::copy_from_aligned(v, Alignment::of::<T>());
+        Ok(Buffer::from_byte_buffer(bytes))
+    }
+
+    fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+    where
+        A: serde::de::SeqAccess<'de>,
+    {
+        let mut buffer = seq
+            .size_hint()
+            .map(|hint| BufferMut::<T>::with_capacity(hint))
+            .unwrap_or_default();
+
+        while let Some(e) = seq.next_element()? {
+            buffer.push(e);
+        }
+
+        Ok(buffer.freeze())
+    }
+}
+
+impl<'de, T> Deserialize<'de> for Buffer<T>
+where
+    T: Deserialize<'de>,
+{
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        deserializer.deserialize_byte_buf(BufferVisitor::<T>::default())
+    }
+}

vortex-buffer/src/serde.rs

@@ -50,5 +50,10 @@ arbitrary = [
     "vortex-scalar/arbitrary",
     "vortex-dtype/arbitrary",
 ]
-serde = ["dep:serde", "vortex-dtype/serde", "vortex-error/serde"]
+serde = [
+    "dep:serde",
+    "vortex-dtype/serde",
+    "vortex-error/serde",
+    "vortex-buffer/serde",
+]
 test-harness = []

vortex-expr/Cargo.toml

@@ -71,6 +71,12 @@ python = ["vortex-error/python"]
 tokio = ["vortex-file/tokio", "vortex-scan/tokio"]
 zstd = ["dep:vortex-zstd", "vortex-file/zstd", "vortex-layout/zstd"]
 pretty = ["vortex-array/table-display"]
+serde = [
+    "vortex-expr/serde",
+    "vortex-dtype/serde",
+    "vortex-buffer/serde",
+    "vortex-error/serde",
+]
 
 [[bench]]
 name = "single_encoding_throughput"