chore: replace vortex-serde with 3 crates (#1296)

Broke vortex-serde out to avoid confusion with the `serde` public crate.
There are now 3 crates:

1. `vortex-io`: Contains core IO traits like `VortexReadAt` and
`VortexWrite` as well as impls, with feature flags for `compio`,
`tokio`, `object_store`
2. `vortex-ipc` containing the IPC messages and
stream_readers/stream_writers
3. `vortex-file` which imports the other two crates and contains all of
the layout and file related types, with feature flags for `tokio`,
`compio`, etc.

`vortex` top-level crate re-exports these as `vortex::io`, `vortex::ipc`
and `vortex::file` respectively

Author: a10y

Cargo.lock

@@ -12,12 +12,14 @@ members = [
     "vortex-dtype",
     "vortex-error",
     "vortex-expr",
+    "vortex-file",
     "vortex-flatbuffers",
+    "vortex-io",
+    "vortex-ipc",
     "vortex-proto",
     "vortex-sampling-compressor",
     "vortex-scalar",
     "vortex-schema",
-    "vortex-serde",
     "xtask",
 ]
 resolver = "2"
@@ -149,15 +151,17 @@ vortex-dtype = { version = "0.16.0", path = "./vortex-dtype", default-features =
 vortex-error = { version = "0.16.0", path = "./vortex-error" }
 vortex-expr = { version = "0.16.0", path = "./vortex-expr" }
 vortex-fastlanes = { version = "0.16.0", path = "./encodings/fastlanes" }
+vortex-file = { version = "0.16.0", path = "./vortex-file", default-features = false }
 vortex-flatbuffers = { version = "0.16.0", path = "./vortex-flatbuffers" }
 vortex-fsst = { version = "0.16.0", path = "./encodings/fsst" }
+vortex-io = { version = "0.16.0", path = "./vortex-io" }
+vortex-ipc = { version = "0.16.0", path = "./vortex-ipc" }
 vortex-proto = { version = "0.16.0", path = "./vortex-proto" }
 vortex-roaring = { version = "0.16.0", path = "./encodings/roaring" }
 vortex-runend = { version = "0.16.0", path = "./encodings/runend" }
 vortex-runend-bool = { version = "0.16.0", path = "./encodings/runend-bool" }
 vortex-scalar = { version = "0.16.0", path = "./vortex-scalar", default-features = false }
 vortex-schema = { version = "0.16.0", path = "./vortex-schema" }
-vortex-serde = { version = "0.16.0", path = "./vortex-serde", default-features = false }
 vortex-sampling-compressor = { version = "0.16.0", path = "./vortex-sampling-compressor" }
 vortex-zigzag = { version = "0.16.0", path = "./encodings/zigzag" }
 # END crates published by this project

Cargo.toml

@@ -10,8 +10,8 @@ use futures::StreamExt;
 use vortex::array::{PrimitiveArray, VarBinArray, VarBinViewArray};
 use vortex::buffer::Buffer;
 use vortex::dtype::{DType, Nullability};
-use vortex::serde::stream_reader::StreamArrayReader;
-use vortex::serde::stream_writer::StreamArrayWriter;
+use vortex::ipc::stream_reader::StreamArrayReader;
+use vortex::ipc::stream_writer::StreamArrayWriter;
 use vortex::validity::Validity;
 use vortex::{Context, IntoArray, IntoCanonical};
 

bench-vortex/benches/bytes_at.rs

@@ -26,9 +26,9 @@ use vortex::array::{ChunkedArray, StructArray};
 use vortex::buffer::Buffer;
 use vortex::dtype::field::Field;
 use vortex::error::VortexResult;
+use vortex::file::{LayoutContext, LayoutDeserializer, VortexFileWriter, VortexReadBuilder};
 use vortex::sampling_compressor::compressors::fsst::FSSTCompressor;
 use vortex::sampling_compressor::{SamplingCompressor, ALL_ENCODINGS_CONTEXT};
-use vortex::serde::file::{LayoutContext, LayoutDeserializer, VortexFileWriter, VortexReadBuilder};
 use vortex::{Array, ArrayDType, IntoArray, IntoCanonical};
 
 use crate::tokio_runtime::TOKIO_RUNTIME;

bench-vortex/benches/compress_noci.rs

@@ -13,8 +13,8 @@ use vortex::array::ChunkedArray;
 use vortex::arrow::FromArrowType;
 use vortex::dtype::DType;
 use vortex::error::{VortexError, VortexResult};
-use vortex::serde::io::TokioAdapter;
-use vortex::serde::stream_writer::StreamArrayWriter;
+use vortex::io::TokioAdapter;
+use vortex::ipc::stream_writer::StreamArrayWriter;
 use vortex::{Array, IntoArray};
 
 use crate::idempotent;

bench-vortex/src/data_downloads.rs

@@ -27,9 +27,9 @@ use vortex::arrow::FromArrowType;
 use vortex::compress::CompressionStrategy;
 use vortex::dtype::DType;
 use vortex::error::VortexResult;
+use vortex::file::{LayoutContext, LayoutDeserializer, VortexFileWriter, VortexReadBuilder};
+use vortex::io::{ObjectStoreReadAt, TokioFile, VortexReadAt, VortexWrite};
 use vortex::sampling_compressor::{SamplingCompressor, ALL_ENCODINGS_CONTEXT};
-use vortex::serde::file::{LayoutContext, LayoutDeserializer, VortexFileWriter, VortexReadBuilder};
-use vortex::serde::io::{ObjectStoreReadAt, TokioFile, VortexReadAt, VortexWrite};
 use vortex::{Array, IntoArray, IntoCanonical};
 
 pub const BATCH_SIZE: usize = 65_536;

bench-vortex/src/reader.rs

@@ -5,7 +5,7 @@ use std::path::PathBuf;
 use futures::executor::block_on;
 use vortex::buffer::io_buf::IoBuf;
 use vortex::error::VortexError;
-use vortex::serde::io::VortexWrite;
+use vortex::io::VortexWrite;
 
 use crate::data_downloads::{data_vortex_uncompressed, download_data};
 use crate::reader::rewrite_parquet_as_vortex;

bench-vortex/src/taxi_data.rs

@@ -15,8 +15,8 @@ use vortex::aliases::hash_map::HashMap;
 use vortex::array::{ChunkedArray, StructArray};
 use vortex::arrow::FromArrowArray;
 use vortex::dtype::DType;
+use vortex::file::{VortexFileWriter, VORTEX_FILE_EXTENSION};
 use vortex::sampling_compressor::SamplingCompressor;
-use vortex::serde::file::{VortexFileWriter, VORTEX_FILE_EXTENSION};
 use vortex::variants::StructArrayTrait;
 use vortex::{Array, ArrayDType, IntoArray, IntoArrayVariant};
 use vortex_datafusion::memory::VortexMemTableOptions;

bench-vortex/src/tpch/mod.rs

@@ -10,12 +10,12 @@ use vortex::arrow::infer_schema;
 use vortex::dtype::field::Field;
 use vortex::dtype::DType;
 use vortex::error::{vortex_err, VortexResult};
-use vortex::sampling_compressor::ALL_ENCODINGS_CONTEXT;
-use vortex::serde::file::{
+use vortex::file::{
     read_initial_bytes, LayoutContext, LayoutDeserializer, Projection, RowFilter,
     VortexFileArrayStream, VortexReadBuilder, VortexRecordBatchReader,
 };
-use vortex::serde::io::{ObjectStoreReadAt, TokioFile, VortexReadAt};
+use vortex::io::{ObjectStoreReadAt, TokioFile, VortexReadAt};
+use vortex::sampling_compressor::ALL_ENCODINGS_CONTEXT;
 use vortex::Array;
 
 use crate::expr::PyExpr;

pyvortex/src/dataset.rs

@@ -4,8 +4,8 @@ use pyo3::prelude::*;
 use pyo3::pyfunction;
 use pyo3::types::PyString;
 use tokio::fs::File;
+use vortex::file::VortexFileWriter;
 use vortex::sampling_compressor::SamplingCompressor;
-use vortex::serde::file::VortexFileWriter;
 use vortex::Array;
 
 use crate::dataset::{ObjectStoreUrlDataset, TokioFileDataset};