feat: Remove 'parquet' feature and some other dependency bloat (#4251)

AdamGS · web-flow · commit 7a89e7fa10c8 · 2025-08-15T10:26:27.000Z
This is a step in the right direction, removes two of our slowest-to-build dependencies which are not really needed. I just re-implemented the cast logic we care about and parquet was just there for an error variant. To measure the difference, I ran ```bash cargo clean; cargo build --release -p vortex --timings --all-features ``` Currently on develop `vortex` has 413 dependencies, with the top 10 in build time being: <img width="521" height="405" alt="Screenshot 2025-08-15 at 10 58 51" src="https://github.com/user-attachments/assets/7ed61150-174b-4ec1-a916-82f06491df9d" /> With this PR it has 327 dependencies and the top 10 are: <img width="527" height="435" alt="Screenshot 2025-08-15 at 11 03 50" src="https://github.com/user-attachments/assets/49d68b0d-19fd-437b-b346-7b90916c3d5a" /> `arrow-ord` and `arrow-select` do some heavier lifting that I think will be harder to replace, but we should also do that at some point. --------- Signed-off-by: Adam Gutglick <adam@spiraldb.com>
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -121,7 +121,7 @@ moka = { version = "0.12.10", default-features = false }
 multiversion = "0.8.0"
 num-traits = "0.2.19"
 num_enum = { version = "0.7.3", default-features = false }
-object_store = { version = "0.12.3", features = ["aws"] }
+object_store = { version = "0.12.3", default-features = false }
 once_cell = "1.21"
 opentelemetry = "0.30.0"
 opentelemetry-otlp = "0.30.0"
diff --git a/bench-vortex/Cargo.toml b/bench-vortex/Cargo.toml
@@ -72,7 +72,6 @@ url = { workspace = true }
 uuid = { workspace = true, features = ["v4"] }
 vortex = { workspace = true, features = [
     "object_store",
-    "parquet",
     "files",
     "tokio",
     "zstd",
diff --git a/encodings/decimal-byte-parts/Cargo.toml b/encodings/decimal-byte-parts/Cargo.toml
@@ -17,7 +17,6 @@ version = { workspace = true }
 workspace = true
 
 [dependencies]
-itertools = { workspace = true }
 num-traits = { workspace = true }
 prost = { workspace = true }
 vortex-array = { workspace = true }
diff --git a/encodings/pco/Cargo.toml b/encodings/pco/Cargo.toml
@@ -17,7 +17,6 @@ version = { workspace = true }
 workspace = true
 
 [dependencies]
-half = { workspace = true }
 pco = { workspace = true }
 prost = { workspace = true }
 vortex-array = { workspace = true }
diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs
@@ -18,7 +18,7 @@ use vortex_array::vtable::{
 };
 use vortex_array::{ArrayRef, Canonical, EncodingId, EncodingRef, IntoArray, ToCanonical, vtable};
 use vortex_buffer::{BufferMut, ByteBuffer, ByteBufferMut};
-use vortex_dtype::{DType, PType};
+use vortex_dtype::{DType, PType, half};
 use vortex_error::{VortexError, VortexResult, vortex_bail, vortex_err};
 use vortex_scalar::Scalar;
 
diff --git a/encodings/sequence/Cargo.toml b/encodings/sequence/Cargo.toml
@@ -14,7 +14,6 @@ rust-version = { workspace = true }
 version = { workspace = true }
 
 [dependencies]
-arcref = { workspace = true }
 num-traits = { workspace = true }
 prost = { workspace = true }
 vortex-array = { workspace = true }
diff --git a/encodings/sparse/Cargo.toml b/encodings/sparse/Cargo.toml
@@ -20,7 +20,6 @@ workspace = true
 itertools = { workspace = true }
 num-traits = { workspace = true }
 prost = { workspace = true }
-rstest_reuse = { workspace = true }
 vortex-array = { workspace = true }
 vortex-buffer = { workspace = true }
 vortex-dtype = { workspace = true }
diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml
@@ -22,7 +22,6 @@ arcref = { workspace = true }
 arrow-arith = { workspace = true }
 arrow-array = { workspace = true, features = ["ffi"] }
 arrow-buffer = { workspace = true }
-arrow-cast = { workspace = true }
 arrow-data = { workspace = true }
 arrow-ord = { workspace = true }
 arrow-schema = { workspace = true }
@@ -73,6 +72,7 @@ table-display = ["dep:tabled"]
 test-harness = ["dep:goldenfile", "dep:rstest", "dep:rstest_reuse"]
 
 [dev-dependencies]
+arrow-cast = { workspace = true }
 divan = { workspace = true }
 rstest = { workspace = true }
 vortex-array = { path = ".", features = ["test-harness"] }
diff --git a/vortex-array/src/arrays/varbin/canonical.rs b/vortex-array/src/arrays/varbin/canonical.rs
@@ -1,6 +1,10 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+use std::sync::Arc;
+
+use arrow_array::cast::AsArray;
+use arrow_array::{BinaryViewArray, StringViewArray};
 use arrow_schema::DataType;
 use vortex_dtype::DType;
 use vortex_error::VortexResult;
@@ -17,11 +21,28 @@ impl CanonicalVTable<VarBinVTable> for VarBinVTable {
         let nullable = dtype.is_nullable();
 
         let array_ref = array.to_array().into_arrow_preferred()?;
-        let array = match dtype {
-            DType::Utf8(_) => arrow_cast::cast(array_ref.as_ref(), &DataType::Utf8View)?,
-            DType::Binary(_) => arrow_cast::cast(array_ref.as_ref(), &DataType::BinaryView)?,
 
-            _ => unreachable!("VarBinArray must have Utf8 or Binary dtype"),
+        let array = match (&dtype, array_ref.data_type()) {
+            (DType::Utf8(_), DataType::Utf8) => {
+                Arc::new(StringViewArray::from(array_ref.as_string::<i32>()))
+                    as Arc<dyn arrow_array::Array>
+            }
+            (DType::Utf8(_), DataType::LargeUtf8) => {
+                Arc::new(StringViewArray::from(array_ref.as_string::<i64>()))
+                    as Arc<dyn arrow_array::Array>
+            }
+
+            (DType::Binary(_), DataType::Binary) => {
+                Arc::new(BinaryViewArray::from(array_ref.as_binary::<i32>()))
+            }
+            (DType::Binary(_), DataType::LargeBinary) => {
+                Arc::new(BinaryViewArray::from(array_ref.as_binary::<i64>()))
+            }
+            // If its already a view, no need to do anything
+            (DType::Binary(_), DataType::BinaryView) | (DType::Utf8(_), DataType::Utf8View) => {
+                array_ref
+            }
+            _ => unreachable!("VarBinArray must have Utf8 or Binary dtype, instead got: {dtype}",),
         };
         Ok(Canonical::VarBinView(
             ArrayRef::from_arrow(array.as_ref(), nullable).to_varbinview()?,
diff --git a/vortex-cxx/Cargo.toml b/vortex-cxx/Cargo.toml
@@ -25,9 +25,7 @@ anyhow = { workspace = true }
 arrow-array = { workspace = true, features = ["ffi"] }
 arrow-schema = { workspace = true }
 cxx = "1.0"
-futures = { workspace = true, features = ["thread-pool"] }
 paste = { workspace = true }
-prost = { workspace = true }
 take_mut = { workspace = true }
 tokio = { workspace = true, features = ["rt", "rt-multi-thread", "macros"] }
 vortex = { workspace = true, features = ["tokio"] }
diff --git a/vortex-duckdb/Cargo.toml b/vortex-duckdb/Cargo.toml
@@ -23,16 +23,14 @@ crate-type = ["staticlib", "cdylib", "rlib"]
 anyhow = { workspace = true }
 arrow-array = { workspace = true }
 arrow-buffer = { workspace = true }
-arrow-schema = { workspace = true }
 bitvec = { workspace = true }
-crossbeam-queue = { workspace = true }
 dashmap = { workspace = true }
 futures = { workspace = true }
 glob = { workspace = true }
 itertools = { workspace = true }
 log = { workspace = true }
 num-traits = { workspace = true }
-object_store = { workspace = true }
+object_store = { workspace = true, features = ["aws"] }
 parking_lot = { workspace = true }
 tempfile = { workspace = true }
 tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread"] }
diff --git a/vortex-error/Cargo.toml b/vortex-error/Cargo.toml
@@ -23,7 +23,6 @@ arrow-schema = { workspace = true }
 flatbuffers = { workspace = true, optional = true }
 jiff = { workspace = true }
 object_store = { workspace = true, optional = true }
-parquet = { workspace = true, optional = true }
 prost = { workspace = true, optional = true }
 pyo3 = { workspace = true, optional = true }
 serde_json = { workspace = true, optional = true }
diff --git a/vortex-error/src/lib.rs b/vortex-error/src/lib.rs
@@ -103,9 +103,6 @@ pub enum VortexError {
     IOError(io::Error, Box<Backtrace>),
     /// A wrapper for UTF-8 conversion errors.
     Utf8Error(std::str::Utf8Error, Box<Backtrace>),
-    /// A wrapper for errors from the Parquet library.
-    #[cfg(feature = "parquet")]
-    ParquetError(parquet::errors::ParquetError, Box<Backtrace>),
     /// A wrapper for errors from the standard library when converting a slice to an array.
     TryFromSliceError(std::array::TryFromSliceError, Box<Backtrace>),
     /// A wrapper for errors from the Object Store library.
@@ -139,6 +136,11 @@ impl VortexError {
     pub fn with_context<T: Into<ErrString>>(self, msg: T) -> Self {
         VortexError::Context(msg.into(), Box::new(self))
     }
+
+    /// Wrap an a generic error into a Vortex error
+    pub fn generic(err: Box<dyn Error + Send + Sync + 'static>) -> Self {
+        Self::Generic(err, Box::new(Backtrace::capture()))
+    }
 }
 
 impl Display for VortexError {
@@ -200,10 +202,6 @@ impl Display for VortexError {
             VortexError::Utf8Error(err, backtrace) => {
                 write!(f, "{err}\nBacktrace:\n{backtrace}")
             }
-            #[cfg(feature = "parquet")]
-            VortexError::ParquetError(err, backtrace) => {
-                write!(f, "{err}\nBacktrace:\n{backtrace}")
-            }
             VortexError::TryFromSliceError(err, backtrace) => {
                 write!(f, "{err}\nBacktrace:\n{backtrace}")
             }
@@ -253,19 +251,13 @@ impl Error for VortexError {
             VortexError::ArrowError(err, _) => Some(err),
             #[cfg(feature = "flatbuffers")]
             VortexError::FlatBuffersError(err, _) => Some(err),
-            VortexError::FmtError(err, _) => Some(err),
             VortexError::IOError(err, _) => Some(err),
-            VortexError::Utf8Error(err, _) => Some(err),
-            #[cfg(feature = "parquet")]
-            VortexError::ParquetError(err, _) => Some(err),
-            VortexError::TryFromSliceError(err, _) => Some(err),
             #[cfg(feature = "object_store")]
             VortexError::ObjectStore(err, _) => Some(err),
             VortexError::JiffError(err, _) => Some(err),
             #[cfg(feature = "tokio")]
             VortexError::JoinError(err, _) => Some(err),
             VortexError::UrlError(err, _) => Some(err),
-            VortexError::TryFromInt(err, _) => Some(err),
             #[cfg(feature = "serde")]
             VortexError::SerdeJsonError(err, _) => Some(err),
             #[cfg(feature = "prost")]
@@ -486,12 +478,6 @@ impl From<flatbuffers::InvalidFlatbuffer> for VortexError {
     }
 }
 
-impl From<fmt::Error> for VortexError {
-    fn from(value: fmt::Error) -> Self {
-        VortexError::FmtError(value, Box::new(Backtrace::capture()))
-    }
-}
-
 impl From<io::Error> for VortexError {
     fn from(value: io::Error) -> Self {
         VortexError::IOError(value, Box::new(Backtrace::capture()))
@@ -504,13 +490,6 @@ impl From<std::str::Utf8Error> for VortexError {
     }
 }
 
-#[cfg(feature = "parquet")]
-impl From<parquet::errors::ParquetError> for VortexError {
-    fn from(value: parquet::errors::ParquetError) -> Self {
-        VortexError::ParquetError(value, Box::new(Backtrace::capture()))
-    }
-}
-
 impl From<std::array::TryFromSliceError> for VortexError {
     fn from(value: std::array::TryFromSliceError) -> Self {
         VortexError::TryFromSliceError(value, Box::new(Backtrace::capture()))
diff --git a/vortex-file/Cargo.toml b/vortex-file/Cargo.toml
@@ -14,7 +14,6 @@ rust-version = { workspace = true }
 version = { workspace = true }
 
 [dependencies]
-arcref = { workspace = true }
 async-trait = { workspace = true }
 bytes = { workspace = true }
 dashmap = { workspace = true }
@@ -29,7 +28,6 @@ moka = { workspace = true, features = ["future"] }
 object_store = { workspace = true, optional = true }
 rustc-hash = { workspace = true }
 tokio = { workspace = true, features = ["rt"], optional = true }
-tracing = { workspace = true, optional = true }
 # Needed to pickup the "js" feature for wasm targets from the workspace configuration
 uuid = { workspace = true }
 vortex-alp = { workspace = true }
@@ -81,5 +79,5 @@ tokio = [
     "vortex-layout/tokio",
     "vortex-scan/tokio",
 ]
-tracing = ["dep:tracing", "vortex-io/tracing", "vortex-layout/tracing"]
+tracing = ["vortex-io/tracing", "vortex-layout/tracing"]
 zstd = ["dep:vortex-zstd", "vortex-layout/zstd"]
diff --git a/vortex-io/Cargo.toml b/vortex-io/Cargo.toml
@@ -22,7 +22,7 @@ futures = { workspace = true, features = ["std"] }
 futures-util = { workspace = true }
 # Needed to pickup the "wasm_js" feature for wasm targets from the workspace configuration
 getrandom_v03 = { workspace = true }
-object_store = { workspace = true, optional = true }
+object_store = { workspace = true, optional = true, features = ["fs"] }
 pin-project = { workspace = true }
 # this is the maximum subset of fetaures that is safe for wasm32 targets
 tokio = { workspace = true, features = ["io-util", "rt", "sync"] }
diff --git a/vortex-layout/Cargo.toml b/vortex-layout/Cargo.toml
diff --git a/vortex-scan/Cargo.toml b/vortex-scan/Cargo.toml
diff --git a/vortex-tui/Cargo.toml b/vortex-tui/Cargo.toml
diff --git a/vortex-tui/src/convert.rs b/vortex-tui/src/convert.rs
diff --git a/vortex/Cargo.toml b/vortex/Cargo.toml
diff --git a/vortex/src/lib.rs b/vortex/src/lib.rs
