chore: Replace dependency on top level arrow create with individual modules now that pyarrow is a standalone crate (#3864)

Signed-off-by: Robert Kruszewski <[email protected]>

Author: robert3005

Cargo.lock

@@ -57,16 +57,16 @@ anyhow = "1.0.95"
 arbitrary = "1.3.2"
 arcref = "0.2.0"
 arrayref = "0.3.7"
-arrow = { version = "55.1", default-features = false }
-arrow-arith = "55.1"
-arrow-array = "55.1"
-arrow-buffer = "55.1"
-arrow-cast = "55.1"
-arrow-data = "55.1"
-arrow-ord = "55.1"
-arrow-schema = "55.1"
-arrow-select = "55.1"
-arrow-string = "55.1"
+arrow-arith = "55.2.0"
+arrow-array = "55.2.0"
+arrow-buffer = "55.2.0"
+arrow-cast = "55.2.0"
+arrow-data = "55.2.0"
+arrow-ord = "55.2.0"
+arrow-pyarrow = "55.2.0"
+arrow-schema = "55.2.0"
+arrow-select = "55.2.0"
+arrow-string = "55.2.0"
 async-stream = "0.3.6"
 async-trait = "0.1.88"
 bindgen = "0.72.0"
@@ -123,7 +123,7 @@ opentelemetry = "0.30.0"
 opentelemetry-otlp = "0.30.0"
 opentelemetry_sdk = "0.30.0"
 parking_lot = { version = "0.12.3", features = ["nightly"] }
-parquet = "55.1"
+parquet = "55.2.0"
 paste = "1.0.15"
 pco = "0.4.4"
 pin-project = "1.1.5"

Cargo.toml

@@ -21,7 +21,9 @@ crate-type = ["staticlib", "cdylib", "rlib"]
 
 [dependencies]
 anyhow = { workspace = true }
-arrow = { workspace = true }
+arrow-array = { workspace = true }
+arrow-buffer = { workspace = true }
+arrow-schema = { workspace = true }
 bitvec = { workspace = true }
 crossbeam-queue = { workspace = true }
 glob = { workspace = true }

vortex-duckdb/Cargo.toml

@@ -4,15 +4,16 @@
 /// Copied of duckdb-rs (https://github.com/duckdb/duckdb-rs/blob/main/crates/duckdb/src/vtab/arrow.rs)
 use std::sync::Arc;
 
-use arrow::array::{
-    Array, BooleanArray, Date32Array, GenericBinaryBuilder, PrimitiveArray, StringArray,
-    TimestampMicrosecondArray, TimestampNanosecondArray,
-};
-use arrow::buffer::{BooleanBuffer, NullBuffer};
-use arrow::datatypes::{
+use arrow_array::builder::GenericBinaryBuilder;
+use arrow_array::types::{
     Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type, Time64MicrosecondType,
     UInt8Type, UInt16Type, UInt32Type, UInt64Type,
 };
+use arrow_array::{
+    Array, BooleanArray, Date32Array, PrimitiveArray, StringArray, TimestampMicrosecondArray,
+    TimestampNanosecondArray,
+};
+use arrow_buffer::buffer::{BooleanBuffer, NullBuffer};
 use bitvec::macros::internal::funty::Fundamental;
 use vortex::ArrayRef;
 use vortex::arrays::StructArray;

vortex-duckdb/src/convert/vector.rs

@@ -17,7 +17,8 @@ rust-version = { workspace = true }
 categories = { workspace = true }
 
 [dependencies]
-arrow = { workspace = true, features = ["ffi"] }
+arrow-array = { workspace = true, features = ["ffi"] }
+arrow-schema = { workspace = true }
 jni = "0.21.1"
 log = { workspace = true }
 object_store = { workspace = true, features = ["aws", "azure", "gcp"] }

vortex-jni/Cargo.toml

@@ -1,8 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-use arrow::datatypes::{DataType, FieldRef, Fields};
-use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema};
+use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema};
+use arrow_schema::{DataType, FieldRef, Fields};
 use jni::JNIEnv;
 use jni::objects::{JClass, JIntArray, JLongArray, JObject, JValue};
 use jni::sys::{
@@ -86,7 +86,7 @@ pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_exportToArrow<'loc
 
         let arrow_array = array_ref.inner.clone().into_arrow(&viewless_arrow_type)?;
         let (ffi_array, ffi_schema) =
-            arrow::ffi::to_ffi(&arrow_array.to_data()).map_err(VortexError::from)?;
+            arrow_array::ffi::to_ffi(&arrow_array.to_data()).map_err(VortexError::from)?;
 
         let ffi_schema_ptr = Box::into_raw(Box::new(ffi_schema));
         let ffi_array_ptr = Box::into_raw(Box::new(ffi_array));

vortex-jni/src/array.rs

@@ -23,7 +23,10 @@ crate-type = ["rlib", "cdylib"]
 doctest = false                 # TODO(ngates): why?
 
 [dependencies]
-arrow = { workspace = true, features = ["pyarrow"] }
+arrow-array = { workspace = true }
+arrow-data = { workspace = true }
+arrow-pyarrow = { workspace = true }
+arrow-schema = { workspace = true }
 futures = { workspace = true }
 itertools = { workspace = true }
 log = { workspace = true }

vortex-python/Cargo.toml

@@ -1,11 +1,11 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-use arrow::array::{ArrayData as ArrowArrayData, make_array};
-use arrow::datatypes::{DataType, Field};
-use arrow::ffi_stream::ArrowArrayStreamReader;
-use arrow::pyarrow::FromPyArrow;
-use arrow::record_batch::RecordBatchReader;
+use arrow_array::ffi_stream::ArrowArrayStreamReader;
+use arrow_array::{RecordBatchReader, make_array};
+use arrow_data::ArrayData as ArrowArrayData;
+use arrow_pyarrow::FromPyArrow;
+use arrow_schema::{DataType, Field};
 use itertools::Itertools;
 use pyo3::exceptions::PyValueError;
 use pyo3::prelude::*;

vortex-python/src/arrays/from_arrow.rs

@@ -8,8 +8,8 @@ pub(crate) mod from_arrow;
 mod native;
 pub(crate) mod py;
 
-use arrow::array::{Array as ArrowArray, ArrayRef as ArrowArrayRef};
-use arrow::pyarrow::ToPyArrow;
+use arrow_array::{Array as ArrowArray, ArrayRef as ArrowArrayRef};
+use arrow_pyarrow::ToPyArrow;
 use pyo3::exceptions::{PyTypeError, PyValueError};
 use pyo3::prelude::*;
 use pyo3::types::{PyDict, PyList};

vortex-python/src/arrays/mod.rs

@@ -3,9 +3,9 @@
 
 use std::sync::Arc;
 
-use arrow::array::RecordBatchReader;
-use arrow::datatypes::SchemaRef;
-use arrow::pyarrow::{IntoPyArrow, ToPyArrow};
+use arrow_array::RecordBatchReader;
+use arrow_pyarrow::{IntoPyArrow, ToPyArrow};
+use arrow_schema::SchemaRef;
 use pyo3::exceptions::PyTypeError;
 use pyo3::prelude::*;
 use pyo3::types::PyString;