refactor: remove remaining uses of arrow_array and use reexport in arrow instead (#14528)

* refactor: remove remaining uses of arrow_array and use reexport in arrow instead

* fix: remove blank

* fix: run carg fmt

* fix: import record_batch

* fix: update cargo.lock

* fix: add chrono-tz feature in arrow

* fix: remove remaining arrow-array & update cargo.lock

---------

Co-authored-by: Cheng-Yuan-Lai <a186235@g,ail.com>
Co-authored-by: Ian Lai <[email protected]>
Co-authored-by: Andrew Lamb <[email protected]>

Author: 4 people

Cargo.toml

@@ -80,8 +80,6 @@ ahash = { version = "0.8", default-features = false, features = [
 ] }
 arrow = { version = "54.1.0", features = [
     "prettyprint",
-] }
-arrow-array = { version = "54.1.0", default-features = false, features = [
     "chrono-tz",
 ] }
 arrow-buffer = { version = "54.1.0", default-features = false }

datafusion-cli/Cargo.lock

@@ -24,7 +24,6 @@ publish = false
 [dependencies]
 abi_stable = "0.11.3"
 arrow = { workspace = true }
-arrow-array = { workspace = true }
 arrow-schema = { workspace = true }
 datafusion = { workspace = true }
 datafusion-ffi = { workspace = true }

datafusion-examples/examples/ffi/ffi_example_table_provider/Cargo.toml

@@ -18,12 +18,9 @@
 use std::sync::Arc;
 
 use abi_stable::{export_root_module, prefix_type::PrefixTypeTrait};
-use arrow_array::RecordBatch;
-use datafusion::{
-    arrow::datatypes::{DataType, Field, Schema},
-    common::record_batch,
-    datasource::MemTable,
-};
+use arrow::array::RecordBatch;
+use arrow::datatypes::{DataType, Field, Schema};
+use datafusion::{common::record_batch, datasource::MemTable};
 use datafusion_ffi::table_provider::FFI_TableProvider;
 use ffi_module_interface::{TableProviderModule, TableProviderModuleRef};
 

datafusion-examples/examples/ffi/ffi_example_table_provider/src/lib.rs

@@ -51,7 +51,6 @@ apache-avro = { version = "0.17", default-features = false, features = [
     "zstandard",
 ], optional = true }
 arrow = { workspace = true }
-arrow-array = { workspace = true }
 arrow-ipc = { workspace = true }
 arrow-schema = { workspace = true }
 base64 = "0.22.1"

datafusion/common/Cargo.toml

@@ -21,6 +21,10 @@
 //! kernels in arrow-rs such as `as_boolean_array` do.
 
 use crate::{downcast_value, DataFusionError, Result};
+use arrow::array::{
+    BinaryViewArray, Float16Array, Int16Array, Int8Array, LargeBinaryArray,
+    LargeStringArray, StringViewArray, UInt16Array,
+};
 use arrow::{
     array::{
         Array, BinaryArray, BooleanArray, Date32Array, Date64Array, Decimal128Array,
@@ -36,10 +40,6 @@ use arrow::{
     },
     datatypes::{ArrowDictionaryKeyType, ArrowPrimitiveType},
 };
-use arrow_array::{
-    BinaryViewArray, Float16Array, Int16Array, Int8Array, LargeBinaryArray,
-    LargeStringArray, StringViewArray, UInt16Array,
-};
 
 // Downcast ArrayRef to Date32Array
 pub fn as_date32_array(array: &dyn Array) -> Result<&Date32Array> {

datafusion/common/src/cast.rs

@@ -17,9 +17,8 @@
 
 //! Conversions between PyArrow and DataFusion types
 
-use arrow::array::ArrayData;
+use arrow::array::{Array, ArrayData};
 use arrow::pyarrow::{FromPyArrow, ToPyArrow};
-use arrow_array::Array;
 use pyo3::exceptions::PyException;
 use pyo3::prelude::PyErr;
 use pyo3::types::{PyAnyMethods, PyList};

datafusion/common/src/pyarrow.rs

@@ -40,22 +40,24 @@ use crate::cast::{
 use crate::error::{DataFusionError, Result, _exec_err, _internal_err, _not_impl_err};
 use crate::hash_utils::create_hashes;
 use crate::utils::SingleRowListArrayBuilder;
-use arrow::array::types::{IntervalDayTime, IntervalMonthDayNano};
+use arrow::array::{
+    types::{IntervalDayTime, IntervalMonthDayNano},
+    *,
+};
 use arrow::buffer::ScalarBuffer;
-use arrow::compute::kernels::numeric::*;
-use arrow::util::display::{array_value_to_string, ArrayFormatter, FormatOptions};
-use arrow::{
-    array::*,
-    compute::kernels::cast::{cast_with_options, CastOptions},
-    datatypes::{
-        i256, ArrowDictionaryKeyType, ArrowNativeType, ArrowTimestampType, DataType,
-        Date32Type, Date64Type, Field, Float32Type, Int16Type, Int32Type, Int64Type,
-        Int8Type, IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit,
-        IntervalYearMonthType, TimeUnit, TimestampMicrosecondType,
-        TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType,
-        UInt16Type, UInt32Type, UInt64Type, UInt8Type, DECIMAL128_MAX_PRECISION,
-    },
+use arrow::compute::kernels::{
+    cast::{cast_with_options, CastOptions},
+    numeric::*,
 };
+use arrow::datatypes::{
+    i256, ArrowDictionaryKeyType, ArrowNativeType, ArrowTimestampType, DataType,
+    Date32Type, Date64Type, Field, Float32Type, Int16Type, Int32Type, Int64Type,
+    Int8Type, IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit,
+    IntervalYearMonthType, TimeUnit, TimestampMicrosecondType, TimestampMillisecondType,
+    TimestampNanosecondType, TimestampSecondType, UInt16Type, UInt32Type, UInt64Type,
+    UInt8Type, DECIMAL128_MAX_PRECISION,
+};
+use arrow::util::display::{array_value_to_string, ArrayFormatter, FormatOptions};
 use arrow_schema::{UnionFields, UnionMode};
 
 use crate::format::DEFAULT_CAST_OPTIONS;
@@ -165,7 +167,7 @@ pub use struct_builder::ScalarStructBuilder;
 /// ```
 /// # use std::sync::Arc;
 /// # use arrow::datatypes::{DataType, Field, Fields};
-/// # use arrow_array::{ArrayRef, Int32Array, StructArray, StringArray};
+/// # use arrow::array::{ArrayRef, Int32Array, StructArray, StringArray};
 /// # use datafusion_common::ScalarValue;
 /// // Build a struct like: {a: 1, b: "foo"}
 /// // Field description
@@ -1674,7 +1676,7 @@ impl ScalarValue {
     ///
     /// assert_eq!(&result, &expected);
     /// ```
-    /// [`Datum`]: arrow_array::Datum
+    /// [`Datum`]: arrow::array::Datum
     pub fn to_scalar(&self) -> Result<Scalar<ArrayRef>> {
         Ok(Scalar::new(self.to_array_of_size(1)?))
     }

datafusion/common/src/scalar/mod.rs

@@ -34,7 +34,7 @@ use std::{error::Error, path::PathBuf};
 /// ```
 /// # use std::sync::Arc;
 /// # use arrow::record_batch::RecordBatch;
-/// # use arrow_array::{ArrayRef, Int32Array};
+/// # use arrow::array::{ArrayRef, Int32Array};
 /// # use datafusion_common::assert_batches_eq;
 /// let col: ArrayRef = Arc::new(Int32Array::from(vec![1, 2]));
 ///  let batch = RecordBatch::try_from_iter([("column", col)]).unwrap();
@@ -344,7 +344,7 @@ macro_rules! record_batch {
                 )*
             ]));
 
-            let batch = arrow_array::RecordBatch::try_new(
+            let batch = arrow::array::RecordBatch::try_new(
                 schema,
                 vec![$(
                     $crate::create_array!($type, $values),
@@ -416,7 +416,7 @@ mod tests {
 
     #[test]
     fn test_create_record_batch() -> Result<()> {
-        use arrow_array::Array;
+        use arrow::array::Array;
 
         let batch = record_batch!(
             ("a", Int32, vec![1, 2, 3, 4]),

datafusion/common/src/test_util.rs

@@ -24,14 +24,13 @@ pub mod string_utils;
 
 use crate::error::{_internal_datafusion_err, _internal_err};
 use crate::{DataFusionError, Result, ScalarValue};
-use arrow::array::ArrayRef;
+use arrow::array::{
+    cast::AsArray, Array, ArrayRef, FixedSizeListArray, LargeListArray, ListArray,
+    OffsetSizeTrait,
+};
 use arrow::buffer::OffsetBuffer;
 use arrow::compute::{partition, SortColumn, SortOptions};
 use arrow::datatypes::{Field, SchemaRef};
-use arrow_array::cast::AsArray;
-use arrow_array::{
-    Array, FixedSizeListArray, LargeListArray, ListArray, OffsetSizeTrait,
-};
 use arrow_schema::DataType;
 use sqlparser::ast::Ident;
 use sqlparser::dialect::GenericDialect;
@@ -329,8 +328,8 @@ pub fn longest_consecutive_prefix<T: Borrow<usize>>(
 /// # Example
 /// ```
 /// # use std::sync::Arc;
-/// # use arrow_array::{Array, ListArray};
-/// # use arrow_array::types::Int64Type;
+/// # use arrow::array::{Array, ListArray};
+/// # use arrow::array::types::Int64Type;
 /// # use datafusion_common::utils::SingleRowListArrayBuilder;
 /// // Array is [1, 2, 3]
 /// let arr = ListArray::from_iter_primitive::<Int64Type, _, _>(vec![