Remove direct arrow2 exposure (#698)

arrow2 is deprecated:
jorgecarleitao/arrow2#1429

Breaking change:

- Still supported: batch-by-batch processing as DataFrames via
`query_polars()`. That preserves the common Polars flow, including
streaming over batches without collecting everything first.
- Still supported: raw Arrow-rs access via `step()`.
- No longer supported: "give me the raw Polars-arrow/arrow2 batch and
let me do my own low-level conversion or array operations on that type".
That was the unique role of `step2()`.

Author: mlafeldt

Cargo.lock

@@ -42,7 +42,6 @@ num = { version = "0.4", default-features = false }
 num-integer = "0.1.46"
 pkg-config = "0.3.24"
 polars = "0.49.1"
-polars-arrow = "0.49.1"
 polars-core = "0.49.1"
 pretty_assertions = "1.4.1"
 prettyplease = "0.2.20"

Cargo.toml

@@ -33,7 +33,7 @@ vtab-full = ["vtab-excel", "vtab-arrow", "appender-arrow"]
 extensions-full = ["json", "parquet", "vtab-full"]
 buildtime_bindgen = ["libduckdb-sys/buildtime_bindgen"]
 modern-full = ["chrono", "serde_json", "url", "r2d2", "uuid", "polars"]
-polars = ["dep:polars", "dep:polars-arrow"]
+polars = ["dep:polars", "dep:polars-core"]
 # Warning: experimental feature
 loadable-extension = ["vtab", "duckdb-loadable-macros", "libduckdb-sys/loadable-extension"]
 
@@ -50,7 +50,7 @@ libduckdb-sys = { workspace = true }
 num = { workspace = true, features = ["std"], optional = true }
 num-integer = { workspace = true }
 polars = { workspace = true, features = ["dtype-full"], optional = true }
-polars-arrow = { workspace = true, optional = true }
+polars-core = { workspace = true, optional = true }
 r2d2 = { workspace = true, optional = true }
 rust_decimal = { workspace = true }
 serde_json = { workspace = true, optional = true }

crates/duckdb/Cargo.toml

@@ -95,8 +95,6 @@ pub use arrow;
 pub use duckdb_loadable_macros::duckdb_entrypoint_c_api;
 #[cfg(feature = "polars")]
 pub use polars;
-#[cfg(feature = "polars")]
-pub use polars_arrow as arrow2;
 
 /// The core module contains the main functionality of the DuckDB crate.
 pub mod core;

crates/duckdb/src/lib.rs

@@ -26,8 +26,9 @@ impl<'stmt> Iterator for Polars<'stmt> {
     type Item = DataFrame;
 
     fn next(&mut self) -> Option<Self::Item> {
-        let struct_array = self.stmt?.step2()?;
-        let df = DataFrame::try_from(struct_array).expect("Failed to construct DataFrame from StructArray");
+        let struct_array = self.stmt?.step_polars()?;
+        let df = DataFrame::try_from(struct_array)
+            .unwrap_or_else(|e| panic!("Failed to construct DataFrame from StructArray: {e}"));
 
         Some(df)
     }

crates/duckdb/src/polars_dataframe.rs

@@ -7,9 +7,9 @@ use arrow::{
 };
 
 use super::{Result, ffi};
-#[cfg(feature = "polars")]
-use crate::arrow2;
 use crate::{Error, core::LogicalTypeHandle, error::result_from_duckdb_arrow};
+#[cfg(feature = "polars")]
+use polars_core::utils::arrow as polars_arrow;
 
 /// Private newtype for DuckDB prepared statements that finalize themselves when dropped.
 ///
@@ -165,54 +165,52 @@ impl RawStatement {
 
     #[cfg(feature = "polars")]
     #[inline]
-    pub fn step2(&self) -> Option<arrow2::array::StructArray> {
-        self.result?;
+    pub(crate) fn step_polars(&self) -> Option<polars_arrow::array::StructArray> {
+        let result = self.result?;
 
         unsafe {
-            let mut ffi_arrow2_array = arrow2::ffi::ArrowArray::empty();
+            let mut ffi_arrow_array = polars_arrow::ffi::ArrowArray::empty();
 
             if ffi::duckdb_query_arrow_array(
-                self.result_unwrap(),
-                &mut std::ptr::addr_of_mut!(ffi_arrow2_array) as *mut _ as *mut ffi::duckdb_arrow_array,
+                result,
+                &mut std::ptr::addr_of_mut!(ffi_arrow_array) as *mut _ as *mut ffi::duckdb_arrow_array,
             )
             .ne(&ffi::DuckDBSuccess)
             {
                 return None;
             }
 
-            let mut ffi_arrow2_schema = arrow2::ffi::ArrowSchema::empty();
+            let mut ffi_arrow_schema = polars_arrow::ffi::ArrowSchema::empty();
 
             if ffi::duckdb_query_arrow_schema(
-                self.result_unwrap(),
-                &mut std::ptr::addr_of_mut!(ffi_arrow2_schema) as *mut _ as *mut ffi::duckdb_arrow_schema,
+                result,
+                &mut std::ptr::addr_of_mut!(ffi_arrow_schema) as *mut _ as *mut ffi::duckdb_arrow_schema,
             )
             .ne(&ffi::DuckDBSuccess)
             {
                 return None;
             }
 
-            let arrow2_field =
-                arrow2::ffi::import_field_from_c(&ffi_arrow2_schema).expect("Failed to import arrow2 Field from C");
-            let import_arrow2_array = arrow2::ffi::import_array_from_c(ffi_arrow2_array, arrow2_field.dtype);
-
-            if let Err(err) = import_arrow2_array {
-                // When array is empty, import_array_from_c returns error with message
-                // "ComputeError("An ArrowArray of type X must have non-null children")
-                // Therefore, we return None when encountering this error.
-                match err {
-                    polars::error::PolarsError::ComputeError(_) => return None,
-                    _ => panic!("Failed to import arrow2 Array from C: {err}"),
-                }
-            }
+            let field =
+                polars_arrow::ffi::import_field_from_c(&ffi_arrow_schema).expect("Failed to import Polars Arrow field");
+            let import_array = polars_arrow::ffi::import_array_from_c(ffi_arrow_array, field.dtype);
 
-            let arrow2_array = import_arrow2_array.unwrap();
-            let arrow2_struct_array = arrow2_array
+            let array = match import_array {
+                Ok(array) => array,
+                // When array is empty, import_array_from_c returns ComputeError with message
+                // "An ArrowArray of type X must have non-null children".
+                Err(polars::error::PolarsError::ComputeError(msg)) if msg.to_string().contains("non-null children") => {
+                    return None;
+                }
+                Err(err) => panic!("Failed to import Polars Arrow array from C: {err}"),
+            };
+            let struct_array = array
                 .as_any()
-                .downcast_ref::<arrow2::array::StructArray>()
-                .expect("Failed to downcast arrow2 Array to arrow2 StructArray")
+                .downcast_ref::<polars_arrow::array::StructArray>()
+                .expect("Failed to downcast Polars Arrow array to StructArray")
                 .to_owned();
 
-            Some(arrow2_struct_array)
+            Some(struct_array)
         }
     }
 

crates/duckdb/src/raw_statement.rs

@@ -3,13 +3,15 @@ use std::{convert, ffi::c_void, fmt, mem, os::raw::c_char, ptr, str};
 use arrow::{array::StructArray, datatypes::SchemaRef};
 
 use super::{AndThenRows, Connection, Error, MappedRows, Params, RawStatement, Result, Row, Rows, ValueRef, ffi};
+#[cfg(feature = "polars")]
+use crate::polars_dataframe::Polars;
 use crate::{
     arrow_batch::{Arrow, ArrowStream},
     error::result_from_duckdb_prepare,
     types::{ToSql, ToSqlOutput},
 };
 #[cfg(feature = "polars")]
-use crate::{arrow2, polars_dataframe::Polars};
+use polars_core::utils::arrow as polars_arrow;
 
 /// A prepared statement.
 ///
@@ -403,10 +405,9 @@ impl Statement<'_> {
     }
 
     #[cfg(feature = "polars")]
-    /// Get next batch records in arrow2
     #[inline]
-    pub fn step2(&self) -> Option<arrow2::array::StructArray> {
-        self.stmt.step2()
+    pub(crate) fn step_polars(&self) -> Option<polars_arrow::array::StructArray> {
+        self.stmt.step_polars()
     }
 
     #[inline]