Try to fix python bindings.

Author: mbutrovich

bindings/python/Cargo.lock

@@ -31,9 +31,9 @@ license = "Apache-2.0"
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow = { version = "56", features = ["pyarrow", "chrono-tz"] }
+arrow = { version = "57", features = ["pyarrow", "chrono-tz"] }
 iceberg = { path = "../../crates/iceberg" }
-pyo3 = { version = "0.25", features = ["extension-module", "abi3-py39"] }
+pyo3 = { version = "0.26", features = ["extension-module", "abi3-py39"] }
 iceberg-datafusion = { path = "../../crates/integrations/datafusion" }
-datafusion-ffi = { version = "50" }
+datafusion-ffi = { git = "https://github.com/apache/datafusion.git", rev = "f32984b2dbf9e5a193c20643ce624167295fbd61" }
 tokio = { version = "1.46.1", default-features = false }

bindings/python/Cargo.toml

@@ -24,54 +24,54 @@ use pyo3::prelude::*;
 use crate::error::to_py_err;
 
 #[pyfunction]
-pub fn identity(py: Python, array: PyObject) -> PyResult<PyObject> {
+pub fn identity(py: Python, array: Py<PyAny>) -> PyResult<Py<PyAny>> {
     apply(py, array, Transform::Identity)
 }
 
 #[pyfunction]
-pub fn void(py: Python, array: PyObject) -> PyResult<PyObject> {
+pub fn void(py: Python, array: Py<PyAny>) -> PyResult<Py<PyAny>> {
     apply(py, array, Transform::Void)
 }
 
 #[pyfunction]
-pub fn year(py: Python, array: PyObject) -> PyResult<PyObject> {
+pub fn year(py: Python, array: Py<PyAny>) -> PyResult<Py<PyAny>> {
     apply(py, array, Transform::Year)
 }
 
 #[pyfunction]
-pub fn month(py: Python, array: PyObject) -> PyResult<PyObject> {
+pub fn month(py: Python, array: Py<PyAny>) -> PyResult<Py<PyAny>> {
     apply(py, array, Transform::Month)
 }
 
 #[pyfunction]
-pub fn day(py: Python, array: PyObject) -> PyResult<PyObject> {
+pub fn day(py: Python, array: Py<PyAny>) -> PyResult<Py<PyAny>> {
     apply(py, array, Transform::Day)
 }
 
 #[pyfunction]
-pub fn hour(py: Python, array: PyObject) -> PyResult<PyObject> {
+pub fn hour(py: Python, array: Py<PyAny>) -> PyResult<Py<PyAny>> {
     apply(py, array, Transform::Hour)
 }
 
 #[pyfunction]
-pub fn bucket(py: Python, array: PyObject, num_buckets: u32) -> PyResult<PyObject> {
+pub fn bucket(py: Python, array: Py<PyAny>, num_buckets: u32) -> PyResult<Py<PyAny>> {
     apply(py, array, Transform::Bucket(num_buckets))
 }
 
 #[pyfunction]
-pub fn truncate(py: Python, array: PyObject, width: u32) -> PyResult<PyObject> {
+pub fn truncate(py: Python, array: Py<PyAny>, width: u32) -> PyResult<Py<PyAny>> {
     apply(py, array, Transform::Truncate(width))
 }
 
-fn apply(py: Python, array: PyObject, transform: Transform) -> PyResult<PyObject> {
+fn apply(py: Python, array: Py<PyAny>, transform: Transform) -> PyResult<Py<PyAny>> {
     // import
     let array = ArrayData::from_pyarrow_bound(array.bind(py))?;
     let array = make_array(array);
     let transform_function = create_transform_function(&transform).map_err(to_py_err)?;
     let array = transform_function.transform(array).map_err(to_py_err)?;
     // export
     let array = array.into_data();
-    array.to_pyarrow(py)
+    Ok(array.to_pyarrow(py)?.unbind())
 }
 
 pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {

bindings/python/src/transform.rs

@@ -1783,7 +1783,7 @@ message schema {
         assert_eq!(err.kind(), ErrorKind::DataInvalid);
         assert_eq!(
             err.to_string(),
-            "DataInvalid => Unsupported Arrow data type: Duration(Microsecond)".to_string()
+            "DataInvalid => Unsupported Arrow data type: Duration(µs)".to_string()
         );
 
         // Omitting field c2, we still get an error due to c3 being selected

crates/iceberg/src/arrow/reader.rs

@@ -296,18 +296,18 @@ mod tests {
         check_record_batches(
             record_batch.try_collect::<Vec<_>>().await.unwrap(),
             expect![[r#"
-                Field { name: "content", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "14"} },
-                Field { name: "path", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "1"} },
-                Field { name: "length", data_type: Int64, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "2"} },
-                Field { name: "partition_spec_id", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "3"} },
-                Field { name: "added_snapshot_id", data_type: Int64, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "4"} },
-                Field { name: "added_data_files_count", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "5"} },
-                Field { name: "existing_data_files_count", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "6"} },
-                Field { name: "deleted_data_files_count", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "7"} },
-                Field { name: "added_delete_files_count", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "15"} },
-                Field { name: "existing_delete_files_count", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "16"} },
-                Field { name: "deleted_delete_files_count", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "17"} },
-                Field { name: "partition_summaries", data_type: List(Field { name: "item", data_type: Struct([Field { name: "contains_null", data_type: Boolean, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "10"} }, Field { name: "contains_nan", data_type: Boolean, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "11"} }, Field { name: "lower_bound", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "12"} }, Field { name: "upper_bound", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "13"} }]), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "9"} }), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "8"} }"#]],
+                Field { "content": Int32, metadata: {"PARQUET:field_id": "14"} },
+                Field { "path": Utf8, metadata: {"PARQUET:field_id": "1"} },
+                Field { "length": Int64, metadata: {"PARQUET:field_id": "2"} },
+                Field { "partition_spec_id": Int32, metadata: {"PARQUET:field_id": "3"} },
+                Field { "added_snapshot_id": Int64, metadata: {"PARQUET:field_id": "4"} },
+                Field { "added_data_files_count": Int32, metadata: {"PARQUET:field_id": "5"} },
+                Field { "existing_data_files_count": Int32, metadata: {"PARQUET:field_id": "6"} },
+                Field { "deleted_data_files_count": Int32, metadata: {"PARQUET:field_id": "7"} },
+                Field { "added_delete_files_count": Int32, metadata: {"PARQUET:field_id": "15"} },
+                Field { "existing_delete_files_count": Int32, metadata: {"PARQUET:field_id": "16"} },
+                Field { "deleted_delete_files_count": Int32, metadata: {"PARQUET:field_id": "17"} },
+                Field { "partition_summaries": List(Struct("contains_null": Boolean, metadata: {"PARQUET:field_id": "10"}, "contains_nan": nullable Boolean, metadata: {"PARQUET:field_id": "11"}, "lower_bound": nullable Utf8, metadata: {"PARQUET:field_id": "12"}, "upper_bound": nullable Utf8, metadata: {"PARQUET:field_id": "13"}), metadata: {"PARQUET:field_id": "9"}), metadata: {"PARQUET:field_id": "8"} }"#]],
             expect![[r#"
                 content: PrimitiveArray<Int32>
                 [

crates/iceberg/src/inspect/manifests.rs

@@ -151,14 +151,14 @@ mod tests {
         check_record_batches(
             batch_stream.try_collect::<Vec<_>>().await.unwrap(),
             expect![[r#"
-                Field { name: "committed_at", data_type: Timestamp(Microsecond, Some("+00:00")), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "1"} },
-                Field { name: "snapshot_id", data_type: Int64, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "2"} },
-                Field { name: "parent_id", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "3"} },
-                Field { name: "operation", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "4"} },
-                Field { name: "manifest_list", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "5"} },
-                Field { name: "summary", data_type: Map(Field { name: "key_value", data_type: Struct([Field { name: "key", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "7"} }, Field { name: "value", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "8"} }]), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, false), nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "6"} }"#]],
+                Field { "committed_at": Timestamp(µs, "+00:00"), metadata: {"PARQUET:field_id": "1"} },
+                Field { "snapshot_id": Int64, metadata: {"PARQUET:field_id": "2"} },
+                Field { "parent_id": nullable Int64, metadata: {"PARQUET:field_id": "3"} },
+                Field { "operation": nullable Utf8, metadata: {"PARQUET:field_id": "4"} },
+                Field { "manifest_list": nullable Utf8, metadata: {"PARQUET:field_id": "5"} },
+                Field { "summary": nullable Map("key_value": Struct("key": Utf8, metadata: {"PARQUET:field_id": "7"}, "value": nullable Utf8, metadata: {"PARQUET:field_id": "8"}), unsorted), metadata: {"PARQUET:field_id": "6"} }"#]],
             expect![[r#"
-                committed_at: PrimitiveArray<Timestamp(Microsecond, Some("+00:00"))>
+                committed_at: PrimitiveArray<Timestamp(µs, "+00:00")>
                 [
                   2018-01-04T21:22:35.770+00:00,
                   2019-04-12T20:29:15.770+00:00,

crates/iceberg/src/inspect/snapshots.rs

@@ -244,6 +244,7 @@ mod tests {
             constraints: Constraints::default(),
             column_defaults: Default::default(),
             if_not_exists: Default::default(),
+            or_replace: false,
             temporary: false,
             definition: Default::default(),
             unbounded: Default::default(),