feat!: arrow 57, MSRV 1.85+ (#1424)

## What changes are proposed in this pull request?
add `arrow-57` feature flag, remove `arrow-55`. also, since arrow 57
MSRV is 1.85 now, we bump from MSRV 1.84 to 1.85

### This PR affects the following public APIs
`arrow` flag now defaults to arrow 57 instead of arrow 56, MSRV now
rustc 1.85


## How was this change tested?
existing

Author: zachschuermann

Cargo.toml

@@ -26,5 +26,5 @@ keywords = ["deltalake", "delta", "datalake"]
 license = "Apache-2.0"
 repository = "https://github.com/delta-io/delta-kernel-rs"
 readme = "README.md"
-rust-version = "1.84"
+rust-version = "1.85"
 version = "0.16.0"

README.md

@@ -1,11 +1,11 @@
-# Delta Kernel (rust)   [![build-status]][actions] [![latest-version]][crates.io] [![docs]][docs.rs] [![rustc-version-1.84+]][rustc]
+# Delta Kernel (rust)   [![build-status]][actions] [![latest-version]][crates.io] [![docs]][docs.rs] [![rustc-version-1.85+]][rustc]
 
 [build-status]: https://img.shields.io/github/actions/workflow/status/delta-io/delta-kernel-rs/build.yml?branch=main
 [actions]: https://github.com/delta-io/delta-kernel-rs/actions/workflows/build.yml?query=branch%3Amain
 [latest-version]: https://img.shields.io/crates/v/delta_kernel.svg
 [crates.io]: https://crates.io/crates/delta\_kernel
-[rustc-version-1.84+]: https://img.shields.io/badge/rustc-1.84+-lightgray.svg
-[rustc]: https://blog.rust-lang.org/2025/01/09/Rust-1.84.0/
+[rustc-version-1.85+]: https://img.shields.io/badge/rustc-1.85+-lightgray.svg
+[rustc]: https://blog.rust-lang.org/2025/02/20/Rust-1.85.0/
 [docs]: https://img.shields.io/docsrs/delta_kernel
 [docs.rs]: https://docs.rs/delta_kernel/latest/delta_kernel/
 
@@ -85,8 +85,8 @@ arrow versions as we can.
 We allow selecting the version of arrow to use via feature flags. Currently we support the following
 flags:
 
-- `arrow-55`: Use arrow version 55
 - `arrow-56`: Use arrow version 56
+- `arrow-57`: Use arrow version 57
 - `arrow`: Use the latest arrow version. Note that this is an _unstable_ flag: we will bump this to
   the latest arrow version at every arrow version release. Only removing old arrow versions will
   cause a breaking change for kernel. If you require a specific version N of arrow, you should

ffi/src/transaction/mod.rs

@@ -241,7 +241,7 @@ mod tests {
         // writer must be closed to write footer
         let res = writer.close().unwrap();
 
-        create_file_metadata(file_path, res.num_rows, metadata_schema)
+        create_file_metadata(file_path, res.file_metadata().num_rows(), metadata_schema)
     }
 
     #[tokio::test]

kernel/Cargo.toml

@@ -67,27 +67,27 @@ object_store = { version = "0.12.3", optional = true, features = ["aws", "azure"
 # TODO: Remove this once https://github.com/apache/arrow-rs/pull/8244 ships
 comfy-table = { version = "~7.1", optional = true }
 
-# arrow 55
-[dependencies.arrow_55]
+# arrow 56
+[dependencies.arrow_56]
 package = "arrow"
-version = "55"
+version = "56"
 features = ["chrono-tz", "ffi", "json", "prettyprint"]
 optional = true
-[dependencies.parquet_55]
+[dependencies.parquet_56]
 package = "parquet"
-version = "55"
+version = "56"
 features = ["async", "object_store"]
 optional = true
 
-# arrow 56
-[dependencies.arrow_56]
+# arrow 57
+[dependencies.arrow_57]
 package = "arrow"
-version = "56"
+version = "57"
 features = ["chrono-tz", "ffi", "json", "prettyprint"]
 optional = true
-[dependencies.parquet_56]
+[dependencies.parquet_57]
 package = "parquet"
-version = "56"
+version = "57"
 features = ["async", "object_store"]
 optional = true
 
@@ -100,11 +100,11 @@ internal-api = []
 integration-test = ["hdfs-native-object-store/integration-test"]
 
 # The default versions for arrow/parquet/object_store
-arrow = ["arrow-56"] # latest arrow version
+arrow = ["arrow-57"] # latest arrow version
 need-arrow = [] # need-arrow is a marker that the feature needs arrow dep
 
-arrow-55 = ["dep:arrow_55", "dep:parquet_55", "object_store", "comfy-table"]
 arrow-56 = ["dep:arrow_56", "dep:parquet_56", "object_store", "comfy-table"]
+arrow-57 = ["dep:arrow_57", "dep:parquet_57", "object_store", "comfy-table"]
 arrow-conversion = ["need-arrow"]
 arrow-expression = ["need-arrow"]
 

kernel/examples/read-table-multi-threaded/Cargo.toml

@@ -5,11 +5,12 @@ edition = "2021"
 publish = false
 
 [dependencies]
-arrow = { version = "56", features = ["prettyprint", "chrono-tz"] }
+arrow = { version = "57", features = ["prettyprint", "chrono-tz"] }
 clap = { version = "4.5", features = ["derive"] }
+# common pulls in arrow latest so we have to keep all these in sync here
 common = { path = "../common" }
 delta_kernel = { path = "../../../kernel", features = [
-  "arrow-56",
+  "arrow",
   "default-engine-rustls",
   "internal-api",
 ] }

kernel/examples/read-table-single-threaded/Cargo.toml

@@ -5,11 +5,11 @@ edition = "2021"
 publish = false
 
 [dependencies]
-arrow = { version = "56", features = ["prettyprint", "chrono-tz"] }
+arrow = { version = "57", features = ["prettyprint", "chrono-tz"] }
 clap = { version = "4.5", features = ["derive"] }
 common = { path = "../common" }
 delta_kernel = { path = "../../../kernel", features = [
-  "arrow-56",
+  "arrow",
   "default-engine-rustls",
   "internal-api",
 ] }

kernel/examples/write-table/Cargo.toml

@@ -5,11 +5,12 @@ edition = "2021"
 publish = false
 
 [dependencies]
-arrow = { version = "56", features = ["prettyprint", "chrono-tz"] }
+arrow = { version = "57", features = ["prettyprint", "chrono-tz"] }
 clap = { version = "4.5", features = ["derive"] }
+# NB: common depends on 'arrow' (latest) so have to match here
 common = { path = "../common" }
 delta_kernel = { path = "../../../kernel", features = [
-  "arrow-56",
+  "arrow",
   "default-engine-rustls",
   "internal-api",
 ] }

kernel/src/arrow_compat.rs

@@ -1,25 +1,25 @@
 //! This module re-exports the different versions of arrow, parquet, and object_store we support.
 
-#[cfg(feature = "arrow-56")]
+#[cfg(feature = "arrow-57")]
 mod arrow_compat_shims {
-    pub use arrow_56 as arrow;
-    pub use parquet_56 as parquet;
+    pub use arrow_57 as arrow;
+    pub use parquet_57 as parquet;
 }
 
-#[cfg(all(feature = "arrow-55", not(feature = "arrow-56")))]
+#[cfg(all(feature = "arrow-56", not(feature = "arrow-57")))]
 mod arrow_compat_shims {
-    pub use arrow_55 as arrow;
-    pub use parquet_55 as parquet;
+    pub use arrow_56 as arrow;
+    pub use parquet_56 as parquet;
 }
 
 // if nothing is enabled but we need arrow because of some other feature flag, throw compile-time
 // error
 #[cfg(all(
     feature = "need-arrow",
-    not(feature = "arrow-55"),
-    not(feature = "arrow-56")
+    not(feature = "arrow-56"),
+    not(feature = "arrow-57")
 ))]
-compile_error!("Requested a feature that needs arrow without enabling arrow. Please enable the `arrow-55` or `arrow-56` feature");
+compile_error!("Requested a feature that needs arrow without enabling arrow. Please enable the `arrow-56` or `arrow-57` feature");
 
-#[cfg(any(feature = "arrow-55", feature = "arrow-56"))]
+#[cfg(any(feature = "arrow-56", feature = "arrow-57"))]
 pub use arrow_compat_shims::*;

kernel/src/checkpoint/tests.rs

@@ -6,6 +6,10 @@ use crate::action_reconciliation::{
 use crate::actions::{Add, Metadata, Protocol, Remove};
 use crate::arrow::array::{ArrayRef, StructArray};
 use crate::arrow::datatypes::{DataType, Schema};
+use crate::arrow::{
+    array::{create_array, RecordBatch},
+    datatypes::Field,
+};
 use crate::checkpoint::create_last_checkpoint_data;
 use crate::engine::arrow_data::ArrowEngineData;
 use crate::engine::default::{executor::tokio::TokioBackgroundExecutor, DefaultEngine};
@@ -14,11 +18,6 @@ use crate::schema::{DataType as KernelDataType, StructField, StructType};
 use crate::utils::test_utils::Action;
 use crate::{DeltaResult, FileMeta, LogPath, Snapshot};
 
-use arrow_56::{
-    array::{create_array, RecordBatch},
-    datatypes::Field,
-};
-
 use object_store::{memory::InMemory, path::Path, ObjectStore};
 use serde_json::{from_slice, json, Value};
 use test_utils::delta_path_for_version;

kernel/src/engine/ensure_data_types.rs

@@ -352,7 +352,7 @@ mod tests {
                 &incorrect_variant_arrow_type(),
                 true,
             ),
-            "Invalid argument error: Incorrect datatype. Expected Struct(metadata Binary, value Binary), got Struct(field_1 Binary, field_2 Binary)",
+            "Invalid argument error: Incorrect datatype. Expected Struct(\"metadata\": Binary, \"value\": Binary), got Struct(\"field_1\": nullable Binary, \"field_2\": nullable Binary)",
         )
     }