Merge branch 'main' into append-should-not-conflict

Author: itamarst

.github/workflows/python_build.yml

@@ -160,7 +160,7 @@ jobs:
       - uses: actions/setup-java@v2
         with:
           distribution: "zulu"
-          java-version: "11"
+          java-version: "21"
 
       - name: Build and install deltalake
         run: make develop-pyspark

Cargo.toml

@@ -26,34 +26,27 @@ debug = true
 debug = "line-tables-only"
 
 [workspace.dependencies]
-delta_kernel = { version = "0.11.0", features = ["arrow-55", "internal-api"] }
+delta_kernel = { version = "0.12.1", features = ["arrow-55", "internal-api"] }
 
 # arrow
-arrow = { version = "=55.0.0" }
-arrow-arith = { version = "=55.0.0" }
-arrow-array = { version = "=55.0.0", features = ["chrono-tz"] }
-arrow-buffer = { version = "=55.0.0" }
-arrow-cast = { version = "=55.0.0" }
-arrow-ipc = { version = "=55.0.0" }
-arrow-json = { version = "=55.0.0" }
-arrow-ord = { version = "=55.0.0" }
-arrow-row = { version = "=55.0.0" }
-arrow-schema = { version = "=55.0.0" }
-arrow-select = { version = "=55.0.0" }
+arrow = { version = "55.2.0" }
+arrow-arith = { version = "55.2.0" }
+arrow-array = { version = "55.2.0", features = ["chrono-tz"] }
+arrow-buffer = { version = "55.2.0" }
+arrow-cast = { version = "55.2.0" }
+arrow-ipc = { version = "55.2.0" }
+arrow-json = { version = "55.2.0" }
+arrow-ord = { version = "55.2.0" }
+arrow-row = { version = "55.2.0" }
+arrow-schema = { version = "55.2.0" }
+arrow-select = { version = "55.2.0" }
 object_store = { version = "0.12.1" }
-parquet = { version = "=55.0.0" }
+parquet = { version = "55.2.0" }
 
 # datafusion
-datafusion = "47.0.0"
-datafusion-expr = "47.0.0"
-datafusion-common = "47.0.0"
-datafusion-ffi = "47.0.0"
-datafusion-functions = "47.0.0"
-datafusion-functions-aggregate = "47.0.0"
-datafusion-physical-expr = "47.0.0"
-datafusion-physical-plan = "47.0.0"
-datafusion-proto = "47.0.0"
-datafusion-sql = "47.0.0"
+datafusion = "48.0.0"
+datafusion-ffi = "48.0.0"
+datafusion-proto = "48.0.0"
 
 # serde
 serde = { version = "1.0.194", features = ["derive"] }
@@ -93,3 +86,5 @@ Arro = "Arro"
 arro3 = "arro3"
 Arro3 = "Arro3"
 AKS = "AKS"
+# to avoid using 'type' as a field name.
+tpe = "tpe"

crates/aws/Cargo.toml

@@ -38,11 +38,13 @@ aws-config = { version = "1.5", default-features = false, features = [
     "credentials-process",
     "sso",
 ] }
-aws-sdk-dynamodb = { version = "1.45", default-features = false, features = [
+aws-sdk-sso = { version = "=1.72.0"}
+aws-sdk-ssooidc = { version = "=1.73.0"}
+aws-sdk-dynamodb = { version = "=1.79.0", default-features = false, features = [
     "behavior-version-latest",
     "rt-tokio",
 ] }
-aws-sdk-sts = { version = "1.42", default-features = false, features = [
+aws-sdk-sts = { version = "=1.73.0", default-features = false, features = [
     "behavior-version-latest",
     "rt-tokio",
 ] }

crates/azure/tests/integration.rs

@@ -97,7 +97,7 @@ async fn read_write_test_onelake(context: &IntegrationContext, path: &Path) -> T
 #[serial]
 fn list_delta_tables_using_listing_provider_with_missing_account_name() -> TestResult {
     let context = IntegrationContext::new(Box::new(MsftIntegration::default()))?;
-    // Removing the the envs set by the `IntegrationContext (az_cli::prepare_env())` to illustrate the issue if e.g. account_name is not set from custom `storage_options`, but still preserving the use of the `IntegrationContext`
+    // Removing the envs set by the `IntegrationContext (az_cli::prepare_env())` to illustrate the issue if e.g. account_name is not set from custom `storage_options`, but still preserving the use of the `IntegrationContext`
     std::env::remove_var("AZURE_STORAGE_USE_EMULATOR");
     std::env::remove_var("AZURE_STORAGE_ACCOUNT_NAME");
     std::env::remove_var("AZURE_STORAGE_TOKEN");
@@ -116,7 +116,7 @@ fn list_delta_tables_using_listing_provider_with_missing_account_name() -> TestR
 #[serial]
 async fn list_delta_tables_using_listing_provider_with_account_name() -> TestResult {
     let context = IntegrationContext::new(Box::new(MsftIntegration::default()))?;
-    // Removing the the envs set by the `IntegrationContext (az_cli::prepare_env())` to illustrate the issue if e.g. account_name is not set from custom `storage_options`, but still preserving the use of the `IntegrationContext`
+    // Removing the envs set by the `IntegrationContext (az_cli::prepare_env())` to illustrate the issue if e.g. account_name is not set from custom `storage_options`, but still preserving the use of the `IntegrationContext`
     std::env::remove_var("AZURE_STORAGE_USE_EMULATOR");
     std::env::remove_var("AZURE_STORAGE_ACCOUNT_NAME");
     std::env::remove_var("AZURE_STORAGE_TOKEN");

crates/benchmarks/Cargo.toml

@@ -34,11 +34,6 @@ serde_json = { workspace = true }
 
 # datafusion
 datafusion = { workspace = true }
-datafusion-expr = { workspace = true }
-datafusion-common = { workspace = true }
-datafusion-proto = { workspace = true }
-datafusion-sql = { workspace = true }
-datafusion-physical-expr = { workspace = true }
 
 [dependencies.deltalake-core]
 path = "../core"

crates/benchmarks/src/bin/merge.rs

@@ -7,10 +7,10 @@ use arrow::datatypes::Schema as ArrowSchema;
 use arrow_array::{RecordBatch, StringArray, UInt32Array};
 use chrono::Duration;
 use clap::{command, Args, Parser, Subcommand};
+use datafusion::common::DataFusionError;
 use datafusion::functions::expr_fn::random;
+use datafusion::logical_expr::{cast, col, lit};
 use datafusion::{datasource::MemTable, prelude::DataFrame};
-use datafusion_common::DataFusionError;
-use datafusion_expr::{cast, col, lit};
 use deltalake_core::protocol::SaveMode;
 use deltalake_core::{
     arrow::{
@@ -223,7 +223,7 @@ async fn benchmark_merge_tpcds(
 
     let row_sample = ctx.table("t1").await?.join(
         ctx.table("file_sample").await?,
-        datafusion_common::JoinType::Inner,
+        datafusion::common::JoinType::Inner,
         &["file_path"],
         &["file"],
         None,
@@ -619,7 +619,7 @@ async fn main() {
             before_stats
                 .join(
                     after_stats,
-                    datafusion_common::JoinType::Inner,
+                    datafusion::common::JoinType::Inner,
                     &["before_name"],
                     &["after_name"],
                     None,

crates/catalog-glue/Cargo.toml

@@ -14,7 +14,7 @@ rust-version.workspace = true
 [dependencies]
 async-trait = { workspace = true }
 aws-config = "1"
-aws-sdk-glue = "1"
+aws-sdk-glue = "=1.78.0"
 deltalake-core = { version = "0.26.0", path = "../core" }
 thiserror = { workspace = true }
 

crates/catalog-unity/Cargo.toml

@@ -30,7 +30,6 @@ reqwest-middleware = { version = "0.4.0", features = ["json"] }
 rand = "0.8"
 dashmap = "6"
 datafusion = { workspace = true, optional = true }
-datafusion-common = { workspace = true, optional = true }
 moka = { version = "0.12", optional = true, features = ["future"] }
 
 [dev-dependencies]
@@ -45,7 +44,7 @@ aws = ["deltalake-aws"]
 azure = ["deltalake-azure"]
 gcp = ["deltalake-gcp"]
 r2 = ["deltalake-aws"]
-datafusion = ["dep:datafusion", "datafusion-common", "deltalake-core/datafusion", "moka"]
+datafusion = ["dep:datafusion", "deltalake-core/datafusion", "moka"]
 
 [[example]]
 name = "uc_example"

crates/catalog-unity/src/datafusion.rs

@@ -4,8 +4,8 @@ use chrono::prelude::*;
 use dashmap::DashMap;
 use datafusion::catalog::SchemaProvider;
 use datafusion::catalog::{CatalogProvider, CatalogProviderList};
+use datafusion::common::DataFusionError;
 use datafusion::datasource::TableProvider;
-use datafusion_common::DataFusionError;
 use futures::FutureExt;
 use moka::future::Cache;
 use moka::Expiry;
@@ -204,7 +204,10 @@ impl SchemaProvider for UnitySchemaProvider {
         self.table_names.clone()
     }
 
-    async fn table(&self, name: &str) -> datafusion_common::Result<Option<Arc<dyn TableProvider>>> {
+    async fn table(
+        &self,
+        name: &str,
+    ) -> datafusion::common::Result<Option<Arc<dyn TableProvider>>> {
         let maybe_table = self
             .client
             .get_table(&self.catalog_name, &self.schema_name, name)

crates/catalog-unity/src/lib.rs

@@ -109,7 +109,7 @@ pub enum UnityCatalogError {
 
     #[cfg(feature = "datafusion")]
     #[error("Datafusion error: {0}")]
-    DatafusionError(#[from] datafusion_common::DataFusionError),
+    DatafusionError(#[from] ::datafusion::common::DataFusionError),
 
     /// Cannot initialize DynamoDbConfiguration due to some sort of threading issue
     #[error("Unable to initialize Unity Catalog, potentially a threading issue")]