Merge pull request #736 from splitgraph/datafusion-43-upgrade

Upgrade to DataFusion 43

Author: gruuya

Cargo.lock

@@ -2,51 +2,40 @@
 members = ["clade", "object_store_factory"]
 
 [workspace.dependencies]
-arrow = { version = "52.2.0", features = ["test_utils"] }
-arrow-buffer = "52.2.0"
-arrow-csv = "52.2.0"
-arrow-flight = "52.2.0"
+arrow = { version = "53.2.0", features = ["test_utils"] }
+arrow-buffer = "53.2.0"
+arrow-csv = "53.2.0"
+arrow-flight = "53.2.0"
 # For the JSON format support
 # https://github.com/apache/arrow-rs/pull/2868
 # https://github.com/apache/arrow-rs/pull/2724
-arrow-integration-test = "52.2.0"
-arrow-row = "52.2.0"
-arrow-schema = "52.2.0"
+arrow-integration-test = "53.2.0"
+arrow-row = "53.2.0"
+arrow-schema = "53.2.0"
 async-trait = "0.1.83"
 
-datafusion = { version = "41.0.0", features = ["backtrace"] }
-datafusion-common = "41.0.0"
-datafusion-expr = "41.0.0"
-datafusion-functions-nested = "41.0.0"
+datafusion = { version = "43.0.0", features = ["backtrace"] }
+datafusion-common = "43.0.0"
+datafusion-expr = "43.0.0"
+datafusion-functions-nested = "43.0.0"
 
 futures = "0.3"
 
 itertools = ">=0.10.0"
-object_store = { version = "0.10.2", features = ["aws", "azure", "gcp"] }
-prost = "0.12.6"
+object_store = { version = "0.11", features = ["aws", "azure", "gcp"] }
+prost = "0.13"
 
 serde = "1.0.213"
 serde_json = "1.0.132"
 
 tempfile = "3"
 tokio = { version = "1.40", features = ["macros", "rt", "rt-multi-thread", "signal", "process"] }
+tonic = "0.12"
 tracing = { version = "0.1", features = ["log"] }
 tracing-log = "0.2"
 tracing-subscriber = { version = "0.3.18", features = ["json", "env-filter"] }
 url = "2.5"
 
-[patch.crates-io]
-arrow = { git = "https://github.com/splitgraph/arrow-rs", branch = "backport-pr6729" }
-arrow-array = { git = "https://github.com/splitgraph/arrow-rs", branch = "backport-pr6729" }
-arrow-buffer = { git = "https://github.com/splitgraph/arrow-rs", branch = "backport-pr6729" }
-arrow-csv = { git = "https://github.com/splitgraph/arrow-rs", branch = "backport-pr6729" }
-arrow-data = { git = "https://github.com/splitgraph/arrow-rs", branch = "backport-pr6729" }
-arrow-flight = { git = "https://github.com/splitgraph/arrow-rs", branch = "backport-pr6729" }
-arrow-integration-test = { git = "https://github.com/splitgraph/arrow-rs", branch = "backport-pr6729" }
-arrow-ipc = { git = "https://github.com/splitgraph/arrow-rs", branch = "backport-pr6729" }
-arrow-row = { git = "https://github.com/splitgraph/arrow-rs", branch = "backport-pr6729" }
-arrow-schema = { git = "https://github.com/splitgraph/arrow-rs", branch = "backport-pr6729" }
-
 [package]
 name = "seafowl"
 build = "build.rs"
@@ -95,8 +84,8 @@ clap = { version = "4.5.21", features = [ "derive" ] }
 config = "0.14.0"
 
 # PG wire protocol support
-convergence = { git = "https://github.com/splitgraph/convergence", branch = "datafusion-41-upgrade", optional = true }
-convergence-arrow = { git = "https://github.com/splitgraph/convergence", branch = "datafusion-41-upgrade", optional = true }
+convergence = { git = "https://github.com/splitgraph/convergence", branch = "datafusion-43-upgrade", optional = true }
+convergence-arrow = { git = "https://github.com/splitgraph/convergence", branch = "datafusion-43-upgrade", optional = true }
 
 dashmap = "6.1.0"
 
@@ -107,8 +96,7 @@ datafusion-functions-nested = { workspace = true }
 
 datafusion-remote-tables = { path = "./datafusion_remote_tables", optional = true }
 
-# pr-2975-backport, pick up https://github.com/delta-io/delta-rs/pull/2975
-deltalake = { git = "https://github.com/splitgraph/delta-rs", branch = "pr-2975-with-arrow-pr-6729-backport", features = ["datafusion"] }
+deltalake = { git = "https://github.com/splitgraph/delta-rs", branch = "fix-decimal-stat-overflow", features = ["datafusion"] }
 
 futures = "0.3"
 hex = ">=0.4.0"
@@ -118,7 +106,7 @@ lazy_static = ">=1.4.0"
 metrics = { version = "0.23.0" }
 metrics-exporter-prometheus = { version = "0.15.3" }
 moka = { version = "0.12.5", default-features = false, features = ["future", "atomic64", "quanta"] }
-object_store = { version = "0.10.2", features = ["aws", "azure", "gcp"] }
+object_store = { workspace = true }
 object_store_factory = { path = "object_store_factory" }
 percent-encoding = "2.2.0"
 prost = { workspace = true }
@@ -135,15 +123,15 @@ rustyline = "14.0"
 serde = { workspace = true }
 serde_json = { workspace = true }
 sha2 = ">=0.10.1"
-sqlparser = { version = "0.49", features = ["visitor"] }
+sqlparser = { version = "0.51", features = ["visitor"] }
 sqlx = { version = "0.7.1", features = [ "runtime-tokio-rustls", "sqlite", "any", "uuid" ] }
 strum = ">=0.24"
 strum_macros = ">=0.24"
 tempfile = "3"
 thiserror = "2"
 tokio = { workspace = true }
 tokio-graceful-shutdown = { version = "0.15" }
-tonic = { version = "0.11.0", optional = true }
+tonic = { workspace = true, optional = true }
 tower = "0.5"
 tracing = { workspace = true }
 tracing-log = "0.2"
@@ -165,7 +153,7 @@ aws-credential-types = { version = "1.2.1", features = ["hardcoded-credentials"]
 aws-sdk-sts = { version = "1.46.0", features = ["behavior-version-latest"] }
 rstest = "*"
 serial_test = "3"
-tonic-reflection = "0.11"
+tonic-reflection = "0.12"
 wiremock = "0.6"
 
 [build-dependencies]

Cargo.toml

@@ -7,7 +7,7 @@ license = "Apache-2.0"
 [dependencies]
 arrow-flight = { workspace = true }
 prost = { workspace = true }
-tonic = "0.11"
+tonic = { workspace = true }
 
 [build-dependencies]
-tonic-build = "0.11"
+tonic-build = "0.12"

clade/Cargo.toml

@@ -8,7 +8,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .build_server(true)
         .build_client(true)
         .type_attribute("clade.sync.ColumnDescriptor", "#[derive(Eq, Hash)]")
-        .compile(&["proto/schema.proto", "proto/sync.proto"], &["proto"])?;
+        .compile_protos(&["proto/schema.proto", "proto/sync.proto"], &["proto"])?;
 
     Ok(())
 }

clade/build.rs

@@ -19,7 +19,7 @@ arrow-schema = { workspace = true }
 async-trait = { workspace = true }
 
 # Remote query execution for a variety of DBs
-connectorx = { git = "https://github.com/splitgraph/connector-x", branch = "datafusion-41-upgrade", features = [ "dst_arrow", "src_postgres", "src_mysql", "src_sqlite" ] }
+connectorx = { git = "https://github.com/splitgraph/connector-x", branch = "datafusion-43-upgrade", features = [ "dst_arrow", "src_postgres", "src_mysql", "src_sqlite" ] }
 
 datafusion = { workspace = true }
 datafusion-common = { workspace = true }

datafusion_remote_tables/Cargo.toml

@@ -9,6 +9,7 @@ use std::ops::Deref;
 use std::sync::Arc;
 
 /// Factory for creating remote tables
+#[derive(Debug)]
 pub struct RemoteTableFactory {}
 
 #[async_trait]

datafusion_remote_tables/src/factory.rs

@@ -23,6 +23,7 @@ use tokio::task;
 use tracing::debug;
 
 // Implementation of a remote table, capable of querying Postgres, MySQL, SQLite, etc...
+#[derive(Debug)]
 pub struct RemoteTable {
     // We manually escape the field names during scans, but expect the user to escape the table name
     // appropriately in the remote table definition

datafusion_remote_tables/src/provider.rs

@@ -9,7 +9,7 @@ use clade::schema::ListSchemaResponse;
 
 use super::CatalogError;
 
-#[derive(Clone)]
+#[derive(Debug, Clone)]
 pub struct EmptyStore {}
 
 #[async_trait]

src/catalog/empty.rs

@@ -8,7 +8,7 @@ use tonic::transport::{channel::Channel, Endpoint, Error};
 use tonic::Request;
 
 // An external store, facilitated via a remote clade server implementation
-#[derive(Clone)]
+#[derive(Debug, Clone)]
 pub struct ExternalStore {
     client: SchemaStoreServiceClient<Channel>,
 }

src/catalog/external.rs

@@ -4,7 +4,7 @@ use crate::catalog::{
 use crate::repository::interface::AllDatabaseFunctionsResult;
 use clade::schema::ListSchemaResponse;
 
-#[derive(Clone)]
+#[derive(Debug, Clone)]
 pub struct MemoryStore {
     pub schemas: ListSchemaResponse,
 }