Update to DF 43.0.0

Author: timsaucer

Cargo.lock

@@ -37,10 +37,10 @@ substrait = ["dep:datafusion-substrait"]
 tokio = { version = "1.39", features = ["macros", "rt", "rt-multi-thread", "sync"] }
 pyo3 = { version = "0.22", features = ["extension-module", "abi3", "abi3-py38"] }
 arrow = { version = "53", features = ["pyarrow"] }
-datafusion = { version = "42.0.0", features = ["pyarrow", "avro", "unicode_expressions"] }
-datafusion-substrait = { version = "42.0.0", optional = true }
-datafusion-proto = { version = "42.0.0" }
-datafusion-functions-window-common = { version = "42.0.0" }
+datafusion = { version = "43.0.0", features = ["pyarrow", "avro", "unicode_expressions"] }
+datafusion-substrait = { version = "43.0.0", optional = true }
+datafusion-proto = { version = "43.0.0" }
+datafusion-functions-window-common = { version = "43.0.0" }
 prost = "0.13" # keep in line with `datafusion-substrait`
 uuid = { version = "1.11", features = ["v4"] }
 mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] }
@@ -49,12 +49,6 @@ futures = "0.3"
 object_store = { version = "0.11.0", features = ["aws", "gcp", "azure", "http"] }
 url = "2"
 
-[patch.crates-io]
-datafusion = { git = "https://github.com/apache/datafusion.git", branch = "main" }
-datafusion-substrait = { git = "https://github.com/apache/datafusion.git", branch = "main" }
-datafusion-proto = { git = "https://github.com/apache/datafusion.git", branch = "main" }
-datafusion-functions-window-common = { git = "https://github.com/apache/datafusion.git", branch = "main" }
-
 [build-dependencies]
 prost-types = "0.13" # keep in line with `datafusion-substrait`
 pyo3-build-config = "0.22"

Cargo.toml

@@ -468,6 +468,13 @@ def test_simple_select(ctx, tmp_path, arr):
     batches = ctx.sql("SELECT a AS tt FROM t").collect()
     result = batches[0].column(0)
 
+    # In DF 43.0.0 we now default to having BinaryView and StringView
+    # so the array that is saved to the parquet is slightly different
+    # than the array read. Convert to values for comparison.
+    if isinstance(result, pa.BinaryViewArray) or isinstance(result, pa.StringViewArray):
+        arr = arr.tolist()
+        result = result.tolist()
+
     np.testing.assert_equal(result, arr)
 
 

python/tests/test_sql.py

@@ -90,6 +90,7 @@ impl PyLogicalPlan {
             | LogicalPlan::Ddl(_)
             | LogicalPlan::Copy(_)
             | LogicalPlan::DescribeTable(_)
+            | LogicalPlan::Execute(_)
             | LogicalPlan::RecursiveQuery(_) => Err(py_unsupported_variant_err(format!(
                 "Conversion of variant not implemented: {:?}",
                 self.plan