DataFusion 49 upgrade

Cargo.toml

@@ -219,10 +219,10 @@ convert_case = "0.6"
 criterion = "0.5"
 cros-codecs = "0.0.6"
 crossbeam = "0.8"
-datafusion = { version = "47", default-features = false, features = [
+datafusion = { version = "49.0.2", default-features = false, features = [
   "nested_expressions",
 ] }
-datafusion-ffi = "47"
+datafusion-ffi = "49.0.2"
 directories = "5"
 document-features = "0.2.8"
 econtext = "0.2" # Prints error contexts on crashes

crates/store/re_datafusion/src/dataframe_query_common.rs

@@ -223,8 +223,8 @@ impl DataframeQueryTableProvider {
             }
             Expr::BinaryExpr(binary) => {
                 if binary.op == Operator::NotEq
-                    && let (Expr::Column(col), Expr::Literal(sv))
-                    | (Expr::Literal(sv), Expr::Column(col)) =
+                    && let (Expr::Column(col), Expr::Literal(sv, _))
+                    | (Expr::Literal(sv, _), Expr::Column(col)) =
                         (binary.left.as_ref(), binary.right.as_ref())
                     && sv.is_null()
                 {

crates/store/re_datafusion/src/dataframe_query_provider.rs

@@ -233,13 +233,16 @@ impl PartitionStreamExec {
                     SortOptions::new(false, true),
                 ));
             }
-            vec![LexOrdering::new(physical_ordering)]
+            vec![
+                LexOrdering::new(physical_ordering)
+                    .expect("LexOrdering should return Some since input is not empty"),
+            ]
         } else {
             vec![]
         };
 
         let eq_properties =
-            EquivalenceProperties::new_with_orderings(Arc::clone(&projected_schema), &orderings);
+            EquivalenceProperties::new_with_orderings(Arc::clone(&projected_schema), orderings);
 
         let partition_in_output_schema = projection.map(|p| p.contains(&0)).unwrap_or(false);
 

crates/store/re_datafusion/src/dataframe_query_provider_wasm.rs

@@ -232,10 +232,13 @@ impl PartitionStreamExec {
             ));
         }
 
-        let orderings = vec![LexOrdering::new(physical_ordering)];
+        let orderings = vec![
+            LexOrdering::new(physical_ordering)
+                .expect("LexOrdering should return Some when non-empty vec is passed"),
+        ];
 
         let eq_properties =
-            EquivalenceProperties::new_with_orderings(Arc::clone(&projected_schema), &orderings);
+            EquivalenceProperties::new_with_orderings(Arc::clone(&projected_schema), orderings);
 
         let partition_in_output_schema = projection.map(|p| p.contains(&0)).unwrap_or(false);
 

crates/viewer/re_dataframe_ui/src/filters.rs

@@ -1,6 +1,6 @@
 use arrow::datatypes::{DataType, Field};
-use datafusion::common::DFSchema;
-use datafusion::prelude::{Column, Expr, array_has, array_to_string, col, lit, lower};
+use datafusion::common::{DFSchema, ExprSchema as _};
+use datafusion::prelude::{Column, Expr, array_has, array_to_string, col, contains, lit, lower};
 
 #[derive(Debug, Clone, thiserror::Error)]
 pub enum FilterError {
@@ -104,7 +104,7 @@ impl FilterOperation {
                     }
                 };
 
-                Ok(contains_patch(lower(operand), lower(lit(query_string))))
+                Ok(contains(lower(operand), lower(lit(query_string))))
             }
 
             Self::BooleanEquals(value) => match field.data_type() {
@@ -125,14 +125,3 @@ impl FilterOperation {
         }
     }
 }
-
-// TODO(ab): this is a workaround for https://github.com/apache/datafusion/pull/16046. Next time we
-// update datafusion, this should break compilation. Remove this function and replace
-// `contains_patch` by `datafusion::prelude::contains` in the method above.
-fn contains_patch(arg1: Expr, arg2: Expr) -> Expr {
-    // make sure we break compilation when we update datafusion
-    #[cfg(debug_assertions)]
-    let _ = datafusion::prelude::contains();
-
-    datafusion::functions::string::contains().call(<[_]>::into_vec(Box::new([arg1, arg2])))
-}

deny.toml

@@ -29,7 +29,6 @@ all-features = true
 [advisories]
 version = 2
 ignore = [
-  "RUSTSEC-2024-0384", # Waiting for https://github.com/console-rs/indicatif/pull/666
   "RUSTSEC-2024-0436", # https://rustsec.org/advisories/RUSTSEC-2024-0436 - paste is unmaintained - https://github.com/dtolnay/paste
   "RUSTSEC-2024-0014", # https://rustsec.org/advisories/RUSTSEC-2024-0014 - generational-arena is unmaintained
 ]
@@ -54,6 +53,7 @@ deny = [
 skip = [
   { name = "base64" },           # Too popular
   { name = "block2" },           # Old version via rfd
+  { name = "bzip2" },            # Remove after https://github.com/apache/datafusion/pull/17509 closes
   { name = "cargo_metadata" },   # Older version used by ply-rs. It's small, and it's build-time only!
   { name = "core-foundation" },  # Currently, e.g. `webbrowser` and `winit` use different versions.
   { name = "hashbrown" },        # Old version used by polar-rs
@@ -62,7 +62,7 @@ skip = [
   { name = "objc2-foundation" }, # `accesskit_macos` uses a different version than `arboard`
   { name = "objc2" },            # `accesskit_macos` uses a different version than `arboard`
   { name = "ordered-float" },    # Old version being used by parquet, but super small!
-  { name = "pollster" },         # rfd is still on 0.3
+  { name = "petgraph" },         # Remove after https://github.com/tokio-rs/prost/pull/1268 resolves
   { name = "pulldown-cmark" },   # Build-dependency via `ply-rs` (!). TODO(emilk): use a better crate for .ply parsing
   { name = "redox_syscall" },    # Plenty of versions in the wild
   { name = "rustc-hash" },       # numpy with compatible pyo3 requires different version than wgpu
@@ -94,8 +94,8 @@ allow = [
   "OFL-1.1",                        # https://spdx.org/licenses/OFL-1.1.html
   "Ubuntu-font-1.0",                # https://ubuntu.com/legal/font-licence
   "Unicode-3.0",                    # https://www.unicode.org/license.txt
-  "Unicode-DFS-2016",               # https://spdx.org/licenses/Unicode-DFS-2016.html
   "Zlib",                           # https://tldrlegal.com/license/zlib-libpng-license-(zlib)
+  "bzip2-1.0.6",                    # https://github.com/trifectatechfoundation/libbzip2-rs/blob/v0.2.2/COPYING
 ]
 exceptions = []
 
@@ -109,7 +109,6 @@ name = "ring"
 expression = "MIT AND ISC AND OpenSSL"
 license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }]
 
-
 [sources]
 unknown-registry = "deny"
 unknown-git = "deny"