Upgrade df47 (#55)

Author: xudong963

.gitignore

@@ -25,3 +25,4 @@ Cargo.lock
 /target
 
 .idea
+.DS_Store

Cargo.toml

@@ -28,23 +28,23 @@ keywords = ["arrow", "arrow-rs", "datafusion"]
 rust-version = "1.80"
 
 [dependencies]
-arrow = "54"
-arrow-schema = "54"
+arrow = "55"
+arrow-schema = "55"
 async-trait = "0.1"
 dashmap = "6"
-datafusion = "46"
-datafusion-common = "46"
-datafusion-expr = "46"
-datafusion-functions = "46"
-datafusion-functions-aggregate = "46"
-datafusion-optimizer = "46"
-datafusion-physical-expr = "46"
-datafusion-physical-plan = "46"
-datafusion-sql = "46"
+datafusion = "47"
+datafusion-common = "47"
+datafusion-expr = "47"
+datafusion-functions = "47"
+datafusion-functions-aggregate = "47"
+datafusion-optimizer = "47"
+datafusion-physical-expr = "47"
+datafusion-physical-plan = "47"
+datafusion-sql = "47"
 futures = "0.3"
 itertools = "0.14"
 log = "0.4"
-object_store = "0.11"
+object_store = "0.12"
 ordered-float = "5.0.0"
 
 [dev-dependencies]

deny.toml

@@ -23,6 +23,7 @@ allow = [
   "BSD-2-Clause",
   "BSD-3-Clause",
   "CC0-1.0",
-  "Unicode-3.0"
+  "Unicode-3.0",
+  "Zlib"
 ]
 version = 2

src/materialized/dependencies.rs

@@ -31,7 +31,6 @@ use datafusion_expr::{
     col, lit, utils::split_conjunction, Expr, LogicalPlan, LogicalPlanBuilder, TableScan,
 };
 use datafusion_functions::string::expr_fn::{concat, concat_ws};
-use datafusion_optimizer::{analyzer::expand_wildcard_rule::ExpandWildcardRule, AnalyzerRule};
 use datafusion_sql::TableReference;
 use itertools::{Either, Itertools};
 use std::{collections::HashSet, sync::Arc};
@@ -111,14 +110,14 @@ impl TableFunctionImpl for FileDependenciesUdtf {
             "mv_dependencies: table '{table_name} is not a materialized view. (Materialized TableProviders must be registered using register_materialized"),
         ))?;
 
-        Ok(Arc::new(ViewTable::try_new(
+        Ok(Arc::new(ViewTable::new(
             mv_dependencies_plan(
                 mv,
                 self.row_metadata_registry.as_ref(),
                 &self.config_options,
             )?,
             None,
-        )?))
+        )))
     }
 }
 
@@ -216,7 +215,7 @@ impl TableFunctionImpl for StaleFilesUdtf {
                 ])?
                 .build()?;
 
-        Ok(Arc::new(ViewTable::try_new(logical_plan, None)?))
+        Ok(Arc::new(ViewTable::new(logical_plan, None)))
     }
 }
 
@@ -250,9 +249,6 @@ pub fn mv_dependencies_plan(
         .filter_map(|(i, f)| partition_cols.contains(f.name()).then_some(i))
         .collect();
 
-    // First expand all wildcards
-    let plan = ExpandWildcardRule {}.analyze(plan, config_options)?;
-
     let pruned_plan_with_source_files = if partition_cols.is_empty() {
         get_source_files_all_partitions(
             materialized_view,

src/materialized/file_metadata.rs

@@ -668,7 +668,7 @@ impl FileMetadataBuilder {
             .append_value(format!("{store_url}{}", meta.location));
         self.last_modified
             .append_option(meta.last_modified.timestamp_nanos_opt());
-        self.size.append_value(meta.size as u64); // this is not lossy assuming we're on a 64-bit platform
+        self.size.append_value(meta.size); // this is not lossy assuming we're on a 64-bit platform
     }
 
     fn finish(mut self) -> Result<RecordBatch> {

src/materialized/hive_partition.rs

@@ -22,8 +22,8 @@ use arrow_schema::DataType;
 
 use datafusion_common::{DataFusionError, Result, ScalarValue};
 use datafusion_expr::{
-    expr::ScalarFunction, ColumnarValue, Expr, ScalarUDF, ScalarUDFImpl, Signature, TypeSignature,
-    Volatility,
+    expr::ScalarFunction, ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl,
+    Signature, TypeSignature, Volatility,
 };
 
 pub static HIVE_PARTITION_UDF_NAME: &str = "hive_partition";
@@ -101,7 +101,8 @@ impl ScalarUDFImpl for HivePartitionUdf {
         Ok(DataType::Utf8)
     }
 
-    fn invoke(&self, values: &[ColumnarValue]) -> Result<ColumnarValue> {
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+        let values = args.args;
         let null_if_missing = values
             .get(2)
             .map(|val| match val {
@@ -113,7 +114,7 @@ impl ScalarUDFImpl for HivePartitionUdf {
             .transpose()?
             .unwrap_or(false);
 
-        let arrays = ColumnarValue::values_to_arrays(values)?;
+        let arrays = ColumnarValue::values_to_arrays(&values)?;
 
         let [file_paths, table_partition_columns]: [Option<&StringArray>; 2] =
             [&arrays[0], &arrays[1]].map(|arg| arg.as_any().downcast_ref());

src/rewrite/exploitation.rs

@@ -296,12 +296,6 @@ impl UserDefinedLogicalNodeCore for OneOf {
         write!(f, "OneOf")
     }
 
-    fn from_template(&self, _exprs: &[datafusion::prelude::Expr], inputs: &[LogicalPlan]) -> Self {
-        Self {
-            branches: inputs.to_vec(),
-        }
-    }
-
     fn with_exprs_and_inputs(
         &self,
         _exprs: Vec<datafusion::prelude::Expr>,
@@ -459,6 +453,7 @@ impl DisplayAs for OneOfExec {
                     )
                 )
             }
+            DisplayFormatType::TreeRender => Ok(()),
         }
     }
 }