diff --git a/.gitignore b/.gitignore index 6463c2e..e007bda 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ Cargo.lock /target .idea +.DS_Store diff --git a/Cargo.toml b/Cargo.toml index a526568..f2ee4fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,23 +28,23 @@ keywords = ["arrow", "arrow-rs", "datafusion"] rust-version = "1.80" [dependencies] -arrow = "54" -arrow-schema = "54" +arrow = "55" +arrow-schema = "55" async-trait = "0.1" dashmap = "6" -datafusion = "46" -datafusion-common = "46" -datafusion-expr = "46" -datafusion-functions = "46" -datafusion-functions-aggregate = "46" -datafusion-optimizer = "46" -datafusion-physical-expr = "46" -datafusion-physical-plan = "46" -datafusion-sql = "46" +datafusion = "47" +datafusion-common = "47" +datafusion-expr = "47" +datafusion-functions = "47" +datafusion-functions-aggregate = "47" +datafusion-optimizer = "47" +datafusion-physical-expr = "47" +datafusion-physical-plan = "47" +datafusion-sql = "47" futures = "0.3" itertools = "0.14" log = "0.4" -object_store = "0.11" +object_store = "0.12" ordered-float = "5.0.0" [dev-dependencies] diff --git a/deny.toml b/deny.toml index 90ed3af..a24420a 100644 --- a/deny.toml +++ b/deny.toml @@ -23,6 +23,7 @@ allow = [ "BSD-2-Clause", "BSD-3-Clause", "CC0-1.0", - "Unicode-3.0" + "Unicode-3.0", + "Zlib" ] version = 2 diff --git a/src/materialized/dependencies.rs b/src/materialized/dependencies.rs index e5bc282..3afbd96 100644 --- a/src/materialized/dependencies.rs +++ b/src/materialized/dependencies.rs @@ -31,7 +31,6 @@ use datafusion_expr::{ col, lit, utils::split_conjunction, Expr, LogicalPlan, LogicalPlanBuilder, TableScan, }; use datafusion_functions::string::expr_fn::{concat, concat_ws}; -use datafusion_optimizer::{analyzer::expand_wildcard_rule::ExpandWildcardRule, AnalyzerRule}; use datafusion_sql::TableReference; use itertools::{Either, Itertools}; use std::{collections::HashSet, sync::Arc}; @@ -111,14 +110,14 @@ impl TableFunctionImpl for FileDependenciesUdtf { "mv_dependencies: table '{table_name} is not a materialized view. (Materialized TableProviders must be registered using register_materialized"), ))?; - Ok(Arc::new(ViewTable::try_new( + Ok(Arc::new(ViewTable::new( mv_dependencies_plan( mv, self.row_metadata_registry.as_ref(), &self.config_options, )?, None, - )?)) + ))) } } @@ -216,7 +215,7 @@ impl TableFunctionImpl for StaleFilesUdtf { ])? .build()?; - Ok(Arc::new(ViewTable::try_new(logical_plan, None)?)) + Ok(Arc::new(ViewTable::new(logical_plan, None))) } } @@ -250,9 +249,6 @@ pub fn mv_dependencies_plan( .filter_map(|(i, f)| partition_cols.contains(f.name()).then_some(i)) .collect(); - // First expand all wildcards - let plan = ExpandWildcardRule {}.analyze(plan, config_options)?; - let pruned_plan_with_source_files = if partition_cols.is_empty() { get_source_files_all_partitions( materialized_view, diff --git a/src/materialized/file_metadata.rs b/src/materialized/file_metadata.rs index de67d1c..02fcac1 100644 --- a/src/materialized/file_metadata.rs +++ b/src/materialized/file_metadata.rs @@ -668,7 +668,7 @@ impl FileMetadataBuilder { .append_value(format!("{store_url}{}", meta.location)); self.last_modified .append_option(meta.last_modified.timestamp_nanos_opt()); - self.size.append_value(meta.size as u64); // this is not lossy assuming we're on a 64-bit platform + self.size.append_value(meta.size); // this is not lossy assuming we're on a 64-bit platform } fn finish(mut self) -> Result { diff --git a/src/materialized/hive_partition.rs b/src/materialized/hive_partition.rs index 075750e..43ebfde 100644 --- a/src/materialized/hive_partition.rs +++ b/src/materialized/hive_partition.rs @@ -22,8 +22,8 @@ use arrow_schema::DataType; use datafusion_common::{DataFusionError, Result, ScalarValue}; use datafusion_expr::{ - expr::ScalarFunction, ColumnarValue, Expr, ScalarUDF, ScalarUDFImpl, Signature, TypeSignature, - Volatility, + expr::ScalarFunction, ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, + Signature, TypeSignature, Volatility, }; pub static HIVE_PARTITION_UDF_NAME: &str = "hive_partition"; @@ -101,7 +101,8 @@ impl ScalarUDFImpl for HivePartitionUdf { Ok(DataType::Utf8) } - fn invoke(&self, values: &[ColumnarValue]) -> Result { + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let values = args.args; let null_if_missing = values .get(2) .map(|val| match val { @@ -113,7 +114,7 @@ impl ScalarUDFImpl for HivePartitionUdf { .transpose()? .unwrap_or(false); - let arrays = ColumnarValue::values_to_arrays(values)?; + let arrays = ColumnarValue::values_to_arrays(&values)?; let [file_paths, table_partition_columns]: [Option<&StringArray>; 2] = [&arrays[0], &arrays[1]].map(|arg| arg.as_any().downcast_ref()); diff --git a/src/rewrite/exploitation.rs b/src/rewrite/exploitation.rs index 4819bc4..69c485b 100644 --- a/src/rewrite/exploitation.rs +++ b/src/rewrite/exploitation.rs @@ -296,12 +296,6 @@ impl UserDefinedLogicalNodeCore for OneOf { write!(f, "OneOf") } - fn from_template(&self, _exprs: &[datafusion::prelude::Expr], inputs: &[LogicalPlan]) -> Self { - Self { - branches: inputs.to_vec(), - } - } - fn with_exprs_and_inputs( &self, _exprs: Vec, @@ -459,6 +453,7 @@ impl DisplayAs for OneOfExec { ) ) } + DisplayFormatType::TreeRender => Ok(()), } } }