From 3b9a1bfa2026527b2dbce25d7bfb9ceb12d0d85e Mon Sep 17 00:00:00 2001 From: Matt Green Date: Thu, 5 Sep 2024 14:14:10 -0700 Subject: [PATCH 1/2] Use datafusion re-exported crates --- .gitignore | 1 + Cargo.lock | 5 ---- Cargo.toml | 7 +----- src/common/data_type.rs | 4 ++-- src/common/df_schema.rs | 2 +- src/common/schema.rs | 14 +++++------ src/config.rs | 2 +- src/context.rs | 6 ++--- src/dataframe.rs | 6 ++--- src/dataset.rs | 2 +- src/dataset_exec.rs | 4 ++-- src/expr.rs | 8 +++---- src/expr/aggregate.rs | 8 +++---- src/expr/aggregate_expr.rs | 2 +- src/expr/alias.rs | 2 +- src/expr/analyze.rs | 2 +- src/expr/between.rs | 2 +- src/expr/binary_expr.rs | 2 +- src/expr/bool_expr.rs | 2 +- src/expr/case.rs | 2 +- src/expr/cast.rs | 2 +- src/expr/column.rs | 2 +- src/expr/conditional_expr.rs | 2 +- src/expr/create_memory_table.rs | 2 +- src/expr/create_view.rs | 2 +- src/expr/cross_join.rs | 2 +- src/expr/distinct.rs | 2 +- src/expr/drop_table.rs | 2 +- src/expr/empty_relation.rs | 2 +- src/expr/exists.rs | 2 +- src/expr/explain.rs | 2 +- src/expr/extension.rs | 2 +- src/expr/filter.rs | 2 +- src/expr/grouping_set.rs | 2 +- src/expr/in_list.rs | 2 +- src/expr/in_subquery.rs | 2 +- src/expr/indexed_field.rs | 2 +- src/expr/join.rs | 2 +- src/expr/like.rs | 2 +- src/expr/limit.rs | 2 +- src/expr/literal.rs | 2 +- src/expr/placeholder.rs | 2 +- src/expr/projection.rs | 4 ++-- src/expr/repartition.rs | 2 +- src/expr/scalar_subquery.rs | 2 +- src/expr/signature.rs | 2 +- src/expr/sort.rs | 4 ++-- src/expr/sort_expr.rs | 2 +- src/expr/subquery.rs | 2 +- src/expr/subquery_alias.rs | 2 +- src/expr/table_scan.rs | 4 ++-- src/expr/union.rs | 2 +- src/expr/unnest.rs | 2 +- src/expr/unnest_expr.rs | 2 +- src/expr/window.rs | 6 ++--- src/functions.rs | 41 ++++++++++++++++---------------- src/lib.rs | 15 ++++++++---- src/pyarrow_filter_expression.rs | 4 ++-- src/sql/logical.rs | 2 +- src/udaf.rs | 4 +++- src/udf.rs | 6 ++--- src/utils.rs | 2 +- 62 files changed, 119 insertions(+), 118 deletions(-) diff --git a/.gitignore b/.gitignore index aaeaaa5b1..614d82327 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ dist # intended to run in multiple environments; otherwise, check them in: .python-version venv +.venv apache-rat-*.jar *rat.txt diff --git a/Cargo.lock b/Cargo.lock index bca4bf066..43ee3055d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1072,11 +1072,6 @@ dependencies = [ "arrow", "async-trait", "datafusion", - "datafusion-common", - "datafusion-expr", - "datafusion-functions-nested", - "datafusion-optimizer", - "datafusion-sql", "datafusion-substrait", "futures", "mimalloc", diff --git a/Cargo.toml b/Cargo.toml index 8881884b4..e9a4ababb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,11 +39,6 @@ rand = "0.8" pyo3 = { version = "0.21", features = ["extension-module", "abi3", "abi3-py38"] } arrow = { version = "52", feature = ["pyarrow"] } datafusion = { version = "41.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } -datafusion-common = { version = "41.0.0", features = ["pyarrow"] } -datafusion-expr = { version = "41.0.0" } -datafusion-functions-nested = { version = "41.0.0" } -datafusion-optimizer = { version = "41.0.0" } -datafusion-sql = { version = "41.0.0" } datafusion-substrait = { version = "41.0.0", optional = true } prost = "0.12" # keep in line with `datafusion-substrait` prost-types = "0.12" # keep in line with `datafusion-substrait` @@ -67,4 +62,4 @@ crate-type = ["cdylib", "rlib"] [profile.release] lto = true codegen-units = 1 - \ No newline at end of file + diff --git a/src/common/data_type.rs b/src/common/data_type.rs index bdfc5a6b1..a29d1799c 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -17,8 +17,8 @@ use datafusion::arrow::array::Array; use datafusion::arrow::datatypes::{DataType, IntervalUnit, TimeUnit}; -use datafusion_common::{DataFusionError, ScalarValue}; -use datafusion_expr::sqlparser::ast::NullTreatment as DFNullTreatment; +use datafusion::common::{DataFusionError, ScalarValue}; +use datafusion::logical_expr::sqlparser::ast::NullTreatment as DFNullTreatment; use pyo3::{exceptions::PyValueError, prelude::*}; use crate::errors::py_datafusion_err; diff --git a/src/common/df_schema.rs b/src/common/df_schema.rs index c16b8eba0..4e1d84060 100644 --- a/src/common/df_schema.rs +++ b/src/common/df_schema.rs @@ -17,7 +17,7 @@ use std::sync::Arc; -use datafusion_common::DFSchema; +use datafusion::common::DFSchema; use pyo3::prelude::*; #[derive(Debug, Clone)] diff --git a/src/common/schema.rs b/src/common/schema.rs index 00113a510..5806c90e2 100644 --- a/src/common/schema.rs +++ b/src/common/schema.rs @@ -18,10 +18,10 @@ use std::any::Any; use datafusion::arrow::datatypes::SchemaRef; -use datafusion_expr::{Expr, TableProviderFilterPushDown, TableSource}; +use datafusion::logical_expr::{Expr, TableProviderFilterPushDown, TableSource}; use pyo3::prelude::*; -use datafusion_expr::utils::split_conjunction; +use datafusion::logical_expr::utils::split_conjunction; use super::{data_type::DataTypeMap, function::SqlFunction}; @@ -166,7 +166,7 @@ impl TableSource for SqlTableSource { fn supports_filter_pushdown( &self, filter: &Expr, - ) -> datafusion_common::Result { + ) -> datafusion::common::Result { let filters = split_conjunction(filter); if filters.iter().all(|f| is_supported_push_down_expr(f)) { // Push down filters to the tablescan operation if all are supported @@ -180,22 +180,22 @@ impl TableSource for SqlTableSource { } } - fn table_type(&self) -> datafusion_expr::TableType { - datafusion_expr::TableType::Base + fn table_type(&self) -> datafusion::logical_expr::TableType { + datafusion::logical_expr::TableType::Base } #[allow(deprecated)] fn supports_filters_pushdown( &self, filters: &[&Expr], - ) -> datafusion_common::Result> { + ) -> datafusion::common::Result> { filters .iter() .map(|f| self.supports_filter_pushdown(f)) .collect() } - fn get_logical_plan(&self) -> Option<&datafusion_expr::LogicalPlan> { + fn get_logical_plan(&self) -> Option<&datafusion::logical_expr::LogicalPlan> { None } } diff --git a/src/config.rs b/src/config.rs index 82a4f93ab..3f2a05580 100644 --- a/src/config.rs +++ b/src/config.rs @@ -18,8 +18,8 @@ use pyo3::prelude::*; use pyo3::types::*; +use datafusion::common::ScalarValue; use datafusion::config::ConfigOptions; -use datafusion_common::ScalarValue; #[pyclass(name = "Config", module = "datafusion", subclass)] #[derive(Clone)] diff --git a/src/context.rs b/src/context.rs index 3ab783495..11b9fed5f 100644 --- a/src/context.rs +++ b/src/context.rs @@ -46,6 +46,7 @@ use crate::utils::{get_tokio_runtime, wait_for_future}; use datafusion::arrow::datatypes::{DataType, Schema, SchemaRef}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::arrow::record_batch::RecordBatch; +use datafusion::common::ScalarValue; use datafusion::datasource::file_format::file_compression_type::FileCompressionType; use datafusion::datasource::file_format::parquet::ParquetFormat; use datafusion::datasource::listing::{ @@ -61,7 +62,6 @@ use datafusion::physical_plan::SendableRecordBatchStream; use datafusion::prelude::{ AvroReadOptions, CsvReadOptions, DataFrame, NdJsonReadOptions, ParquetReadOptions, }; -use datafusion_common::ScalarValue; use pyo3::types::{PyDict, PyList, PyTuple}; use tokio::task::JoinHandle; @@ -962,7 +962,7 @@ impl PySessionContext { // create a Tokio runtime to run the async code let rt = &get_tokio_runtime(py).0; let plan = plan.plan.clone(); - let fut: JoinHandle> = + let fut: JoinHandle> = rt.spawn(async move { plan.execute(part, Arc::new(ctx)) }); let stream = wait_for_future(py, fut).map_err(py_datafusion_err)?; Ok(PyRecordBatchStream::new(stream?)) @@ -970,7 +970,7 @@ impl PySessionContext { } impl PySessionContext { - async fn _table(&self, name: &str) -> datafusion_common::Result { + async fn _table(&self, name: &str) -> datafusion::common::Result { self.ctx.table(name).await } } diff --git a/src/dataframe.rs b/src/dataframe.rs index 3fb8b2292..f33622cc0 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -27,12 +27,12 @@ use arrow::util::display::{ArrayFormatter, FormatOptions}; use datafusion::arrow::datatypes::Schema; use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow}; use datafusion::arrow::util::pretty; +use datafusion::common::UnnestOptions; use datafusion::config::{CsvOptions, TableParquetOptions}; use datafusion::dataframe::{DataFrame, DataFrameWriteOptions}; use datafusion::execution::SendableRecordBatchStream; use datafusion::parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel}; use datafusion::prelude::*; -use datafusion_common::UnnestOptions; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::*; use pyo3::pybacked::PyBackedStr; @@ -541,7 +541,7 @@ impl PyDataFrame { // create a Tokio runtime to run the async code let rt = &get_tokio_runtime(py).0; let df = self.df.as_ref().clone(); - let fut: JoinHandle> = + let fut: JoinHandle> = rt.spawn(async move { df.execute_stream().await }); let stream = wait_for_future(py, fut).map_err(py_datafusion_err)?; Ok(PyRecordBatchStream::new(stream?)) @@ -551,7 +551,7 @@ impl PyDataFrame { // create a Tokio runtime to run the async code let rt = &get_tokio_runtime(py).0; let df = self.df.as_ref().clone(); - let fut: JoinHandle>> = + let fut: JoinHandle>> = rt.spawn(async move { df.execute_stream_partitioned().await }); let stream = wait_for_future(py, fut).map_err(py_datafusion_err)?; diff --git a/src/dataset.rs b/src/dataset.rs index b5704164f..de7402fd6 100644 --- a/src/dataset.rs +++ b/src/dataset.rs @@ -31,9 +31,9 @@ use datafusion::arrow::datatypes::SchemaRef; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::datasource::{TableProvider, TableType}; use datafusion::error::{DataFusionError, Result as DFResult}; +use datafusion::logical_expr::Expr; use datafusion::logical_expr::TableProviderFilterPushDown; use datafusion::physical_plan::ExecutionPlan; -use datafusion_expr::Expr; use crate::dataset_exec::DatasetExec; use crate::pyarrow_filter_expression::PyArrowFilterExpression; diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs index 5fe1f4d1b..a377e2555 100644 --- a/src/dataset_exec.rs +++ b/src/dataset_exec.rs @@ -32,14 +32,14 @@ use datafusion::arrow::pyarrow::PyArrowType; use datafusion::arrow::record_batch::RecordBatch; use datafusion::error::{DataFusionError as InnerDataFusionError, Result as DFResult}; use datafusion::execution::context::TaskContext; +use datafusion::logical_expr::utils::conjunction; +use datafusion::logical_expr::Expr; use datafusion::physical_expr::{EquivalenceProperties, PhysicalSortExpr}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionMode, ExecutionPlan, ExecutionPlanProperties, Partitioning, SendableRecordBatchStream, Statistics, }; -use datafusion_expr::utils::conjunction; -use datafusion_expr::Expr; use crate::errors::DataFusionError; use crate::pyarrow_filter_expression::PyArrowFilterExpression; diff --git a/src/expr.rs b/src/expr.rs index 697682d4c..ab16f2872 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::utils::exprlist_to_fields; -use datafusion_expr::{ExprFuncBuilder, ExprFunctionExt, LogicalPlan}; +use datafusion::logical_expr::utils::exprlist_to_fields; +use datafusion::logical_expr::{ExprFuncBuilder, ExprFunctionExt, LogicalPlan}; use pyo3::{basic::CompareOp, prelude::*}; use std::convert::{From, Into}; use std::sync::Arc; @@ -26,12 +26,12 @@ use arrow::pyarrow::ToPyArrow; use datafusion::arrow::datatypes::{DataType, Field}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::functions::core::expr_ext::FieldAccessor; -use datafusion::scalar::ScalarValue; -use datafusion_expr::{ +use datafusion::logical_expr::{ col, expr::{AggregateFunction, InList, InSubquery, ScalarFunction, Sort, WindowFunction}, lit, Between, BinaryExpr, Case, Cast, Expr, Like, Operator, TryCast, }; +use datafusion::scalar::ScalarValue; use crate::common::data_type::{DataTypeMap, NullTreatment, RexType}; use crate::errors::{py_runtime_err, py_type_err, py_unsupported_variant_err, DataFusionError}; diff --git a/src/expr/aggregate.rs b/src/expr/aggregate.rs index e3d1bb136..389bfb332 100644 --- a/src/expr/aggregate.rs +++ b/src/expr/aggregate.rs @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -use datafusion_common::DataFusionError; -use datafusion_expr::expr::{AggregateFunction, Alias}; -use datafusion_expr::logical_plan::Aggregate; -use datafusion_expr::Expr; +use datafusion::common::DataFusionError; +use datafusion::logical_expr::expr::{AggregateFunction, Alias}; +use datafusion::logical_expr::logical_plan::Aggregate; +use datafusion::logical_expr::Expr; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/aggregate_expr.rs b/src/expr/aggregate_expr.rs index 15097e007..09471097f 100644 --- a/src/expr/aggregate_expr.rs +++ b/src/expr/aggregate_expr.rs @@ -16,7 +16,7 @@ // under the License. use crate::expr::PyExpr; -use datafusion_expr::expr::AggregateFunction; +use datafusion::logical_expr::expr::AggregateFunction; use pyo3::prelude::*; use std::fmt::{Display, Formatter}; diff --git a/src/expr/alias.rs b/src/expr/alias.rs index 3208800ad..e8e03cfad 100644 --- a/src/expr/alias.rs +++ b/src/expr/alias.rs @@ -19,7 +19,7 @@ use crate::expr::PyExpr; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; -use datafusion_expr::expr::Alias; +use datafusion::logical_expr::expr::Alias; #[pyclass(name = "Alias", module = "datafusion.expr", subclass)] #[derive(Clone)] diff --git a/src/expr/analyze.rs b/src/expr/analyze.rs index bbec3a808..084513971 100644 --- a/src/expr/analyze.rs +++ b/src/expr/analyze.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::logical_plan::Analyze; +use datafusion::logical_expr::logical_plan::Analyze; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/between.rs b/src/expr/between.rs index 9b78b9eeb..a2cac1442 100644 --- a/src/expr/between.rs +++ b/src/expr/between.rs @@ -16,7 +16,7 @@ // under the License. use crate::expr::PyExpr; -use datafusion_expr::expr::Between; +use datafusion::logical_expr::expr::Between; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/binary_expr.rs b/src/expr/binary_expr.rs index 5f382b770..740299211 100644 --- a/src/expr/binary_expr.rs +++ b/src/expr/binary_expr.rs @@ -16,7 +16,7 @@ // under the License. use crate::expr::PyExpr; -use datafusion_expr::BinaryExpr; +use datafusion::logical_expr::BinaryExpr; use pyo3::prelude::*; #[pyclass(name = "BinaryExpr", module = "datafusion.expr", subclass)] diff --git a/src/expr/bool_expr.rs b/src/expr/bool_expr.rs index d1502a4eb..e67e25d74 100644 --- a/src/expr/bool_expr.rs +++ b/src/expr/bool_expr.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::Expr; +use datafusion::logical_expr::Expr; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/case.rs b/src/expr/case.rs index 605275376..92e28ba56 100644 --- a/src/expr/case.rs +++ b/src/expr/case.rs @@ -16,7 +16,7 @@ // under the License. use crate::expr::PyExpr; -use datafusion_expr::Case; +use datafusion::logical_expr::Case; use pyo3::prelude::*; #[pyclass(name = "Case", module = "datafusion.expr", subclass)] diff --git a/src/expr/cast.rs b/src/expr/cast.rs index a72199876..b8faea634 100644 --- a/src/expr/cast.rs +++ b/src/expr/cast.rs @@ -16,7 +16,7 @@ // under the License. use crate::{common::data_type::PyDataType, expr::PyExpr}; -use datafusion_expr::{Cast, TryCast}; +use datafusion::logical_expr::{Cast, TryCast}; use pyo3::prelude::*; #[pyclass(name = "Cast", module = "datafusion.expr", subclass)] diff --git a/src/expr/column.rs b/src/expr/column.rs index 68123fb04..365dbc0d2 100644 --- a/src/expr/column.rs +++ b/src/expr/column.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_common::Column; +use datafusion::common::Column; use pyo3::prelude::*; #[pyclass(name = "Column", module = "datafusion.expr", subclass)] diff --git a/src/expr/conditional_expr.rs b/src/expr/conditional_expr.rs index 96ef58f56..a8a885c54 100644 --- a/src/expr/conditional_expr.rs +++ b/src/expr/conditional_expr.rs @@ -16,7 +16,7 @@ // under the License. use crate::expr::PyExpr; -use datafusion_expr::conditional_expressions::CaseBuilder; +use datafusion::logical_expr::conditional_expressions::CaseBuilder; use pyo3::prelude::*; #[pyclass(name = "CaseBuilder", module = "datafusion.expr", subclass)] diff --git a/src/expr/create_memory_table.rs b/src/expr/create_memory_table.rs index 509bf2168..01ebb66b0 100644 --- a/src/expr/create_memory_table.rs +++ b/src/expr/create_memory_table.rs @@ -17,7 +17,7 @@ use std::fmt::{self, Display, Formatter}; -use datafusion_expr::CreateMemoryTable; +use datafusion::logical_expr::CreateMemoryTable; use pyo3::prelude::*; use crate::sql::logical::PyLogicalPlan; diff --git a/src/expr/create_view.rs b/src/expr/create_view.rs index febd723c5..d119f5c21 100644 --- a/src/expr/create_view.rs +++ b/src/expr/create_view.rs @@ -17,7 +17,7 @@ use std::fmt::{self, Display, Formatter}; -use datafusion_expr::{CreateView, DdlStatement, LogicalPlan}; +use datafusion::logical_expr::{CreateView, DdlStatement, LogicalPlan}; use pyo3::prelude::*; use crate::{errors::py_type_err, sql::logical::PyLogicalPlan}; diff --git a/src/expr/cross_join.rs b/src/expr/cross_join.rs index 68793f249..5bc202aac 100644 --- a/src/expr/cross_join.rs +++ b/src/expr/cross_join.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::logical_plan::CrossJoin; +use datafusion::logical_expr::logical_plan::CrossJoin; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/distinct.rs b/src/expr/distinct.rs index 5d3a0b459..061ab4824 100644 --- a/src/expr/distinct.rs +++ b/src/expr/distinct.rs @@ -17,7 +17,7 @@ use std::fmt::{self, Display, Formatter}; -use datafusion_expr::Distinct; +use datafusion::logical_expr::Distinct; use pyo3::prelude::*; use crate::sql::logical::PyLogicalPlan; diff --git a/src/expr/drop_table.rs b/src/expr/drop_table.rs index 2a8836db5..330156abe 100644 --- a/src/expr/drop_table.rs +++ b/src/expr/drop_table.rs @@ -17,7 +17,7 @@ use std::fmt::{self, Display, Formatter}; -use datafusion_expr::logical_plan::DropTable; +use datafusion::logical_expr::logical_plan::DropTable; use pyo3::prelude::*; use crate::sql::logical::PyLogicalPlan; diff --git a/src/expr/empty_relation.rs b/src/expr/empty_relation.rs index 0bc222e59..ce7163466 100644 --- a/src/expr/empty_relation.rs +++ b/src/expr/empty_relation.rs @@ -16,7 +16,7 @@ // under the License. use crate::{common::df_schema::PyDFSchema, sql::logical::PyLogicalPlan}; -use datafusion_expr::EmptyRelation; +use datafusion::logical_expr::EmptyRelation; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/exists.rs b/src/expr/exists.rs index fd2aa8c2f..693357836 100644 --- a/src/expr/exists.rs +++ b/src/expr/exists.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::expr::Exists; +use datafusion::logical_expr::expr::Exists; use pyo3::prelude::*; use super::subquery::PySubquery; diff --git a/src/expr/explain.rs b/src/expr/explain.rs index d5d6a7bbd..8e7fb8843 100644 --- a/src/expr/explain.rs +++ b/src/expr/explain.rs @@ -17,7 +17,7 @@ use std::fmt::{self, Display, Formatter}; -use datafusion_expr::{logical_plan::Explain, LogicalPlan}; +use datafusion::logical_expr::{logical_plan::Explain, LogicalPlan}; use pyo3::prelude::*; use crate::{common::df_schema::PyDFSchema, errors::py_type_err, sql::logical::PyLogicalPlan}; diff --git a/src/expr/extension.rs b/src/expr/extension.rs index 81a435c23..a29802b0b 100644 --- a/src/expr/extension.rs +++ b/src/expr/extension.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::Extension; +use datafusion::logical_expr::Extension; use pyo3::prelude::*; use crate::sql::logical::PyLogicalPlan; diff --git a/src/expr/filter.rs b/src/expr/filter.rs index 2def2f7d6..a6d8aa7ee 100644 --- a/src/expr/filter.rs +++ b/src/expr/filter.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::logical_plan::Filter; +use datafusion::logical_expr::logical_plan::Filter; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/grouping_set.rs b/src/expr/grouping_set.rs index b73932863..63a1c0b50 100644 --- a/src/expr/grouping_set.rs +++ b/src/expr/grouping_set.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::GroupingSet; +use datafusion::logical_expr::GroupingSet; use pyo3::prelude::*; #[pyclass(name = "GroupingSet", module = "datafusion.expr", subclass)] diff --git a/src/expr/in_list.rs b/src/expr/in_list.rs index c1a99a3c8..5dfd8d8eb 100644 --- a/src/expr/in_list.rs +++ b/src/expr/in_list.rs @@ -16,7 +16,7 @@ // under the License. use crate::expr::PyExpr; -use datafusion_expr::expr::InList; +use datafusion::logical_expr::expr::InList; use pyo3::prelude::*; #[pyclass(name = "InList", module = "datafusion.expr", subclass)] diff --git a/src/expr/in_subquery.rs b/src/expr/in_subquery.rs index 7dfafdbf0..306b68a6e 100644 --- a/src/expr/in_subquery.rs +++ b/src/expr/in_subquery.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::expr::InSubquery; +use datafusion::logical_expr::expr::InSubquery; use pyo3::prelude::*; use super::{subquery::PySubquery, PyExpr}; diff --git a/src/expr/indexed_field.rs b/src/expr/indexed_field.rs index e0dad6a4a..a22dc6b27 100644 --- a/src/expr/indexed_field.rs +++ b/src/expr/indexed_field.rs @@ -16,7 +16,7 @@ // under the License. use crate::expr::PyExpr; -use datafusion_expr::expr::{GetFieldAccess, GetIndexedField}; +use datafusion::logical_expr::expr::{GetFieldAccess, GetIndexedField}; use pyo3::prelude::*; use std::fmt::{Display, Formatter}; diff --git a/src/expr/join.rs b/src/expr/join.rs index a53ddd3ba..66e677f8a 100644 --- a/src/expr/join.rs +++ b/src/expr/join.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::logical_plan::{Join, JoinConstraint, JoinType}; +use datafusion::logical_expr::logical_plan::{Join, JoinConstraint, JoinType}; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/like.rs b/src/expr/like.rs index 6ed3c2467..2e1f060bd 100644 --- a/src/expr/like.rs +++ b/src/expr/like.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::expr::Like; +use datafusion::logical_expr::expr::Like; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/limit.rs b/src/expr/limit.rs index d7b3f4ca5..876e154c1 100644 --- a/src/expr/limit.rs +++ b/src/expr/limit.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::logical_plan::Limit; +use datafusion::logical_expr::logical_plan::Limit; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/literal.rs b/src/expr/literal.rs index 0333432f6..43084ba4b 100644 --- a/src/expr/literal.rs +++ b/src/expr/literal.rs @@ -16,7 +16,7 @@ // under the License. use crate::errors::DataFusionError; -use datafusion_common::ScalarValue; +use datafusion::common::ScalarValue; use pyo3::prelude::*; #[pyclass(name = "Literal", module = "datafusion.expr", subclass)] diff --git a/src/expr/placeholder.rs b/src/expr/placeholder.rs index ca75ce37e..4ac2c47e3 100644 --- a/src/expr/placeholder.rs +++ b/src/expr/placeholder.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::expr::Placeholder; +use datafusion::logical_expr::expr::Placeholder; use pyo3::prelude::*; use crate::common::data_type::PyDataType; diff --git a/src/expr/projection.rs b/src/expr/projection.rs index 8c1423df4..36534fdb2 100644 --- a/src/expr/projection.rs +++ b/src/expr/projection.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::logical_plan::Projection; -use datafusion_expr::Expr; +use datafusion::logical_expr::logical_plan::Projection; +use datafusion::logical_expr::Expr; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/repartition.rs b/src/expr/repartition.rs index e3e14f878..4e680e181 100644 --- a/src/expr/repartition.rs +++ b/src/expr/repartition.rs @@ -17,7 +17,7 @@ use std::fmt::{self, Display, Formatter}; -use datafusion_expr::{logical_plan::Repartition, Expr, Partitioning}; +use datafusion::logical_expr::{logical_plan::Repartition, Expr, Partitioning}; use pyo3::prelude::*; use crate::{errors::py_type_err, sql::logical::PyLogicalPlan}; diff --git a/src/expr/scalar_subquery.rs b/src/expr/scalar_subquery.rs index c71bb9905..9d35f28a9 100644 --- a/src/expr/scalar_subquery.rs +++ b/src/expr/scalar_subquery.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::Subquery; +use datafusion::logical_expr::Subquery; use pyo3::prelude::*; use super::subquery::PySubquery; diff --git a/src/expr/signature.rs b/src/expr/signature.rs index 7882cebed..e85763555 100644 --- a/src/expr/signature.rs +++ b/src/expr/signature.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::{TypeSignature, Volatility}; +use datafusion::logical_expr::{TypeSignature, Volatility}; use pyo3::prelude::*; #[allow(dead_code)] diff --git a/src/expr/sort.rs b/src/expr/sort.rs index f9f9e5899..b31ebfe0b 100644 --- a/src/expr/sort.rs +++ b/src/expr/sort.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -use datafusion_common::DataFusionError; -use datafusion_expr::logical_plan::Sort; +use datafusion::common::DataFusionError; +use datafusion::logical_expr::logical_plan::Sort; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/sort_expr.rs b/src/expr/sort_expr.rs index 6a8a0cf0c..4299d1f71 100644 --- a/src/expr/sort_expr.rs +++ b/src/expr/sort_expr.rs @@ -16,7 +16,7 @@ // under the License. use crate::expr::PyExpr; -use datafusion_expr::SortExpr; +use datafusion::logical_expr::SortExpr; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/subquery.rs b/src/expr/subquery.rs index f6f7b7fe5..dac8d0a2b 100644 --- a/src/expr/subquery.rs +++ b/src/expr/subquery.rs @@ -17,7 +17,7 @@ use std::fmt::{self, Display, Formatter}; -use datafusion_expr::Subquery; +use datafusion::logical_expr::Subquery; use pyo3::prelude::*; use crate::sql::logical::PyLogicalPlan; diff --git a/src/expr/subquery_alias.rs b/src/expr/subquery_alias.rs index d3abd2e8c..a83cff96d 100644 --- a/src/expr/subquery_alias.rs +++ b/src/expr/subquery_alias.rs @@ -17,7 +17,7 @@ use std::fmt::{self, Display, Formatter}; -use datafusion_expr::SubqueryAlias; +use datafusion::logical_expr::SubqueryAlias; use pyo3::prelude::*; use crate::{common::df_schema::PyDFSchema, sql::logical::PyLogicalPlan}; diff --git a/src/expr/table_scan.rs b/src/expr/table_scan.rs index 386c43b6a..f61be7fe4 100644 --- a/src/expr/table_scan.rs +++ b/src/expr/table_scan.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -use datafusion_common::TableReference; -use datafusion_expr::logical_plan::TableScan; +use datafusion::common::TableReference; +use datafusion::logical_expr::logical_plan::TableScan; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/union.rs b/src/expr/union.rs index 98e8eaae6..62488d9a1 100644 --- a/src/expr/union.rs +++ b/src/expr/union.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::logical_plan::Union; +use datafusion::logical_expr::logical_plan::Union; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/unnest.rs b/src/expr/unnest.rs index 33fb82feb..adc705035 100644 --- a/src/expr/unnest.rs +++ b/src/expr/unnest.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::logical_plan::Unnest; +use datafusion::logical_expr::logical_plan::Unnest; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/unnest_expr.rs b/src/expr/unnest_expr.rs index a2f8230cc..2234d24b1 100644 --- a/src/expr/unnest_expr.rs +++ b/src/expr/unnest_expr.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::expr::Unnest; +use datafusion::logical_expr::expr::Unnest; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/window.rs b/src/expr/window.rs index 7eb586082..f17a6dd9b 100644 --- a/src/expr/window.rs +++ b/src/expr/window.rs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -use datafusion_common::{DataFusionError, ScalarValue}; -use datafusion_expr::expr::WindowFunction; -use datafusion_expr::{Expr, Window, WindowFrame, WindowFrameBound, WindowFrameUnits}; +use datafusion::common::{DataFusionError, ScalarValue}; +use datafusion::logical_expr::expr::WindowFunction; +use datafusion::logical_expr::{Expr, Window, WindowFrame, WindowFrameBound, WindowFrameUnits}; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/functions.rs b/src/functions.rs index aed4de474..b5b003dfe 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -16,9 +16,9 @@ // under the License. use datafusion::functions_aggregate::all_default_aggregate_functions; -use datafusion_expr::window_function; -use datafusion_expr::ExprFunctionExt; -use datafusion_expr::WindowFrame; +use datafusion::logical_expr::window_function; +use datafusion::logical_expr::ExprFunctionExt; +use datafusion::logical_expr::WindowFrame; use pyo3::{prelude::*, wrap_pyfunction}; use crate::common::data_type::NullTreatment; @@ -28,13 +28,13 @@ use crate::expr::conditional_expr::PyCaseBuilder; use crate::expr::to_sort_expressions; use crate::expr::window::PyWindowFrame; use crate::expr::PyExpr; +use datafusion::common::{Column, ScalarValue, TableReference}; use datafusion::execution::FunctionRegistry; use datafusion::functions; use datafusion::functions_aggregate; -use datafusion_common::{Column, ScalarValue, TableReference}; -use datafusion_expr::expr::Alias; -use datafusion_expr::sqlparser::ast::NullTreatment as DFNullTreatment; -use datafusion_expr::{ +use datafusion::logical_expr::expr::Alias; +use datafusion::logical_expr::sqlparser::ast::NullTreatment as DFNullTreatment; +use datafusion::logical_expr::{ expr::{find_df_window_func, AggregateFunction, Sort, WindowFunction}, lit, Expr, WindowFunctionDefinition, }; @@ -376,7 +376,7 @@ pub fn last_value( #[pyfunction] fn in_list(expr: PyExpr, value: Vec, negated: bool) -> PyExpr { - datafusion_expr::in_list( + datafusion::logical_expr::in_list( expr.expr, value.into_iter().map(|x| x.expr).collect::>(), negated, @@ -386,14 +386,14 @@ fn in_list(expr: PyExpr, value: Vec, negated: bool) -> PyExpr { #[pyfunction] fn make_array(exprs: Vec) -> PyExpr { - datafusion_functions_nested::expr_fn::make_array(exprs.into_iter().map(|x| x.into()).collect()) + datafusion::functions_nested::expr_fn::make_array(exprs.into_iter().map(|x| x.into()).collect()) .into() } #[pyfunction] fn array_concat(exprs: Vec) -> PyExpr { let exprs = exprs.into_iter().map(|x| x.into()).collect(); - datafusion_functions_nested::expr_fn::array_concat(exprs).into() + datafusion::functions_nested::expr_fn::array_concat(exprs).into() } #[pyfunction] @@ -405,12 +405,13 @@ fn array_cat(exprs: Vec) -> PyExpr { fn array_position(array: PyExpr, element: PyExpr, index: Option) -> PyExpr { let index = ScalarValue::Int64(index); let index = Expr::Literal(index); - datafusion_functions_nested::expr_fn::array_position(array.into(), element.into(), index).into() + datafusion::functions_nested::expr_fn::array_position(array.into(), element.into(), index) + .into() } #[pyfunction] fn array_slice(array: PyExpr, begin: PyExpr, end: PyExpr, stride: Option) -> PyExpr { - datafusion_functions_nested::expr_fn::array_slice( + datafusion::functions_nested::expr_fn::array_slice( array.into(), begin.into(), end.into(), @@ -476,7 +477,7 @@ fn regexp_replace( #[pyfunction] fn order_by(expr: PyExpr, asc: bool, nulls_first: bool) -> PyResult { Ok(PyExpr { - expr: datafusion_expr::Expr::Sort(Sort { + expr: datafusion::logical_expr::Expr::Sort(Sort { expr: Box::new(expr.expr), asc, nulls_first, @@ -489,7 +490,7 @@ fn order_by(expr: PyExpr, asc: bool, nulls_first: bool) -> PyResult { fn alias(expr: PyExpr, name: &str) -> PyResult { let relation: Option = None; Ok(PyExpr { - expr: datafusion_expr::Expr::Alias(Alias::new(expr.expr, relation, name)), + expr: datafusion::logical_expr::Expr::Alias(Alias::new(expr.expr, relation, name)), }) } @@ -497,7 +498,7 @@ fn alias(expr: PyExpr, name: &str) -> PyResult { #[pyfunction] fn col(name: &str) -> PyResult { Ok(PyExpr { - expr: datafusion_expr::Expr::Column(Column { + expr: datafusion::logical_expr::Expr::Column(Column { relation: None, name: name.to_string(), }), @@ -527,7 +528,7 @@ fn count(expr: PyExpr, distinct: bool) -> PyResult { #[pyfunction] fn case(expr: PyExpr) -> PyResult { Ok(PyCaseBuilder { - case_builder: datafusion_expr::case(expr.expr), + case_builder: datafusion::logical_expr::case(expr.expr), }) } @@ -535,7 +536,7 @@ fn case(expr: PyExpr) -> PyResult { #[pyfunction] fn when(when: PyExpr, then: PyExpr) -> PyResult { Ok(PyCaseBuilder { - case_builder: datafusion_expr::when(when.expr, then.expr), + case_builder: datafusion::logical_expr::when(when.expr, then.expr), }) } @@ -622,7 +623,7 @@ fn window( .unwrap_or(WindowFrame::new(order_by.as_ref().map(|v| !v.is_empty()))); Ok(PyExpr { - expr: datafusion_expr::Expr::WindowFunction(WindowFunction { + expr: datafusion::logical_expr::Expr::WindowFunction(WindowFunction { fun, args: args.into_iter().map(|x| x.expr).collect::>(), partition_by: partition_by @@ -654,7 +655,7 @@ macro_rules! aggregate_function { #[pyfunction] #[pyo3(signature = (*args, distinct=false))] fn $NAME(args: Vec, distinct: bool) -> PyExpr { - let expr = datafusion_expr::Expr::AggregateFunction(AggregateFunction { + let expr = datafusion::logical_expr::Expr::AggregateFunction(AggregateFunction { func: $FUNC(), args: args.into_iter().map(|e| e.into()).collect(), distinct, @@ -724,7 +725,7 @@ macro_rules! array_fn { #[doc = $DOC] #[pyfunction] fn $FUNC($($arg: PyExpr),*) -> PyExpr { - datafusion_functions_nested::expr_fn::$FUNC($($arg.into()),*).into() + datafusion::functions_nested::expr_fn::$FUNC($($arg.into()),*).into() } }; } diff --git a/src/lib.rs b/src/lib.rs index 357eaacd9..784c3d519 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,10 +21,16 @@ use pyo3::prelude::*; // Re-export Apache Arrow DataFusion dependencies pub use datafusion; -pub use datafusion_common; -pub use datafusion_expr; -pub use datafusion_optimizer; -pub use datafusion_sql; +pub mod datafusion_expr { + pub use datafusion::logical_expr::*; +} +pub use datafusion::optimizer; +pub mod datafusion_sql { + pub use datafusion::sql::*; +} +pub mod datafusion_common { + pub use datafusion::common::*; +} #[cfg(feature = "substrait")] pub use datafusion_substrait; @@ -32,6 +38,7 @@ pub use datafusion_substrait; #[allow(clippy::borrow_deref_ref)] pub mod catalog; pub mod common; + #[allow(clippy::borrow_deref_ref)] mod config; #[allow(clippy::borrow_deref_ref)] diff --git a/src/pyarrow_filter_expression.rs b/src/pyarrow_filter_expression.rs index ff447e1ab..6e2a45e1a 100644 --- a/src/pyarrow_filter_expression.rs +++ b/src/pyarrow_filter_expression.rs @@ -22,8 +22,8 @@ use std::convert::TryFrom; use std::result::Result; use arrow::pyarrow::ToPyArrow; -use datafusion_common::{Column, ScalarValue}; -use datafusion_expr::{expr::InList, Between, BinaryExpr, Expr, Operator}; +use datafusion::common::{Column, ScalarValue}; +use datafusion::logical_expr::{expr::InList, Between, BinaryExpr, Expr, Operator}; use crate::errors::DataFusionError; diff --git a/src/sql/logical.rs b/src/sql/logical.rs index c4471f503..89655ab70 100644 --- a/src/sql/logical.rs +++ b/src/sql/logical.rs @@ -35,7 +35,7 @@ use crate::expr::subquery_alias::PySubqueryAlias; use crate::expr::table_scan::PyTableScan; use crate::expr::unnest::PyUnnest; use crate::expr::window::PyWindow; -use datafusion_expr::LogicalPlan; +use datafusion::logical_expr::LogicalPlan; use pyo3::prelude::*; use crate::expr::logical_node::LogicalNode; diff --git a/src/udaf.rs b/src/udaf.rs index 2041e5a74..a6aa59ac3 100644 --- a/src/udaf.rs +++ b/src/udaf.rs @@ -24,7 +24,9 @@ use datafusion::arrow::datatypes::DataType; use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow}; use datafusion::common::ScalarValue; use datafusion::error::{DataFusionError, Result}; -use datafusion_expr::{create_udaf, Accumulator, AccumulatorFactoryFunction, AggregateUDF}; +use datafusion::logical_expr::{ + create_udaf, Accumulator, AccumulatorFactoryFunction, AggregateUDF, +}; use crate::expr::PyExpr; use crate::utils::parse_volatility; diff --git a/src/udf.rs b/src/udf.rs index 8f5ca30b1..8bd9021d4 100644 --- a/src/udf.rs +++ b/src/udf.rs @@ -23,9 +23,9 @@ use datafusion::arrow::array::{make_array, Array, ArrayData, ArrayRef}; use datafusion::arrow::datatypes::DataType; use datafusion::arrow::pyarrow::{FromPyArrow, PyArrowType, ToPyArrow}; use datafusion::error::DataFusionError; -use datafusion_expr::create_udf; -use datafusion_expr::function::ScalarFunctionImplementation; -use datafusion_expr::ScalarUDF; +use datafusion::logical_expr::create_udf; +use datafusion::logical_expr::function::ScalarFunctionImplementation; +use datafusion::logical_expr::ScalarUDF; use crate::expr::PyExpr; use crate::utils::parse_volatility; diff --git a/src/utils.rs b/src/utils.rs index 4334f86cd..0d72eaf75 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -17,7 +17,7 @@ use crate::errors::DataFusionError; use crate::TokioRuntime; -use datafusion_expr::Volatility; +use datafusion::logical_expr::Volatility; use pyo3::prelude::*; use std::future::Future; use tokio::runtime::Runtime; From a4728b9c671993fc1cfa493d75682de522315d20 Mon Sep 17 00:00:00 2001 From: Matt Green Date: Fri, 6 Sep 2024 09:14:22 -0700 Subject: [PATCH 2/2] update dependency export syntax --- src/lib.rs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 784c3d519..e4cc24078 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,16 +21,10 @@ use pyo3::prelude::*; // Re-export Apache Arrow DataFusion dependencies pub use datafusion; -pub mod datafusion_expr { - pub use datafusion::logical_expr::*; -} +pub use datafusion::common as datafusion_common; +pub use datafusion::logical_expr as datafusion_expr; pub use datafusion::optimizer; -pub mod datafusion_sql { - pub use datafusion::sql::*; -} -pub mod datafusion_common { - pub use datafusion::common::*; -} +pub use datafusion::sql as datafusion_sql; #[cfg(feature = "substrait")] pub use datafusion_substrait;