Export datafusion_physical_optimizer, only use datafusion crate in the examples (#14305)

* Only use `datafusion` crate in the examples

* taplo format

Author: alamb

datafusion-examples/Cargo.toml

@@ -54,21 +54,15 @@ path = "examples/external_dependency/query-aws-s3.rs"
 
 [dev-dependencies]
 arrow = { workspace = true }
+# arrow_schema is required for record_batch! macro :sad:
 arrow-flight = { workspace = true }
 arrow-schema = { workspace = true }
 async-trait = { workspace = true }
 bytes = { workspace = true }
 dashmap = { workspace = true }
+# note only use main datafusion crate for examples
 datafusion = { workspace = true, default-features = true, features = ["avro"] }
-datafusion-catalog = { workspace = true }
-datafusion-common = { workspace = true, default-features = true }
-datafusion-expr = { workspace = true }
-datafusion-functions-window-common = { workspace = true }
-datafusion-optimizer = { workspace = true, default-features = true }
-datafusion-physical-expr = { workspace = true, default-features = true }
-datafusion-physical-optimizer = { workspace = true, default-features = true }
 datafusion-proto = { workspace = true }
-datafusion-sql = { workspace = true }
 env_logger = { workspace = true }
 futures = { workspace = true }
 log = { workspace = true }

datafusion-examples/examples/advanced_parquet_index.rs

@@ -24,8 +24,13 @@ use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 
 use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};
-use arrow_schema::SchemaRef;
+use arrow::datatypes::SchemaRef;
+use async_trait::async_trait;
+use bytes::Bytes;
 use datafusion::catalog::Session;
+use datafusion::common::{
+    internal_datafusion_err, DFSchema, DataFusionError, Result, ScalarValue,
+};
 use datafusion::datasource::listing::PartitionedFile;
 use datafusion::datasource::physical_plan::parquet::{
     ParquetAccessPlan, ParquetExecBuilder,
@@ -35,6 +40,8 @@ use datafusion::datasource::physical_plan::{
 };
 use datafusion::datasource::TableProvider;
 use datafusion::execution::object_store::ObjectStoreUrl;
+use datafusion::logical_expr::utils::conjunction;
+use datafusion::logical_expr::{TableProviderFilterPushDown, TableType};
 use datafusion::parquet::arrow::arrow_reader::{
     ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelection, RowSelector,
 };
@@ -43,20 +50,12 @@ use datafusion::parquet::arrow::ArrowWriter;
 use datafusion::parquet::file::metadata::ParquetMetaData;
 use datafusion::parquet::file::properties::{EnabledStatistics, WriterProperties};
 use datafusion::parquet::schema::types::ColumnPath;
+use datafusion::physical_expr::utils::{Guarantee, LiteralGuarantee};
 use datafusion::physical_expr::PhysicalExpr;
+use datafusion::physical_optimizer::pruning::PruningPredicate;
 use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion::prelude::*;
-use datafusion_common::{
-    internal_datafusion_err, DFSchema, DataFusionError, Result, ScalarValue,
-};
-use datafusion_expr::utils::conjunction;
-use datafusion_expr::{TableProviderFilterPushDown, TableType};
-use datafusion_physical_expr::utils::{Guarantee, LiteralGuarantee};
-use datafusion_physical_optimizer::pruning::PruningPredicate;
-
-use async_trait::async_trait;
-use bytes::Bytes;
 use futures::future::BoxFuture;
 use futures::FutureExt;
 use object_store::ObjectStore;
@@ -284,7 +283,7 @@ impl IndexTableProvider {
             .transpose()?
             // if there are no filters, use a literal true to have a predicate
             // that always evaluates to true we can pass to the index
-            .unwrap_or_else(|| datafusion_physical_expr::expressions::lit(true));
+            .unwrap_or_else(|| datafusion::physical_expr::expressions::lit(true));
 
         Ok(predicate)
     }

datafusion-examples/examples/advanced_udaf.rs

@@ -15,27 +15,25 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use arrow_schema::{Field, Schema};
+use arrow::datatypes::{Field, Schema};
+use datafusion::physical_expr::NullState;
 use datafusion::{arrow::datatypes::DataType, logical_expr::Volatility};
-use datafusion_physical_expr::NullState;
 use std::{any::Any, sync::Arc};
 
-use arrow::{
-    array::{
-        ArrayRef, AsArray, Float32Array, PrimitiveArray, PrimitiveBuilder, UInt32Array,
-    },
-    datatypes::{ArrowNativeTypeOp, ArrowPrimitiveType, Float64Type, UInt32Type},
-    record_batch::RecordBatch,
+use arrow::array::{
+    ArrayRef, AsArray, Float32Array, PrimitiveArray, PrimitiveBuilder, UInt32Array,
 };
+use arrow::datatypes::{ArrowNativeTypeOp, ArrowPrimitiveType, Float64Type, UInt32Type};
+use arrow::record_batch::RecordBatch;
+use datafusion::common::{cast::as_float64_array, ScalarValue};
 use datafusion::error::Result;
-use datafusion::prelude::*;
-use datafusion_common::{cast::as_float64_array, ScalarValue};
-use datafusion_expr::{
+use datafusion::logical_expr::{
     expr::AggregateFunction,
     function::{AccumulatorArgs, AggregateFunctionSimplification, StateFieldsArgs},
     simplify::SimplifyInfo,
-    Accumulator, AggregateUDF, AggregateUDFImpl, GroupsAccumulator, Signature,
+    Accumulator, AggregateUDF, AggregateUDFImpl, EmitTo, GroupsAccumulator, Signature,
 };
+use datafusion::prelude::*;
 
 /// This example shows how to use the full AggregateUDFImpl API to implement a user
 /// defined aggregate function. As in the `simple_udaf.rs` example, this struct implements
@@ -308,7 +306,7 @@ impl GroupsAccumulator for GeometricMeanGroupsAccumulator {
     }
 
     /// Generate output, as specified by `emit_to` and update the intermediate state
-    fn evaluate(&mut self, emit_to: datafusion_expr::EmitTo) -> Result<ArrayRef> {
+    fn evaluate(&mut self, emit_to: EmitTo) -> Result<ArrayRef> {
         let counts = emit_to.take_needed(&mut self.counts);
         let prods = emit_to.take_needed(&mut self.prods);
         let nulls = self.null_state.build(emit_to);
@@ -344,7 +342,7 @@ impl GroupsAccumulator for GeometricMeanGroupsAccumulator {
     }
 
     // return arrays for counts and prods
-    fn state(&mut self, emit_to: datafusion_expr::EmitTo) -> Result<Vec<ArrayRef>> {
+    fn state(&mut self, emit_to: EmitTo) -> Result<Vec<ArrayRef>> {
         let nulls = self.null_state.build(emit_to);
         let nulls = Some(nulls);
 

datafusion-examples/examples/advanced_udf.rs

@@ -24,14 +24,14 @@ use arrow::array::{
 use arrow::compute;
 use arrow::datatypes::{DataType, Float64Type};
 use arrow::record_batch::RecordBatch;
+use datafusion::common::{exec_err, internal_err, ScalarValue};
 use datafusion::error::Result;
+use datafusion::logical_expr::sort_properties::{ExprProperties, SortProperties};
 use datafusion::logical_expr::Volatility;
-use datafusion::prelude::*;
-use datafusion_common::{exec_err, internal_err, ScalarValue};
-use datafusion_expr::sort_properties::{ExprProperties, SortProperties};
-use datafusion_expr::{
+use datafusion::logical_expr::{
     ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,
 };
+use datafusion::prelude::*;
 
 /// This example shows how to use the full ScalarUDFImpl API to implement a user
 /// defined function. As in the `simple_udf.rs` example, this struct implements

datafusion-examples/examples/advanced_udwf.rs

@@ -18,22 +18,24 @@
 use datafusion::{arrow::datatypes::DataType, logical_expr::Volatility};
 use std::any::Any;
 
+use arrow::datatypes::Field;
 use arrow::{
     array::{ArrayRef, AsArray, Float64Array},
     datatypes::Float64Type,
 };
-use arrow_schema::Field;
+use datafusion::common::ScalarValue;
 use datafusion::error::Result;
 use datafusion::functions_aggregate::average::avg_udaf;
-use datafusion::prelude::*;
-use datafusion_common::ScalarValue;
-use datafusion_expr::expr::WindowFunction;
-use datafusion_expr::function::{WindowFunctionSimplification, WindowUDFFieldArgs};
-use datafusion_expr::simplify::SimplifyInfo;
-use datafusion_expr::{
-    Expr, PartitionEvaluator, Signature, WindowFrame, WindowUDF, WindowUDFImpl,
+use datafusion::logical_expr::expr::WindowFunction;
+use datafusion::logical_expr::function::{
+    PartitionEvaluatorArgs, WindowFunctionSimplification, WindowUDFFieldArgs,
+};
+use datafusion::logical_expr::simplify::SimplifyInfo;
+use datafusion::logical_expr::{
+    Expr, PartitionEvaluator, Signature, WindowFrame, WindowFunctionDefinition,
+    WindowUDF, WindowUDFImpl,
 };
-use datafusion_functions_window_common::partition::PartitionEvaluatorArgs;
+use datafusion::prelude::*;
 
 /// This example shows how to use the full WindowUDFImpl API to implement a user
 /// defined window function. As in the `simple_udwf.rs` example, this struct implements
@@ -189,7 +191,7 @@ impl WindowUDFImpl for SimplifySmoothItUdf {
     fn simplify(&self) -> Option<WindowFunctionSimplification> {
         let simplify = |window_function: WindowFunction, _: &dyn SimplifyInfo| {
             Ok(Expr::WindowFunction(WindowFunction {
-                fun: datafusion_expr::WindowFunctionDefinition::AggregateUDF(avg_udaf()),
+                fun: WindowFunctionDefinition::AggregateUDF(avg_udaf()),
                 args: window_function.args,
                 partition_by: window_function.partition_by,
                 order_by: window_function.order_by,

datafusion-examples/examples/analyzer_rule.rs

@@ -16,12 +16,12 @@
 // under the License.
 
 use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};
+use datafusion::common::config::ConfigOptions;
+use datafusion::common::tree_node::{Transformed, TreeNode};
+use datafusion::common::Result;
+use datafusion::logical_expr::{col, lit, Expr, LogicalPlan, LogicalPlanBuilder};
+use datafusion::optimizer::analyzer::AnalyzerRule;
 use datafusion::prelude::SessionContext;
-use datafusion_common::config::ConfigOptions;
-use datafusion_common::tree_node::{Transformed, TreeNode};
-use datafusion_common::Result;
-use datafusion_expr::{col, lit, Expr, LogicalPlan, LogicalPlanBuilder};
-use datafusion_optimizer::analyzer::AnalyzerRule;
 use std::sync::{Arc, Mutex};
 
 /// This example demonstrates how to add your own [`AnalyzerRule`] to

datafusion-examples/examples/composed_extension_codec.rs

@@ -36,11 +36,11 @@ use std::ops::Deref;
 use std::sync::Arc;
 
 use datafusion::common::Result;
+use datafusion::common::{internal_err, DataFusionError};
+use datafusion::logical_expr::registry::FunctionRegistry;
+use datafusion::logical_expr::{AggregateUDF, ScalarUDF};
 use datafusion::physical_plan::{DisplayAs, ExecutionPlan};
 use datafusion::prelude::SessionContext;
-use datafusion_common::{internal_err, DataFusionError};
-use datafusion_expr::registry::FunctionRegistry;
-use datafusion_expr::{AggregateUDF, ScalarUDF};
 use datafusion_proto::physical_plan::{AsExecutionPlan, PhysicalExtensionCodec};
 use datafusion_proto::protobuf;
 

datafusion-examples/examples/csv_json_opener.rs

@@ -17,7 +17,7 @@
 
 use std::sync::Arc;
 
-use arrow_schema::{DataType, Field, Schema};
+use arrow::datatypes::{DataType, Field, Schema};
 use datafusion::{
     assert_batches_eq,
     datasource::{

datafusion-examples/examples/custom_file_format.rs

@@ -19,11 +19,12 @@ use std::{any::Any, sync::Arc};
 
 use arrow::{
     array::{AsArray, RecordBatch, StringArray, UInt8Array},
-    datatypes::UInt64Type,
+    datatypes::{DataType, Field, Schema, SchemaRef, UInt64Type},
 };
-use arrow_schema::{DataType, Field, Schema, SchemaRef};
+use datafusion::common::{GetExt, Statistics};
 use datafusion::execution::session_state::SessionStateBuilder;
 use datafusion::physical_expr::LexRequirement;
+use datafusion::physical_expr::PhysicalExpr;
 use datafusion::{
     datasource::{
         file_format::{
@@ -38,8 +39,6 @@ use datafusion::{
     physical_plan::ExecutionPlan,
     prelude::SessionContext,
 };
-use datafusion_common::{GetExt, Statistics};
-use datafusion_physical_expr::PhysicalExpr;
 use object_store::{ObjectMeta, ObjectStore};
 use tempfile::tempdir;
 

datafusion-examples/examples/dataframe.rs

@@ -17,15 +17,15 @@
 
 use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray};
 use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::common::config::CsvOptions;
+use datafusion::common::parsers::CompressionTypeVariant;
+use datafusion::common::DataFusionError;
+use datafusion::common::ScalarValue;
 use datafusion::dataframe::DataFrameWriteOptions;
 use datafusion::error::Result;
 use datafusion::functions_aggregate::average::avg;
 use datafusion::functions_aggregate::min_max::max;
 use datafusion::prelude::*;
-use datafusion_common::config::CsvOptions;
-use datafusion_common::parsers::CompressionTypeVariant;
-use datafusion_common::DataFusionError;
-use datafusion_common::ScalarValue;
 use std::fs::File;
 use std::io::Write;
 use std::sync::Arc;