chore: Remove all subdependencies (#1514)

* Remove all subdependencies

* rebase and resolve conflicts

* fmt

---------

Co-authored-by: Emily Matheys <[email protected]>

Author: EmilyMatt

docs/source/contributor-guide/debugging.md

@@ -142,8 +142,8 @@ org.apache.comet.CometNativeException: Internal error: MIN/MAX is not expected t
 
 backtrace:
   0: std::backtrace::Backtrace::create
-  1: datafusion_physical_expr::aggregate::min_max::min
-  2: <datafusion_physical_expr::aggregate::min_max::MinAccumulator as datafusion_expr::accumulator::Accumulator>::update_batch
+  1: datafusion::physical_expr::aggregate::min_max::min
+  2: <datafusion::physical_expr::aggregate::min_max::MinAccumulator as datafusion_expr::accumulator::Accumulator>::update_batch
   3: <futures_util::stream::stream::fuse::Fuse<S> as futures_core::stream::Stream>::poll_next
   4: comet::execution::jni_api::Java_org_apache_comet_Native_executePlan::{{closure}}
   5: _Java_org_apache_comet_Native_executePlan

native/Cargo.lock

@@ -35,22 +35,10 @@ rust-version = "1.85"
 
 [workspace.dependencies]
 arrow = { version = "54.2.0", features = ["prettyprint", "ffi", "chrono-tz"] }
-arrow-array = { version = "54.2.0" }
-arrow-buffer = { version = "54.2.0" }
-arrow-data = { version = "54.2.0" }
-arrow-schema = { version = "54.2.0" }
 async-trait = { version = "0.1" }
 bytes = { version = "1.10.0" }
 parquet = { version = "54.2.0", default-features = false, features = ["experimental"] }
-datafusion = { version = "46.0.0", default-features = false, features = ["unicode_expressions", "crypto_expressions"] }
-datafusion-common = { version = "46.0.0", default-features = false }
-datafusion-datasource = { version = "46.0.0", default-features = false }
-datafusion-functions = { version = "46.0.0", default-features = false, features = ["crypto_expressions"] }
-datafusion-functions-nested = { version = "46.0.0", default-features = false }
-datafusion-expr = { version = "46.0.0", default-features = false }
-datafusion-expr-common = { version = "46.0.0", default-features = false }
-datafusion-execution = { version = "46.0.0", default-features = false }
-datafusion-physical-expr = { version = "46.0.0", default-features = false }
+datafusion = { version = "46.0.0", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] }
 datafusion-comet-spark-expr = { path = "spark-expr", version = "0.7.0" }
 datafusion-comet-proto = { path = "proto", version = "0.7.0" }
 chrono = { version = "0.4", default-features = false, features = ["clock"] }

native/Cargo.toml

@@ -36,10 +36,6 @@ publish = false
 
 [dependencies]
 arrow = { workspace = true }
-arrow-array = { workspace = true }
-arrow-buffer = { workspace = true }
-arrow-data = { workspace = true }
-arrow-schema = { workspace = true }
 parquet = { workspace = true, default-features = false, features = ["experimental"] }
 futures = { workspace = true }
 mimalloc = { version = "*", default-features = false, optional = true }
@@ -62,12 +58,7 @@ bytes = { workspace = true }
 tempfile = "3.8.0"
 itertools = "0.14.0"
 paste = "1.0.14"
-datafusion-common = { workspace = true, features= ["object_store"] }
 datafusion = {  workspace = true }
-datafusion-functions-nested = { workspace = true }
-datafusion-expr = {  workspace = true }
-datafusion-execution = {  workspace = true }
-datafusion-physical-expr = {  workspace = true }
 once_cell = "1.18.0"
 regex = { workspace = true }
 crc32fast = "1.3.2"

native/core/Cargo.toml

@@ -15,22 +15,22 @@
 // specific language governing permissions and limitations
 // under the License.use arrow::array::{ArrayRef, BooleanBuilder, Int32Builder, RecordBatch, StringBuilder};
 
+use arrow::array::builder::Int64Builder;
+use arrow::array::{ArrayRef, RecordBatch};
+use arrow::datatypes::SchemaRef;
 use arrow::datatypes::{DataType, Field, Schema};
-use arrow_array::builder::Int64Builder;
-use arrow_array::{ArrayRef, RecordBatch};
-use arrow_schema::SchemaRef;
 use comet::execution::expressions::bloom_filter_agg::BloomFilterAgg;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use datafusion::common::ScalarValue;
 use datafusion::datasource::memory::MemorySourceConfig;
 use datafusion::datasource::source::DataSourceExec;
+use datafusion::execution::TaskContext;
+use datafusion::logical_expr::AggregateUDF;
+use datafusion::physical_expr::aggregate::AggregateExprBuilder;
+use datafusion::physical_expr::expressions::{Column, Literal};
 use datafusion::physical_expr::PhysicalExpr;
 use datafusion::physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy};
 use datafusion::physical_plan::ExecutionPlan;
-use datafusion_common::ScalarValue;
-use datafusion_execution::TaskContext;
-use datafusion_expr::AggregateUDF;
-use datafusion_physical_expr::aggregate::AggregateExprBuilder;
-use datafusion_physical_expr::expressions::{Column, Literal};
 use futures::StreamExt;
 use std::sync::Arc;
 use std::time::Duration;

native/core/benches/bloom_filter_agg.rs

@@ -15,11 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use arrow::error::ArrowError;
 use arrow::{
     array::{DictionaryArray, Int64Array, PrimitiveArray},
     datatypes::{ArrowPrimitiveType, Int32Type},
 };
-use arrow_schema::ArrowError;
 use rand::{
     distributions::{Distribution, Standard},
     rngs::StdRng,

native/core/benches/common.rs

@@ -15,10 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.use arrow::array::{ArrayRef, BooleanBuilder, Int32Builder, RecordBatch, StringBuilder};
 
+use arrow::array::builder::{BooleanBuilder, Int32Builder, StringBuilder};
+use arrow::array::{ArrayRef, RecordBatch};
 use arrow::compute::filter_record_batch;
 use arrow::datatypes::{DataType, Field, Schema};
-use arrow_array::builder::{BooleanBuilder, Int32Builder, StringBuilder};
-use arrow_array::{ArrayRef, RecordBatch};
 use comet::execution::operators::comet_filter_record_batch;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use std::sync::Arc;

native/core/benches/filter.rs

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use arrow_buffer::ToByteSlice;
+use arrow::datatypes::ToByteSlice;
 use comet::parquet::read::values::{copy_i32_to_i16, copy_i32_to_u16, copy_i64_to_i64};
 use criterion::{criterion_group, criterion_main, Criterion};
 

native/core/benches/parquet_decode.rs

@@ -15,19 +15,19 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use arrow_array::builder::{Date32Builder, Decimal128Builder, Int32Builder};
-use arrow_array::{builder::StringBuilder, RecordBatch};
-use arrow_schema::{DataType, Field, Schema};
+use arrow::array::builder::{Date32Builder, Decimal128Builder, Int32Builder};
+use arrow::array::{builder::StringBuilder, RecordBatch};
+use arrow::datatypes::{DataType, Field, Schema};
 use comet::execution::shuffle::{CompressionCodec, ShuffleBlockWriter, ShuffleWriterExec};
 use criterion::{criterion_group, criterion_main, Criterion};
 use datafusion::datasource::memory::MemorySourceConfig;
 use datafusion::datasource::source::DataSourceExec;
+use datafusion::physical_expr::{expressions::Column, Partitioning};
 use datafusion::physical_plan::metrics::Time;
 use datafusion::{
     physical_plan::{common::collect, ExecutionPlan},
     prelude::SessionContext,
 };
-use datafusion_physical_expr::{expressions::Column, Partitioning};
 use std::io::Cursor;
 use std::sync::Arc;
 use tokio::runtime::Runtime;

native/core/benches/shuffle_writer.rs

@@ -18,7 +18,7 @@
 //! Common Parquet errors and macros.
 
 use arrow::error::ArrowError;
-use datafusion_common::DataFusionError;
+use datafusion::common::DataFusionError;
 use jni::errors::{Exception, ToException};
 use regex::Regex;