Skip to content

Commit 9285b84

Browse files
AdamGSalamb
andauthored
Move FileSourceConfig and FileStream to the new datafusion-datasource (#14838)
* Initial work * Fix some CI issues * remove cyclical dev-dependency on core * Trying to keep some key things accessible in the same way * ignore rustdoc test for example * Restore doc test with mock parquet source --------- Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org>
1 parent d0ab003 commit 9285b84

File tree

36 files changed

+2727
-2597
lines changed

36 files changed

+2727
-2597
lines changed

datafusion-examples/examples/csv_json_opener.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ use arrow::datatypes::{DataType, Field, Schema};
2121
use datafusion::datasource::physical_plan::JsonSource;
2222
use datafusion::{
2323
assert_batches_eq,
24+
datasource::physical_plan::FileSource,
2425
datasource::{
25-
data_source::FileSource,
2626
file_format::file_compression_type::FileCompressionType,
2727
listing::PartitionedFile,
2828
object_store::ObjectStoreUrl,

datafusion-examples/examples/custom_file_format.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,15 @@ use arrow::{
2121
array::{AsArray, RecordBatch, StringArray, UInt8Array},
2222
datatypes::{DataType, Field, Schema, SchemaRef, UInt64Type},
2323
};
24-
use datafusion::datasource::data_source::FileSource;
25-
use datafusion::execution::session_state::SessionStateBuilder;
2624
use datafusion::physical_expr::LexRequirement;
2725
use datafusion::physical_expr::PhysicalExpr;
2826
use datafusion::{
2927
catalog::Session,
3028
common::{GetExt, Statistics},
3129
};
30+
use datafusion::{
31+
datasource::physical_plan::FileSource, execution::session_state::SessionStateBuilder,
32+
};
3233
use datafusion::{
3334
datasource::{
3435
file_format::{

datafusion/common/src/test_util.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ use std::{error::Error, path::PathBuf};
2828
///
2929
/// Expects to be called about like this:
3030
///
31-
/// `assert_batch_eq!(expected_lines: &[&str], batches: &[RecordBatch])`
31+
/// `assert_batches_eq!(expected_lines: &[&str], batches: &[RecordBatch])`
3232
///
3333
/// # Example
3434
/// ```

datafusion/core/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ nested_expressions = ["datafusion-functions-nested"]
4040
# This feature is deprecated. Use the `nested_expressions` feature instead.
4141
array_expressions = ["nested_expressions"]
4242
# Used to enable the avro format
43-
avro = ["apache-avro", "num-traits", "datafusion-common/avro"]
43+
avro = ["apache-avro", "num-traits", "datafusion-common/avro", "datafusion-datasource/avro"]
4444
backtrace = ["datafusion-common/backtrace"]
4545
compression = ["xz2", "bzip2", "flate2", "zstd", "datafusion-datasource/compression"]
4646
crypto_expressions = ["datafusion-functions/crypto_expressions"]

datafusion/core/src/datasource/file_format/arrow.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,7 @@ use super::write::{create_writer, SharedBuffer};
3131
use super::FileFormatFactory;
3232
use crate::datasource::file_format::write::get_writer_schema;
3333
use crate::datasource::file_format::FileFormat;
34-
use crate::datasource::physical_plan::{
35-
ArrowSource, FileGroupDisplay, FileScanConfig, FileSink, FileSinkConfig,
36-
};
34+
use crate::datasource::physical_plan::{ArrowSource, FileSink, FileSinkConfig};
3735
use crate::error::Result;
3836
use crate::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan};
3937

@@ -49,13 +47,15 @@ use datafusion_common::{
4947
not_impl_err, DataFusionError, GetExt, Statistics, DEFAULT_ARROW_EXTENSION,
5048
};
5149
use datafusion_common_runtime::SpawnedTask;
50+
use datafusion_datasource::display::FileGroupDisplay;
51+
use datafusion_datasource::file::FileSource;
52+
use datafusion_datasource::file_scan_config::FileScanConfig;
5253
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
5354
use datafusion_expr::dml::InsertOp;
5455
use datafusion_physical_expr::PhysicalExpr;
5556
use datafusion_physical_expr_common::sort_expr::LexRequirement;
5657
use datafusion_physical_plan::insert::{DataSink, DataSinkExec};
5758

58-
use crate::datasource::data_source::FileSource;
5959
use async_trait::async_trait;
6060
use bytes::Bytes;
6161
use futures::stream::BoxStream;

datafusion/core/src/datasource/file_format/avro.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,11 @@ use super::file_compression_type::FileCompressionType;
2626
use super::FileFormat;
2727
use super::FileFormatFactory;
2828
use crate::datasource::avro_to_arrow::read_avro_schema_from_reader;
29-
use crate::datasource::physical_plan::{AvroSource, FileScanConfig};
29+
use crate::datasource::physical_plan::AvroSource;
3030
use crate::error::Result;
3131
use crate::physical_plan::ExecutionPlan;
3232
use crate::physical_plan::Statistics;
3333

34-
use crate::datasource::data_source::FileSource;
3534
use arrow::datatypes::Schema;
3635
use arrow::datatypes::SchemaRef;
3736
use async_trait::async_trait;
@@ -40,6 +39,8 @@ use datafusion_common::internal_err;
4039
use datafusion_common::parsers::CompressionTypeVariant;
4140
use datafusion_common::GetExt;
4241
use datafusion_common::DEFAULT_AVRO_EXTENSION;
42+
use datafusion_datasource::file::FileSource;
43+
use datafusion_datasource::file_scan_config::FileScanConfig;
4344
use datafusion_physical_expr::PhysicalExpr;
4445
use object_store::{GetResultPayload, ObjectMeta, ObjectStore};
4546

datafusion/core/src/datasource/file_format/csv.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,7 @@ use super::{
3030
use crate::datasource::file_format::file_compression_type::FileCompressionType;
3131
use crate::datasource::file_format::write::demux::DemuxedStreamReceiver;
3232
use crate::datasource::file_format::write::BatchSerializer;
33-
use crate::datasource::physical_plan::{
34-
CsvSource, FileGroupDisplay, FileScanConfig, FileSink, FileSinkConfig,
35-
};
33+
use crate::datasource::physical_plan::{CsvSource, FileSink, FileSinkConfig};
3634
use crate::error::Result;
3735
use crate::execution::context::SessionState;
3836
use crate::physical_plan::insert::{DataSink, DataSinkExec};
@@ -51,12 +49,14 @@ use datafusion_common::{
5149
exec_err, not_impl_err, DataFusionError, GetExt, DEFAULT_CSV_EXTENSION,
5250
};
5351
use datafusion_common_runtime::SpawnedTask;
52+
use datafusion_datasource::display::FileGroupDisplay;
53+
use datafusion_datasource::file::FileSource;
54+
use datafusion_datasource::file_scan_config::FileScanConfig;
5455
use datafusion_execution::TaskContext;
5556
use datafusion_expr::dml::InsertOp;
5657
use datafusion_physical_expr::PhysicalExpr;
5758
use datafusion_physical_expr_common::sort_expr::LexRequirement;
5859

59-
use crate::datasource::data_source::FileSource;
6060
use async_trait::async_trait;
6161
use bytes::{Buf, Bytes};
6262
use futures::stream::BoxStream;

datafusion/core/src/datasource/file_format/json.rs

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,7 @@ use super::{
3232
use crate::datasource::file_format::file_compression_type::FileCompressionType;
3333
use crate::datasource::file_format::write::demux::DemuxedStreamReceiver;
3434
use crate::datasource::file_format::write::BatchSerializer;
35-
use crate::datasource::physical_plan::{
36-
FileGroupDisplay, FileSink, FileSinkConfig, JsonSource,
37-
};
35+
use crate::datasource::physical_plan::{FileSink, FileSinkConfig, JsonSource};
3836
use crate::error::Result;
3937
use crate::execution::SessionState;
4038
use crate::physical_plan::insert::{DataSink, DataSinkExec};
@@ -52,12 +50,13 @@ use datafusion_common::config::{ConfigField, ConfigFileType, JsonOptions};
5250
use datafusion_common::file_options::json_writer::JsonWriterOptions;
5351
use datafusion_common::{not_impl_err, GetExt, DEFAULT_JSON_EXTENSION};
5452
use datafusion_common_runtime::SpawnedTask;
53+
use datafusion_datasource::display::FileGroupDisplay;
54+
use datafusion_datasource::file::FileSource;
5555
use datafusion_execution::TaskContext;
5656
use datafusion_expr::dml::InsertOp;
5757
use datafusion_physical_expr::PhysicalExpr;
5858
use datafusion_physical_plan::ExecutionPlan;
5959

60-
use crate::datasource::data_source::FileSource;
6160
use async_trait::async_trait;
6261
use bytes::{Buf, Bytes};
6362
use datafusion_physical_expr_common::sort_expr::LexRequirement;

datafusion/core/src/datasource/file_format/mod.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@ pub mod json;
2828
pub mod options;
2929
#[cfg(feature = "parquet")]
3030
pub mod parquet;
31+
use datafusion_datasource::file::FileSource;
3132
pub use datafusion_datasource::file_compression_type;
33+
use datafusion_datasource::file_scan_config::FileScanConfig;
3234
pub use datafusion_datasource::write;
3335

3436
use std::any::Any;
@@ -40,7 +42,7 @@ use std::task::Poll;
4042
use crate::arrow::array::RecordBatch;
4143
use crate::arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef};
4244
use crate::arrow::error::ArrowError;
43-
use crate::datasource::physical_plan::{FileScanConfig, FileSinkConfig};
45+
use crate::datasource::physical_plan::FileSinkConfig;
4446
use crate::error::Result;
4547
use crate::physical_plan::{ExecutionPlan, Statistics};
4648

@@ -50,7 +52,6 @@ use datafusion_common::{internal_err, not_impl_err, GetExt};
5052
use datafusion_expr::Expr;
5153
use datafusion_physical_expr::PhysicalExpr;
5254

53-
use crate::datasource::data_source::FileSource;
5455
use async_trait::async_trait;
5556
use bytes::{Buf, Bytes};
5657
use datafusion_physical_expr_common::sort_expr::LexRequirement;

datafusion/core/src/datasource/file_format/parquet.rs

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,15 @@ use super::write::{create_writer, SharedBuffer};
2828
use super::{
2929
coerce_file_schema_to_string_type, coerce_file_schema_to_view_type,
3030
transform_binary_to_string, transform_schema_to_view, FileFormat, FileFormatFactory,
31-
FilePushdownSupport, FileScanConfig,
31+
FilePushdownSupport,
3232
};
3333
use crate::arrow::array::RecordBatch;
3434
use crate::arrow::datatypes::{Fields, Schema, SchemaRef};
3535
use crate::datasource::file_format::file_compression_type::FileCompressionType;
3636
use crate::datasource::file_format::write::get_writer_schema;
3737
use crate::datasource::physical_plan::parquet::can_expr_be_pushed_down_with_schemas;
3838
use crate::datasource::physical_plan::parquet::source::ParquetSource;
39-
use crate::datasource::physical_plan::{FileGroupDisplay, FileSink, FileSinkConfig};
39+
use crate::datasource::physical_plan::{FileSink, FileSinkConfig};
4040
use crate::datasource::statistics::{create_max_min_accs, get_col_stats};
4141
use crate::error::Result;
4242
use crate::execution::SessionState;
@@ -57,6 +57,9 @@ use datafusion_common::{
5757
DEFAULT_PARQUET_EXTENSION,
5858
};
5959
use datafusion_common_runtime::SpawnedTask;
60+
use datafusion_datasource::display::FileGroupDisplay;
61+
use datafusion_datasource::file::FileSource;
62+
use datafusion_datasource::file_scan_config::FileScanConfig;
6063
use datafusion_execution::memory_pool::{MemoryConsumer, MemoryPool, MemoryReservation};
6164
use datafusion_execution::TaskContext;
6265
use datafusion_expr::dml::InsertOp;
@@ -65,7 +68,6 @@ use datafusion_functions_aggregate::min_max::{MaxAccumulator, MinAccumulator};
6568
use datafusion_physical_expr::PhysicalExpr;
6669
use datafusion_physical_expr_common::sort_expr::LexRequirement;
6770

68-
use crate::datasource::data_source::FileSource;
6971
use async_trait::async_trait;
7072
use bytes::Bytes;
7173
use futures::future::BoxFuture;

0 commit comments

Comments
 (0)