Skip to content

Update DataFusion version from 48.0.0 to 49.0.0 #82

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 7, 2025
Merged

Update DataFusion version from 48.0.0 to 49.0.0 #82

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
192 changes: 129 additions & 63 deletions Cargo.lock
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ version = "0.1.0"
edition = "2021"

[dependencies]
datafusion = { version = "48.0.0" }
datafusion-proto = { version = "48.0.0" }
datafusion = { version = "49.0.0" }
datafusion-proto = { version = "49.0.0" }
arrow-flight = "55.2.0"
async-trait = "0.1.88"
tokio = { version = "1.46.1", features = ["full"] }
Expand Down
2 changes: 1 addition & 1 deletion rust-toolchain.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
[toolchain]
channel = "1.83.0"
channel = "1.85.1"
profile = "default"
30 changes: 16 additions & 14 deletions src/errors/datafusion_error.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,13 +184,13 @@ impl DataFusionErrorProto {
match inner {
DataFusionErrorInnerProto::ArrowError(err) => {
let (err, ctx) = err.to_arrow_error();
DataFusionError::ArrowError(err, ctx)
DataFusionError::ArrowError(Box::new(err), ctx)
}
DataFusionErrorInnerProto::ParquetError(err) => {
DataFusionError::ParquetError(err.to_parquet_error())
DataFusionError::ParquetError(Box::new(err.to_parquet_error()))
}
DataFusionErrorInnerProto::ObjectStoreError(err) => {
DataFusionError::ObjectStore(err.to_object_store_error())
DataFusionError::ObjectStore(Box::new(err.to_object_store_error()))
}
DataFusionErrorInnerProto::IoError(err) => {
let (err, _) = err.to_io_error();
Expand All @@ -200,7 +200,7 @@ impl DataFusionErrorProto {
let backtrace = err.backtrace.clone();
let err = err.err.as_ref().map(|err| err.to_parser_error());
let err = err.unwrap_or(ParserError::ParserError("".to_string()));
DataFusionError::SQL(err, backtrace)
DataFusionError::SQL(Box::new(err), backtrace)
}
DataFusionErrorInnerProto::NotImplemented(msg) => {
DataFusionError::NotImplemented(msg.clone())
Expand All @@ -212,7 +212,7 @@ impl DataFusionErrorProto {
}
DataFusionErrorInnerProto::Schema(err) => {
let (err, backtrace) = err.to_schema_error();
DataFusionError::SchemaError(err, Box::new(backtrace))
DataFusionError::SchemaError(Box::new(err), Box::new(backtrace))
}
DataFusionErrorInnerProto::Execution(msg) => DataFusionError::Execution(msg.clone()),
// We cannot build JoinErrors ourselves, so instead we map it to internal.
Expand Down Expand Up @@ -283,30 +283,32 @@ mod tests {
fn test_datafusion_error_roundtrip() {
let test_cases = vec![
DataFusionError::ArrowError(
ArrowError::ComputeError("compute".to_string()),
Box::new(ArrowError::ComputeError("compute".to_string())),
Some("arrow context".to_string()),
),
DataFusionError::ParquetError(ParquetError::General("parquet error".to_string())),
DataFusionError::ObjectStore(ObjectStoreError::NotFound {
DataFusionError::ParquetError(Box::new(ParquetError::General(
"parquet error".to_string(),
))),
DataFusionError::ObjectStore(Box::new(ObjectStoreError::NotFound {
path: "test/path".to_string(),
source: Box::new(std::io::Error::new(ErrorKind::NotFound, "not found")),
}),
})),
DataFusionError::IoError(IoError::new(
ErrorKind::PermissionDenied,
"permission denied",
)),
DataFusionError::SQL(
ParserError::ParserError("sql parse error".to_string()),
Box::new(ParserError::ParserError("sql parse error".to_string())),
Some("sql backtrace".to_string()),
),
DataFusionError::NotImplemented("not implemented".to_string()),
DataFusionError::Internal("internal error".to_string()),
DataFusionError::Plan("plan error".to_string()),
DataFusionError::Configuration("config error".to_string()),
DataFusionError::SchemaError(
SchemaError::AmbiguousReference {
field: datafusion::common::Column::new_unqualified("test_field"),
},
Box::new(SchemaError::AmbiguousReference {
field: Box::new(datafusion::common::Column::new_unqualified("test_field")),
}),
Box::new(None),
),
DataFusionError::Execution("execution error".to_string()),
Expand Down Expand Up @@ -377,7 +379,7 @@ mod tests {
#[test]
fn test_sql_error_with_backtrace() {
let sql_error = DataFusionError::SQL(
ParserError::ParserError("syntax error".to_string()),
Box::new(ParserError::ParserError("syntax error".to_string())),
Some("test backtrace".to_string()),
);

Expand Down
13 changes: 7 additions & 6 deletions src/errors/schema_error.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,11 +220,12 @@ impl SchemaErrorProto {

let err = match inner {
SchemaErrorInnerProto::AmbiguousReference(err) => SchemaError::AmbiguousReference {
field: err
.field
.as_ref()
.map(|v| v.to_column())
.unwrap_or(Column::new_unqualified("".to_string())),
field: Box::new(
err.field
.as_ref()
.map(|v| v.to_column())
.unwrap_or(Column::new_unqualified("".to_string())),
),
},
SchemaErrorInnerProto::DuplicateQualifiedField(err) => {
SchemaError::DuplicateQualifiedField {
Expand Down Expand Up @@ -266,7 +267,7 @@ mod tests {
fn test_schema_error_roundtrip() {
let test_cases = vec![
SchemaError::AmbiguousReference {
field: Column::new_unqualified("test_field"),
field: Box::new(Column::new_unqualified("test_field")),
},
SchemaError::DuplicateQualifiedField {
qualifier: Box::new(TableReference::bare("table")),
Expand Down
4 changes: 2 additions & 2 deletions src/physical_optimizer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ mod tests {
┌───── Stage 3 Task: partitions: 0,unassigned]
│partitions [out:1 <-- in:1 ] ProjectionExec: expr=[count(*)@0 as count(*), RainToday@1 as RainToday]
│partitions [out:1 <-- in:4 ] SortPreservingMergeExec: [count(Int64(1))@2 ASC NULLS LAST]
│partitions [out:4 <-- in:4 ] SortExec: expr=[count(Int64(1))@2 ASC NULLS LAST], preserve_partitioning=[true]
│partitions [out:4 <-- in:4 ] SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true]
│partitions [out:4 <-- in:4 ] ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday, count(Int64(1))@1 as count(Int64(1))]
│partitions [out:4 <-- in:4 ] AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))]
│partitions [out:4 <-- in:4 ] CoalesceBatchesExec: target_batch_size=8192
Expand Down Expand Up @@ -253,7 +253,7 @@ mod tests {
┌───── Stage 3 Task: partitions: 0,unassigned]
│partitions [out:1 <-- in:1 ] ProjectionExec: expr=[count(*)@0 as count(*), RainToday@1 as RainToday]
│partitions [out:1 <-- in:4 ] SortPreservingMergeExec: [count(Int64(1))@2 ASC NULLS LAST]
│partitions [out:4 <-- in:4 ] SortExec: expr=[count(Int64(1))@2 ASC NULLS LAST], preserve_partitioning=[true]
│partitions [out:4 <-- in:4 ] SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true]
│partitions [out:4 <-- in:4 ] ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday, count(Int64(1))@1 as count(Int64(1))]
│partitions [out:4 <-- in:4 ] AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))]
│partitions [out:4 <-- in:4 ] CoalesceBatchesExec: target_batch_size=8192
Expand Down
6 changes: 5 additions & 1 deletion src/plan/codec.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ pub struct ArrowFlightReadExecProto {
mod tests {
use super::*;
use datafusion::arrow::datatypes::{DataType, Field};
use datafusion::physical_expr::LexOrdering;
use datafusion::{
execution::registry::MemoryFunctionRegistry,
physical_expr::{expressions::col, expressions::Column, Partitioning, PhysicalSortExpr},
Expand Down Expand Up @@ -257,7 +258,10 @@ mod tests {
expr: col("d", &schema)?,
options: Default::default(),
};
let sort = Arc::new(SortExec::new(vec![sort_expr].into(), flight.clone()));
let sort = Arc::new(SortExec::new(
LexOrdering::new(vec![sort_expr]).unwrap(),
flight.clone(),
));

let plan: Arc<dyn ExecutionPlan> = Arc::new(PartitionIsolatorExec::new(sort.clone(), 2));

Expand Down
6 changes: 4 additions & 2 deletions tests/custom_extension_codec.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,8 @@ mod tests {
LexOrdering::new(vec![PhysicalSortExpr::new(
col("numbers", &plan.schema())?,
SortOptions::new(true, false),
)]),
)])
.unwrap(),
plan,
));

Expand All @@ -155,7 +156,8 @@ mod tests {
LexOrdering::new(vec![PhysicalSortExpr::new(
col("numbers", &plan.schema())?,
SortOptions::new(true, false),
)]),
)])
.unwrap(),
plan,
));
}
Expand Down
Loading