Skip to content

Commit fa5c852

Browse files
andygroveclaude
andcommitted
fix: use Spark date_add/date_sub UDFs for Date32 +/- Int arithmetic
DataFusion 52's arrow-arith kernels only support Date32 +/- Interval types, not raw integers. When Spark sends Date32 + Int8/Int16/Int32 arithmetic, the planner now routes these operations to the Spark date_add/date_sub UDFs which handle integer types directly. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 1366047 commit fa5c852

File tree

1 file changed

+35
-0
lines changed

1 file changed

+35
-0
lines changed

native/core/src/execution/planner.rs

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -730,6 +730,41 @@ impl PhysicalPlanner {
730730
Arc::new(ConfigOptions::default()),
731731
)))
732732
}
733+
// Date +/- Int8/Int16/Int32: DataFusion 52's arrow-arith kernels only
734+
// support Date32 +/- Interval types, not raw integers. Use the Spark
735+
// date_add / date_sub UDFs which handle Int8/Int16/Int32 directly.
736+
(
737+
DataFusionOperator::Plus,
738+
Ok(DataType::Date32),
739+
Ok(DataType::Int8 | DataType::Int16 | DataType::Int32),
740+
) => {
741+
let udf = Arc::new(ScalarUDF::new_from_impl(
742+
datafusion_spark::function::datetime::date_add::SparkDateAdd::new(),
743+
));
744+
Ok(Arc::new(ScalarFunctionExpr::new(
745+
"date_add",
746+
udf,
747+
vec![left, right],
748+
Arc::new(Field::new("date_add", DataType::Date32, true)),
749+
Arc::new(ConfigOptions::default()),
750+
)))
751+
}
752+
(
753+
DataFusionOperator::Minus,
754+
Ok(DataType::Date32),
755+
Ok(DataType::Int8 | DataType::Int16 | DataType::Int32),
756+
) => {
757+
let udf = Arc::new(ScalarUDF::new_from_impl(
758+
datafusion_spark::function::datetime::date_sub::SparkDateSub::new(),
759+
));
760+
Ok(Arc::new(ScalarFunctionExpr::new(
761+
"date_sub",
762+
udf,
763+
vec![left, right],
764+
Arc::new(Field::new("date_sub", DataType::Date32, true)),
765+
Arc::new(ConfigOptions::default()),
766+
)))
767+
}
733768
_ => {
734769
let data_type = return_type.map(to_arrow_datatype).unwrap();
735770
if [EvalMode::Try, EvalMode::Ansi].contains(&eval_mode)

0 commit comments

Comments
 (0)