Skip to content

Commit 6d97bab

Browse files
committed
Add cast column format option fields to protobuf
Implement cast column format option fields in the protobuf schema for physical plans. Serialize and reconstruct full cast format options during protobuf conversions, handling format option mapping and fallbacks. Extend the cast column roundtrip test to verify serialization fidelity with non-default format options.
1 parent bf168a0 commit 6d97bab

File tree

10 files changed

+530
-57
lines changed

10 files changed

+530
-57
lines changed

datafusion/physical-expr-adapter/src/schema_rewriter.rs

Lines changed: 38 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -684,12 +684,15 @@ mod tests {
684684
println!("Rewritten expression: {result}");
685685

686686
let expected = expressions::BinaryExpr::new(
687-
Arc::new(CastColumnExpr::new(
688-
Arc::new(Column::new("a", 0)),
689-
Arc::new(Field::new("a", DataType::Int32, false)),
690-
Arc::new(Field::new("a", DataType::Int64, false)),
691-
None,
692-
).unwrap()),
687+
Arc::new(
688+
CastColumnExpr::new(
689+
Arc::new(Column::new("a", 0)),
690+
Arc::new(Field::new("a", DataType::Int32, false)),
691+
Arc::new(Field::new("a", DataType::Int64, false)),
692+
None,
693+
)
694+
.unwrap(),
695+
),
693696
Operator::Plus,
694697
Arc::new(expressions::Literal::new(ScalarValue::Int64(Some(5)))),
695698
);
@@ -768,32 +771,35 @@ mod tests {
768771

769772
let result = adapter.rewrite(column_expr).unwrap();
770773

771-
let expected = Arc::new(CastColumnExpr::new(
772-
Arc::new(Column::new("data", 0)),
773-
Arc::new(Field::new(
774-
"data",
775-
DataType::Struct(
776-
vec![
777-
Field::new("id", DataType::Int32, false),
778-
Field::new("name", DataType::Utf8, true),
779-
]
780-
.into(),
781-
),
782-
false,
783-
)),
784-
Arc::new(Field::new(
785-
"data",
786-
DataType::Struct(
787-
vec![
788-
Field::new("id", DataType::Int64, false),
789-
Field::new("name", DataType::Utf8View, true),
790-
]
791-
.into(),
792-
),
793-
false,
794-
)),
795-
None,
796-
).unwrap()) as Arc<dyn PhysicalExpr>;
774+
let expected = Arc::new(
775+
CastColumnExpr::new(
776+
Arc::new(Column::new("data", 0)),
777+
Arc::new(Field::new(
778+
"data",
779+
DataType::Struct(
780+
vec![
781+
Field::new("id", DataType::Int32, false),
782+
Field::new("name", DataType::Utf8, true),
783+
]
784+
.into(),
785+
),
786+
false,
787+
)),
788+
Arc::new(Field::new(
789+
"data",
790+
DataType::Struct(
791+
vec![
792+
Field::new("id", DataType::Int64, false),
793+
Field::new("name", DataType::Utf8View, true),
794+
]
795+
.into(),
796+
),
797+
false,
798+
)),
799+
None,
800+
)
801+
.unwrap(),
802+
) as Arc<dyn PhysicalExpr>;
797803

798804
assert_eq!(result.to_string(), expected.to_string());
799805
}

datafusion/physical-expr/src/expressions/cast_column.rs

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -444,8 +444,6 @@ mod tests {
444444
)
445445
.expect_err("expected mismatched schema metadata error");
446446

447-
assert!(err
448-
.to_string()
449-
.contains("mismatched schema metadata"));
447+
assert!(err.to_string().contains("mismatched schema metadata"));
450448
}
451449
}

datafusion/physical-expr/src/intervals/utils.rs

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -209,12 +209,9 @@ mod tests {
209209
let target_field = Arc::new(Field::new("a", DataType::Int64, true));
210210

211211
let column_expr = col("a", &schema).unwrap();
212-
let cast_expr = Arc::new(CastColumnExpr::new(
213-
column_expr,
214-
input_field,
215-
target_field,
216-
None,
217-
).unwrap()) as Arc<dyn PhysicalExpr>;
212+
let cast_expr = Arc::new(
213+
CastColumnExpr::new(column_expr, input_field, target_field, None).unwrap(),
214+
) as Arc<dyn PhysicalExpr>;
218215

219216
assert!(check_support(&cast_expr, &schema));
220217
}

datafusion/physical-expr/src/simplifier/unwrap_cast.rs

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -221,12 +221,9 @@ mod tests {
221221

222222
// Create: cast_column(c1 as INT64) > INT64(10)
223223
let column_expr = col("c1", &schema).unwrap();
224-
let cast_expr = Arc::new(CastColumnExpr::new(
225-
column_expr,
226-
input_field,
227-
target_field,
228-
None,
229-
).unwrap());
224+
let cast_expr = Arc::new(
225+
CastColumnExpr::new(column_expr, input_field, target_field, None).unwrap(),
226+
);
230227
let literal_expr = lit(10i64);
231228
let binary_expr =
232229
Arc::new(BinaryExpr::new(cast_expr, Operator::Gt, literal_expr));

datafusion/proto/proto/datafusion.proto

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -991,6 +991,25 @@ message PhysicalCastColumnNode {
991991
datafusion_common.Field input_field = 2;
992992
datafusion_common.Field target_field = 3;
993993
bool safe = 4;
994+
FormatOptions format_options = 5;
995+
}
996+
997+
enum DurationFormat {
998+
DURATION_FORMAT_UNSPECIFIED = 0;
999+
DURATION_FORMAT_ISO8601 = 1;
1000+
DURATION_FORMAT_PRETTY = 2;
1001+
}
1002+
1003+
message FormatOptions {
1004+
bool safe = 1;
1005+
string null = 2;
1006+
optional string date_format = 3;
1007+
optional string datetime_format = 4;
1008+
optional string timestamp_format = 5;
1009+
optional string timestamp_tz_format = 6;
1010+
optional string time_format = 7;
1011+
DurationFormat duration_format = 8;
1012+
bool types_info = 9;
9941013
}
9951014

9961015
message PhysicalNegativeNode {

0 commit comments

Comments
 (0)