Skip to content

Commit abaef98

Browse files
committed
fix SPARK-48498
1 parent c51a9d4 commit abaef98

File tree

1 file changed

+20
-1
lines changed

1 file changed

+20
-1
lines changed

native/spark-expr/src/static_invoke/char_varchar_utils/read_side_padding.rs

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
use crate::utils::make_scalar_function;
1919
use arrow::array::builder::GenericStringBuilder;
2020
use arrow::array::types::Int32Type;
21-
use arrow::array::{Array, AsArray};
21+
use arrow::array::{as_dictionary_array, make_array, Array, AsArray, DictionaryArray};
2222
use arrow::array::{ArrayRef, OffsetSizeTrait};
2323
use arrow::datatypes::DataType;
2424
use datafusion::common::{cast::as_generic_string_array, DataFusionError};
@@ -56,6 +56,25 @@ fn spark_read_side_padding2(
5656
DataType::LargeUtf8 => {
5757
spark_read_side_padding_space_internal::<i64>(array, truncate, array_int)
5858
}
59+
// Dictionary support required for SPARK-48498
60+
DataType::Dictionary(_, value_type) => {
61+
let dict = as_dictionary_array::<Int32Type>(array);
62+
let values = if value_type.as_ref() == &DataType::Utf8 {
63+
spark_read_side_padding_space_internal::<i32>(
64+
dict.values(),
65+
truncate,
66+
array_int,
67+
)?
68+
} else {
69+
spark_read_side_padding_space_internal::<i64>(
70+
dict.values(),
71+
truncate,
72+
array_int,
73+
)?
74+
};
75+
let result = DictionaryArray::try_new(dict.keys().clone(), values)?;
76+
Ok(make_array(result.into()))
77+
}
5978
other => Err(DataFusionError::Internal(format!(
6079
"Unsupported data type {other:?} for function rpad/read_side_padding",
6180
))),

0 commit comments

Comments
 (0)