Skip to content

perf: Optimize to_char to allocate less, fix NULL handling #20635

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
neilconway wants to merge 2 commits into
base: main
Choose a base branch
from
+124 −149
Open

perf: Optimize to_char to allocate less, fix NULL handling #20635

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0a1245c
perf: Optimize to_char to reduce allocations
neilconway Mar 1, 2026
ec21e19
cargo fmt
neilconway Mar 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 24 additions & 37 deletions datafusion/functions/benches/to_char.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,12 @@
use std::hint::black_box;
use std::sync::Arc;

use arrow::array::{ArrayRef, Date32Array, StringArray};
use arrow::array::{ArrayRef, Date32Array, Date64Array, StringArray};
use arrow::datatypes::{DataType, Field};
use chrono::TimeDelta;
use chrono::prelude::*;
use criterion::{Criterion, criterion_group, criterion_main};
use datafusion_common::ScalarValue;
use datafusion_common::ScalarValue::TimestampNanosecond;
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::datetime::to_char;
Expand Down Expand Up @@ -63,6 +62,26 @@ fn generate_date32_array(rng: &mut ThreadRng) -> Date32Array {
Date32Array::from(data)
}

fn generate_date64_array(rng: &mut ThreadRng) -> Date64Array {
let start_date = "1970-01-01"
.parse::<NaiveDate>()
.expect("Date should parse");
let end_date = "2050-12-31"
.parse::<NaiveDate>()
.expect("Date should parse");
let mut data: Vec<i64> = Vec::with_capacity(1000);
for _ in 0..1000 {
let date = pick_date_in_range(rng, start_date, end_date);
let millis = date
.and_hms_opt(0, 0, 0)
.unwrap()
.and_utc()
.timestamp_millis();
data.push(millis);
}
Date64Array::from(data)
}

const DATE_PATTERNS: [&str; 5] =
["%Y:%m:%d", "%d-%m-%Y", "%d%m%Y", "%Y%m%d", "%Y...%m...%d"];

Expand Down Expand Up @@ -155,7 +174,7 @@ fn criterion_benchmark(c: &mut Criterion) {

c.bench_function("to_char_array_datetime_patterns_1000", |b| {
let mut rng = rand::rng();
let data_arr = generate_date32_array(&mut rng);
let data_arr = generate_date64_array(&mut rng);
let batch_len = data_arr.len();
let data = ColumnarValue::Array(Arc::new(data_arr) as ArrayRef);
let patterns = ColumnarValue::Array(Arc::new(generate_datetime_pattern_array(
Expand All @@ -182,7 +201,7 @@ fn criterion_benchmark(c: &mut Criterion) {

c.bench_function("to_char_array_mixed_patterns_1000", |b| {
let mut rng = rand::rng();
let data_arr = generate_date32_array(&mut rng);
let data_arr = generate_date64_array(&mut rng);
let batch_len = data_arr.len();
let data = ColumnarValue::Array(Arc::new(data_arr) as ArrayRef);
let patterns = ColumnarValue::Array(Arc::new(generate_mixed_pattern_array(
Expand Down Expand Up @@ -235,7 +254,7 @@ fn criterion_benchmark(c: &mut Criterion) {

c.bench_function("to_char_scalar_datetime_pattern_1000", |b| {
let mut rng = rand::rng();
let data_arr = generate_date32_array(&mut rng);
let data_arr = generate_date64_array(&mut rng);
let batch_len = data_arr.len();
let data = ColumnarValue::Array(Arc::new(data_arr) as ArrayRef);
let patterns = ColumnarValue::Scalar(ScalarValue::Utf8(Some(
Expand All @@ -259,38 +278,6 @@ fn criterion_benchmark(c: &mut Criterion) {
)
})
});

c.bench_function("to_char_scalar_1000", |b| {
let mut rng = rand::rng();
let timestamp = "2026-07-08T09:10:11"
.parse::<NaiveDateTime>()
.unwrap()
.with_nanosecond(56789)
.unwrap()
.and_utc()
.timestamp_nanos_opt()
.unwrap();
let data = ColumnarValue::Scalar(TimestampNanosecond(Some(timestamp), None));
let pattern =
ColumnarValue::Scalar(ScalarValue::Utf8(Some(pick_date_pattern(&mut rng))));

b.iter(|| {
black_box(
to_char()
.invoke_with_args(ScalarFunctionArgs {
args: vec![data.clone(), pattern.clone()],
arg_fields: vec![
Field::new("a", data.data_type(), true).into(),
Field::new("b", pattern.data_type(), true).into(),
],
number_rows: 1,
return_field: Field::new("f", DataType::Utf8, true).into(),
config_options: Arc::clone(&config_options),
})
.expect("to_char should work on valid values"),
)
})
});
}

criterion_group!(benches, criterion_benchmark);
Expand Down
193 changes: 83 additions & 110 deletions datafusion/functions/src/datetime/to_char.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,15 @@
use std::any::Any;
use std::sync::Arc;

use arrow::array::builder::StringBuilder;
use arrow::array::cast::AsArray;
use arrow::array::{Array, ArrayRef, StringArray, new_null_array};
use arrow::array::{Array, ArrayRef, new_null_array};
use arrow::compute::cast;
use arrow::datatypes::DataType;
use arrow::datatypes::DataType::{
Date32, Date64, Duration, Time32, Time64, Timestamp, Utf8,
};
use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second};
use arrow::error::ArrowError;
use arrow::util::display::{ArrayFormatter, DurationFormat, FormatOptions};
use datafusion_common::{Result, ScalarValue, exec_err, utils::take_function_args};
use datafusion_expr::TypeSignature::Exact;
Expand Down Expand Up @@ -143,20 +143,15 @@ impl ScalarUDFImpl for ToCharFunc {
let [date_time, format] = take_function_args(self.name(), &args)?;

match format {
ColumnarValue::Scalar(ScalarValue::Utf8(None))
| ColumnarValue::Scalar(ScalarValue::Null) => to_char_scalar(date_time, None),
// constant format
ColumnarValue::Scalar(ScalarValue::Utf8(Some(format))) => {
// invoke to_char_scalar with the known string, without converting to array
to_char_scalar(date_time, Some(format))
ColumnarValue::Scalar(ScalarValue::Null) => to_char_scalar(date_time, None),
ColumnarValue::Scalar(ScalarValue::Utf8(fmt)) => {
to_char_scalar(date_time, fmt.as_deref())
}
ColumnarValue::Array(_) => to_char_array(&args),
_ => {
exec_err!(
"Format for `to_char` must be non-null Utf8, received {}",
format.data_type()
)
}
_ => exec_err!(
"Format for `to_char` must be non-null Utf8, received {}",
format.data_type()
),
}
}

Expand All @@ -172,7 +167,7 @@ impl ScalarUDFImpl for ToCharFunc {
fn build_format_options<'a>(
data_type: &DataType,
format: Option<&'a str>,
) -> Result<FormatOptions<'a>, Result<ColumnarValue>> {
) -> Result<FormatOptions<'a>> {
let Some(format) = format else {
return Ok(FormatOptions::new());
};
Expand All @@ -194,24 +189,24 @@ fn build_format_options<'a>(
},
),
other => {
return Err(exec_err!(
return exec_err!(
"to_char only supports date, time, timestamp and duration data types, received {other:?}"
));
);
}
};
Ok(format_options)
}

/// Special version when arg\[1] is a scalar
/// Formats `expression` using a constant `format` string.
fn to_char_scalar(
expression: &ColumnarValue,
format: Option<&str>,
) -> Result<ColumnarValue> {
// it's possible that the expression is a scalar however because
// of the implementation in arrow-rs we need to convert it to an array
// ArrayFormatter requires an array, so scalar expressions must be
// converted to a 1-element array first.
let data_type = &expression.data_type();
let is_scalar_expression = matches!(&expression, ColumnarValue::Scalar(_));
let array = expression.clone().into_array(1)?;
let array = expression.to_array(1)?;

if format.is_none() {
return if is_scalar_expression {
Expand All @@ -221,117 +216,95 @@ fn to_char_scalar(
};
}

let format_options = match build_format_options(data_type, format) {
Ok(value) => value,
Err(value) => return value,
};

let format_options = build_format_options(data_type, format)?;
let formatter = ArrayFormatter::try_new(array.as_ref(), &format_options)?;
let formatted: Result<Vec<Option<String>>, ArrowError> = (0..array.len())
.map(|i| {
if array.is_null(i) {
Ok(None)
} else {
formatter.value(i).try_to_string().map(Some)
}
})
.collect();

if let Ok(formatted) = formatted {
if is_scalar_expression {
Ok(ColumnarValue::Scalar(ScalarValue::Utf8(
formatted.first().unwrap().clone(),
)))

let fmt_len = format.map_or(20, |f| f.len() + 10);
let mut builder = StringBuilder::with_capacity(array.len(), array.len() * fmt_len);

for i in 0..array.len() {
if array.is_null(i) {
builder.append_null();
} else {
Ok(ColumnarValue::Array(
Arc::new(StringArray::from(formatted)) as ArrayRef
))
}
} else {
// if the data type was a Date32, formatting could have failed because the format string
// contained datetime specifiers, so we'll retry by casting the date array as a timestamp array
if data_type == &Date32 {
return to_char_scalar(&expression.cast_to(&Date64, None)?, format);
// Write directly into the builder's internal buffer, then
// commit the value with append_value("").
match formatter.value(i).write(&mut builder) {
Ok(()) => builder.append_value(""),
// Arrow's Date32 formatter only handles date specifiers
// (%Y, %m, %d, ...). Format strings with time specifiers
// (%H, %M, %S, ...) cause it to fail. When this happens,
// we retry by casting to Date64, whose datetime formatter
// handles both date and time specifiers (with zero for
// the time components).
Err(_) if data_type == &Date32 => {
return to_char_scalar(&expression.cast_to(&Date64, None)?, format);
}
Err(e) => return Err(e.into()),
}
}
}

exec_err!("{}", formatted.unwrap_err())
let result = builder.finish();
if is_scalar_expression {
let val = result.is_valid(0).then(|| result.value(0).to_string());
Ok(ColumnarValue::Scalar(ScalarValue::Utf8(val)))
} else {
Ok(ColumnarValue::Array(Arc::new(result) as ArrayRef))
}
}

fn to_char_array(args: &[ColumnarValue]) -> Result<ColumnarValue> {
let arrays = ColumnarValue::values_to_arrays(args)?;
let mut results: Vec<Option<String>> = vec![];
let data_array = &arrays[0];
let format_array = arrays[1].as_string::<i32>();
let data_type = arrays[0].data_type();
let data_type = data_array.data_type();

for idx in 0..arrays[0].len() {
let format = if format_array.is_null(idx) {
None
} else {
Some(format_array.value(idx))
};
if format.is_none() {
results.push(None);
let fmt_len = 30;
let mut builder =
StringBuilder::with_capacity(data_array.len(), data_array.len() * fmt_len);
let mut buffer = String::with_capacity(fmt_len);

for idx in 0..data_array.len() {
if format_array.is_null(idx) || data_array.is_null(idx) {
builder.append_null();
continue;
}
let format_options = match build_format_options(data_type, format) {
Ok(value) => value,
Err(value) => return value,
};
// this isn't ideal but this can't use ValueFormatter as it isn't independent
// from ArrayFormatter
let formatter = ArrayFormatter::try_new(arrays[0].as_ref(), &format_options)?;
let result = formatter.value(idx).try_to_string();
match result {
Ok(value) => results.push(Some(value)),
Err(e) => {
// if the data type was a Date32, formatting could have failed because the format string
// contained datetime specifiers, so we'll treat this specific date element as a timestamp
if data_type == &Date32 {
let failed_date_value = arrays[0].slice(idx, 1);

match retry_date_as_timestamp(&failed_date_value, &format_options) {
Ok(value) => {
results.push(Some(value));
continue;
}
Err(e) => {
return exec_err!("{}", e);
}
}
}

return exec_err!("{}", e);
let format = Some(format_array.value(idx));
let format_options = build_format_options(data_type, format)?;
let formatter = ArrayFormatter::try_new(data_array.as_ref(), &format_options)?;

buffer.clear();

// We'd prefer to write directly to the StringBuilder's internal buffer,
// but the write might fail, and there's no easy way to ensure a partial
// write is removed from the buffer. So instead we write to a temporary
// buffer and `append_value` on success.
match formatter.value(idx).write(&mut buffer) {
Ok(()) => builder.append_value(&buffer),
// Retry with Date64 (see comment in to_char_scalar).
Err(_) if data_type == &Date32 => {
buffer.clear();
let date64_value = cast(&data_array.slice(idx, 1), &Date64)?;
let retry_fmt =
ArrayFormatter::try_new(date64_value.as_ref(), &format_options)?;
retry_fmt.value(0).write(&mut buffer)?;
builder.append_value(&buffer);
}
Err(e) => return Err(e.into()),
}
}

let result = builder.finish();
match args[0] {
ColumnarValue::Array(_) => Ok(ColumnarValue::Array(Arc::new(StringArray::from(
results,
)) as ArrayRef)),
ColumnarValue::Scalar(_) => match results.first().unwrap() {
Some(value) => Ok(ColumnarValue::Scalar(ScalarValue::Utf8(Some(
value.to_string(),
)))),
None => Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None))),
},
ColumnarValue::Scalar(_) => {
let val = result.is_valid(0).then(|| result.value(0).to_string());
Ok(ColumnarValue::Scalar(ScalarValue::Utf8(val)))
}
ColumnarValue::Array(_) => Ok(ColumnarValue::Array(Arc::new(result) as ArrayRef)),
}
}

fn retry_date_as_timestamp(
array_ref: &ArrayRef,
format_options: &FormatOptions,
) -> Result<String> {
let target_data_type = Date64;

let date_value = cast(&array_ref, &target_data_type)?;
let formatter = ArrayFormatter::try_new(date_value.as_ref(), format_options)?;
let result = formatter.value(0).try_to_string()?;

Ok(result)
}

#[cfg(test)]
mod tests {
use crate::datetime::to_char::ToCharFunc;
Expand Down
Loading