Skip to content
Open
6 changes: 6 additions & 0 deletions docs/changelog/136548.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 136548
summary: Locale and timezone argument for `date_parse`
area: ES|QL
type: enhancement
issues:
- 132487

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
Expand Up @@ -644,6 +644,30 @@ emp_no:integer | new_date:datetime | birth_date:datetime | bool:
10050 | 1958-05-21T00:00:00.000Z | 1958-05-21T00:00:00.000Z | true
;

evalDateParseWithTimezoneOption
required_capability: date_parse_options
row a = "10-10-2025" | eval b = date_parse("dd-mm-yyyy", a, {"time_zone":"Europe/Paris"}) | keep b;

b:datetime
2024-12-31T23:00:00.000Z
;

evalDateParseWithLocaleOption
required_capability: date_parse_options
row a = "10 septembre 2025" | eval b = date_parse("dd MMMM yyyy", a, {"locale":"fr"}) | keep b;

b:datetime
2025-09-10T00:00:00.000Z
;

evalDateParseWithLocaleAndTimezoneOption
required_capability: date_parse_options
row a = "10 septembre 2025" | eval b = date_parse("dd MMMM yyyy", a, {"locale":"fr","time_zone":"Europe/Paris"}) | keep b;

b:datetime
2025-09-09T22:00:00.000Z
;

dateFields
from employees | where emp_no == 10049 or emp_no == 10050
| eval year = date_extract("year", birth_date), month = date_extract("month_of_year", birth_date), day = date_extract("day_of_month", birth_date)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1516,6 +1516,11 @@ public enum Cap {
*/
FIX_FILTER_ORDINALS,

/**
* Optional options argument for DATE_PARSE
*/
DATE_PARSE_OPTIONS,

;

private final boolean enabled;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1037,12 +1037,20 @@ public interface BinaryBuilder<T> {
protected static <T extends Function> FunctionDefinition def(Class<T> function, TernaryBuilder<T> ctorRef, String... names) {
FunctionBuilder builder = (source, children, cfg) -> {
boolean hasMinimumTwo = OptionalArgument.class.isAssignableFrom(function);
if (hasMinimumTwo && (children.size() > 3 || children.size() < 2)) {
boolean hasMinimumOne = TwoOptionalArguments.class.isAssignableFrom(function);
if (hasMinimumOne && (children.size() > 3 || children.isEmpty())) {
throw new QlIllegalArgumentException("expects one, two or three arguments");
} else if (hasMinimumTwo && (children.size() > 3 || children.size() < 2)) {
throw new QlIllegalArgumentException("expects two or three arguments");
} else if (hasMinimumTwo == false && children.size() != 3) {
} else if (hasMinimumOne == false && hasMinimumTwo == false && children.size() != 3) {
throw new QlIllegalArgumentException("expects exactly three arguments");
}
return ctorRef.build(source, children.get(0), children.get(1), children.size() == 3 ? children.get(2) : null);
return ctorRef.build(
source,
children.get(0),
children.size() > 1 ? children.get(1) : null,
children.size() == 3 ? children.get(2) : null
);
};
return def(function, builder, names);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,41 +12,58 @@
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.util.LocaleUtils;
import org.elasticsearch.compute.ann.Evaluator;
import org.elasticsearch.compute.ann.Fixed;
import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator;
import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.MapExpression;
import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.expression.function.Example;
import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
import org.elasticsearch.xpack.esql.expression.function.OptionalArgument;
import org.elasticsearch.xpack.esql.expression.function.MapParam;
import org.elasticsearch.xpack.esql.expression.function.Options;
import org.elasticsearch.xpack.esql.expression.function.Param;
import org.elasticsearch.xpack.esql.expression.function.TwoOptionalArguments;
import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;

import java.io.IOException;
import java.time.ZoneId;
import java.time.zone.ZoneRulesException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;

import static java.util.Map.entry;
import static org.elasticsearch.common.time.DateFormatter.forPattern;
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.THIRD;
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isStringAndExact;
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER;
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToLong;

public class DateParse extends EsqlScalarFunction implements OptionalArgument {
public class DateParse extends EsqlScalarFunction implements TwoOptionalArguments {
public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
Expression.class,
"DateParse",
DateParse::new
);

private static final String TIME_ZONE_PARAM_NAME = "time_zone";
private static final String LOCALE_PARAM_NAME = "locale";

private final Expression field;
private final Expression format;
private final Expression options;

@FunctionInfo(
returnType = "date",
Expand All @@ -63,17 +80,51 @@ public DateParse(
name = "dateString",
type = { "keyword", "text" },
description = "Date expression as a string. If `null` or an empty string, the function returns `null`."
) Expression second
) Expression second,
@MapParam(
name = "options",
params = {
@MapParam.MapParamEntry(
name = TIME_ZONE_PARAM_NAME,
type = "keyword",
valueHint = { "standard" },
description = "Coordinated Universal Time (UTC) offset or IANA time zone used to convert date values in the "
+ "query string to UTC."
),
@MapParam.MapParamEntry(
name = LOCALE_PARAM_NAME,
type = "keyword",
valueHint = { "standard" },
description = "The locale to use when parsing the date, relevant when parsing month names or week days."
) },
description = "(Optional) Additional options for date parsing, specifying time zone and locale"
+ "as <<esql-function-named-params,function named parameters>>.",
optional = true
) Expression options
) {
super(source, second != null ? List.of(first, second) : List.of(first));
super(source, fields(first, second, options));
this.field = second != null ? second : first;
this.format = second != null ? first : null;
this.options = options;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This troubles me a bit. We're only allowing the options if the 2 previous parameters are present, even knowing that the format is optional too. That means, we're not allowing something like: DATE_PARSE(date, {})

However, the SVG (docs) shows it correctly.

Now, I don't know if we did something like this before, or if we should allow it. Our function overriding detection is quite manual right now, to begin with. I would like if somebody else from the team can review this first.

The worse that could happen if we ship this is that:

  • We would have incorrect docs
  • We would probably give meaningless errors, as users would expect the map parameter to "work"

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good call... To me both allowing DATE_PARSE(date, {}) or enforcing three params for options make sense, if we clearly communicate this to the user.

Happy for someone else to make the call.

Copy link
Contributor Author

@flash1293 flash1293 Oct 17, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For reference, DATE_PARSE(date_string, {\"time_zone\": \"Europe/Paris\",\"locale\":\"fr\"}) yields

{
  "error": {
    "root_cause": [
      {
        "type": "verification_exception",
        "reason": "Found 1 problem\nline 1:51: second argument of [DATE_PARSE(date_string, {\"time_zone\": \"Europe/Paris\",\"locale\":\"fr\"})] must be [string], found value [{\"time_zone\": \"Europe/Paris\",\"locale\":\"fr\"}] type [unsupported]"
      }
    ],
    "type": "verification_exception",
    "reason": "Found 1 problem\nline 1:51: second argument of [DATE_PARSE(date_string, {\"time_zone\": \"Europe/Paris\",\"locale\":\"fr\"})] must be [string], found value [{\"time_zone\": \"Europe/Paris\",\"locale\":\"fr\"}] type [unsupported]"
  },
  "status": 400
}

right now.

Your call of course, but if you feel comfortable with this behavior, we can get it in, then follow up later with supporting a more flexible calling pattern - we are lucky that the existing behavior on this PR is a subset of what could realistically be supported.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm fine with closing this as-is. I would try to follow up soon though, as to not have the wrong docs there.

Btw, I was discussing this a bit, and I think we could make a working check here. Something like:

if (options == null) {
    if (second == null) {
        // 1 parameter, it's the date
    } else {
        if (second instanceof MapExpression) {
            // Second and options params, no format
        } else {
            // First and second params, no options
        }
    }
} else {
    // 3 params available, no doubt here
}

In general, having an optional param before a required is quite weird. But this is ""historical"" already, so here we are. For this special case, I think the logic to check it shouldn't be too complex, and we can manage to do it.
If there's some weird planning error after doing it, we can check it 👀

}

private static List<Expression> fields(Expression field, Expression format, Expression options) {
List<Expression> list = new ArrayList<>(3);
list.add(field);
if (format != null) {
list.add(format);
}
if (options != null) {
list.add(options);
}
return list;
}

private DateParse(StreamInput in) throws IOException {
this(
Source.readFrom((PlanStreamInput) in),
in.readNamedWriteable(Expression.class),
in.readOptionalNamedWriteable(Expression.class),
in.readOptionalNamedWriteable(Expression.class)
);
}
Expand All @@ -82,7 +133,8 @@ private DateParse(StreamInput in) throws IOException {
public void writeTo(StreamOutput out) throws IOException {
source().writeTo(out);
out.writeNamedWriteable(children().get(0));
out.writeOptionalNamedWriteable(children().size() == 2 ? children().get(1) : null);
out.writeOptionalNamedWriteable(children().size() > 1 ? children().get(1) : null);
out.writeOptionalNamedWriteable(children().size() > 2 ? children().get(2) : null);
}

@Override
Expand Down Expand Up @@ -132,6 +184,21 @@ static long process(BytesRef val, BytesRef formatter) throws IllegalArgumentExce
return dateTimeToLong(val.utf8ToString(), toFormatter(formatter));
}

public static final Map<String, DataType> ALLOWED_OPTIONS = Map.ofEntries(
entry(TIME_ZONE_PARAM_NAME, KEYWORD),
entry(LOCALE_PARAM_NAME, KEYWORD)
);

private Map<String, Object> parseOptions() throws InvalidArgumentException {
Map<String, Object> matchOptions = new HashMap<>();
if (this.options == null) {
return matchOptions;
}

Options.populateMap((MapExpression) this.options, matchOptions, source(), THIRD, ALLOWED_OPTIONS);
return matchOptions;
}

@Override
public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
ExpressionEvaluator.Factory fieldEvaluator = toEvaluator.apply(field);
Expand All @@ -141,9 +208,29 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
if (DataType.isString(format.dataType()) == false) {
throw new IllegalArgumentException("unsupported data type for date_parse [" + format.dataType() + "]");
}
var parsedOptions = this.parseOptions();
String localeAsString = (String) parsedOptions.get(LOCALE_PARAM_NAME);
Locale locale = localeAsString == null ? null : LocaleUtils.parse(localeAsString);

String timezoneAsString = (String) parsedOptions.get(TIME_ZONE_PARAM_NAME);
ZoneId timezone = null;
try {
if (timezoneAsString != null) {
timezone = ZoneId.of(timezoneAsString);
}
} catch (ZoneRulesException e) {
throw new IllegalArgumentException("unsupported timezone [" + timezoneAsString + "]");
}

if (format.foldable()) {
try {
DateFormatter formatter = toFormatter(format.fold(toEvaluator.foldCtx()));
if (locale != null) {
formatter = formatter.withLocale(locale);
}
if (timezone != null) {
formatter = formatter.withZone(timezone);
}
return new DateParseConstantEvaluator.Factory(source(), fieldEvaluator, formatter);
} catch (IllegalArgumentException e) {
throw new InvalidArgumentException(e, "invalid date pattern for [{}]: {}", sourceText(), e.getMessage());
Expand All @@ -159,13 +246,18 @@ private static DateFormatter toFormatter(Object format) {

@Override
public Expression replaceChildren(List<Expression> newChildren) {
return new DateParse(source(), newChildren.get(0), newChildren.size() > 1 ? newChildren.get(1) : null);
return new DateParse(
source(),
newChildren.get(0),
newChildren.size() > 1 ? newChildren.get(1) : null,
newChildren.size() > 2 ? newChildren.get(2) : null
);
}

@Override
protected NodeInfo<? extends Expression> info() {
Expression first = format != null ? format : field;
Expression second = format != null ? field : null;
return NodeInfo.create(this, DateParse::new, first, second);
return NodeInfo.create(this, DateParse::new, first, second, options);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ protected List<TestCaseSupplier> cases() {

@Override
protected Expression build(Source source, List<Expression> args) {
return new DateParse(source, args.get(0), args.size() > 1 ? args.get(1) : null);
return new DateParse(source, args.get(0), args.size() > 1 ? args.get(1) : null, args.size() == 3 ? args.get(2) : null);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ protected DateParse createTestInstance() {
Source source = randomSource();
Expression first = randomChild();
Expression second = randomBoolean() ? null : randomChild();
return new DateParse(source, first, second);
Expression options = second != null && randomBoolean() ? randomChild() : null;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can't we create the options even if "second" is null?

return new DateParse(source, first, second, options);
}

@Override
Expand All @@ -32,6 +33,11 @@ protected DateParse mutateInstance(DateParse instance) throws IOException {
} else {
second = randomValueOtherThan(second, () -> randomBoolean() ? null : randomChild());
}
return new DateParse(source, first, second);
Expression options = instance.children().size() == 3 ? instance.children().get(2) : null;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We usually use a switch here, to only change 1 field. For example:

@Override
protected Top mutateInstance(Top instance) throws IOException {
Source source = instance.source();
Expression field = instance.field();
Expression limit = instance.limitField();
Expression order = instance.orderField();
switch (between(0, 2)) {
case 0 -> field = randomValueOtherThan(field, AbstractExpressionSerializationTests::randomChild);
case 1 -> limit = randomValueOtherThan(limit, AbstractExpressionSerializationTests::randomChild);
case 2 -> order = randomValueOtherThan(order, AbstractExpressionSerializationTests::randomChild);
}
return new Top(source, field, limit, order);
}


if (randomBoolean()) {
options = randomValueOtherThan(first, AbstractExpressionSerializationTests::randomChild);
}
return new DateParse(source, first, second, options);
}
}
Loading
Loading