diff --git a/docs/changelog/136548.yaml b/docs/changelog/136548.yaml new file mode 100644 index 0000000000000..1c360c8e3cb49 --- /dev/null +++ b/docs/changelog/136548.yaml @@ -0,0 +1,6 @@ +pr: 136548 +summary: Locale and timezone argument for `date_parse` +area: ES|QL +type: enhancement +issues: + - 132487 diff --git a/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/date_parse.md b/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/date_parse.md new file mode 100644 index 0000000000000..328494ce830d3 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/date_parse.md @@ -0,0 +1,10 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported function named parameters** + +`time_zone` +: (keyword) Coordinated Universal Time (UTC) offset or IANA time zone used to convert date values in the query string to UTC. + +`locale` +: (keyword) The locale to use when parsing the date, relevant when parsing month names or week days. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/date_parse.md b/docs/reference/query-languages/esql/_snippets/functions/layout/date_parse.md index 4ac4734ef823b..f9c3d45b01c6d 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/layout/date_parse.md +++ b/docs/reference/query-languages/esql/_snippets/functions/layout/date_parse.md @@ -19,5 +19,8 @@ :::{include} ../types/date_parse.md ::: +:::{include} ../functionNamedParams/date_parse.md +::: + :::{include} ../examples/date_parse.md ::: diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/date_parse.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/date_parse.md index adc0a0c86b19d..b10bec180329c 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/parameters/date_parse.md +++ b/docs/reference/query-languages/esql/_snippets/functions/parameters/date_parse.md @@ -8,3 +8,6 @@ `dateString` : Date expression as a string. If `null` or an empty string, the function returns `null`. +`options` +: (Optional) Additional options for date parsing, specifying time zone and locale as [function named parameters](/reference/query-languages/esql/esql-syntax.md#esql-function-named-params). + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/date_parse.md b/docs/reference/query-languages/esql/_snippets/functions/types/date_parse.md index 4f1873cd3796b..4dec105ca46d4 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/date_parse.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/date_parse.md @@ -2,10 +2,10 @@ **Supported types** -| datePattern | dateString | result | -| --- | --- | --- | -| keyword | keyword | date | -| keyword | text | date | -| text | keyword | date | -| text | text | date | +| datePattern | dateString | options | result | +| --- | --- | --- | --- | +| keyword | keyword | | date | +| keyword | text | | date | +| text | keyword | | date | +| text | text | | date | diff --git a/docs/reference/query-languages/esql/images/functions/date_parse.svg b/docs/reference/query-languages/esql/images/functions/date_parse.svg index 0f5e5f624143a..4a16582db66f8 100644 --- a/docs/reference/query-languages/esql/images/functions/date_parse.svg +++ b/docs/reference/query-languages/esql/images/functions/date_parse.svg @@ -1 +1 @@ -DATE_PARSE(datePattern,dateString) \ No newline at end of file +DATE_PARSE(datePattern,dateString,options) \ No newline at end of file diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec index e5efc9e7a37aa..75eee9ba80c00 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec @@ -644,6 +644,30 @@ emp_no:integer | new_date:datetime | birth_date:datetime | bool: 10050 | 1958-05-21T00:00:00.000Z | 1958-05-21T00:00:00.000Z | true ; +evalDateParseWithTimezoneOption +required_capability: date_parse_options +row a = "10-10-2025" | eval b = date_parse("dd-mm-yyyy", a, {"time_zone":"Europe/Paris"}) | keep b; + +b:datetime +2024-12-31T23:00:00.000Z +; + +evalDateParseWithLocaleOption +required_capability: date_parse_options +row a = "10 septembre 2025" | eval b = date_parse("dd MMMM yyyy", a, {"locale":"fr"}) | keep b; + +b:datetime +2025-09-10T00:00:00.000Z +; + +evalDateParseWithLocaleAndTimezoneOption +required_capability: date_parse_options +row a = "10 septembre 2025" | eval b = date_parse("dd MMMM yyyy", a, {"locale":"fr","time_zone":"Europe/Paris"}) | keep b; + +b:datetime +2025-09-09T22:00:00.000Z +; + dateFields from employees | where emp_no == 10049 or emp_no == 10050 | eval year = date_extract("year", birth_date), month = date_extract("month_of_year", birth_date), day = date_extract("day_of_month", birth_date) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index df06a1109ecc9..243dcb94bcaad 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1516,6 +1516,11 @@ public enum Cap { */ FIX_FILTER_ORDINALS, + /** + * Optional options argument for DATE_PARSE + */ + DATE_PARSE_OPTIONS, + ; private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index c6520c8563d6d..81bded3a789a0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -1037,12 +1037,20 @@ public interface BinaryBuilder { protected static FunctionDefinition def(Class function, TernaryBuilder ctorRef, String... names) { FunctionBuilder builder = (source, children, cfg) -> { boolean hasMinimumTwo = OptionalArgument.class.isAssignableFrom(function); - if (hasMinimumTwo && (children.size() > 3 || children.size() < 2)) { + boolean hasMinimumOne = TwoOptionalArguments.class.isAssignableFrom(function); + if (hasMinimumOne && (children.size() > 3 || children.isEmpty())) { + throw new QlIllegalArgumentException("expects one, two or three arguments"); + } else if (hasMinimumTwo && (children.size() > 3 || children.size() < 2)) { throw new QlIllegalArgumentException("expects two or three arguments"); - } else if (hasMinimumTwo == false && children.size() != 3) { + } else if (hasMinimumOne == false && hasMinimumTwo == false && children.size() != 3) { throw new QlIllegalArgumentException("expects exactly three arguments"); } - return ctorRef.build(source, children.get(0), children.get(1), children.size() == 3 ? children.get(2) : null); + return ctorRef.build( + source, + children.get(0), + children.size() > 1 ? children.get(1) : null, + children.size() == 3 ? children.get(2) : null + ); }; return def(function, builder, names); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java index ef1acbc395308..adfdbd7c6747f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java @@ -12,41 +12,58 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.util.LocaleUtils; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.core.InvalidArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.MapExpression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; -import org.elasticsearch.xpack.esql.expression.function.OptionalArgument; +import org.elasticsearch.xpack.esql.expression.function.MapParam; +import org.elasticsearch.xpack.esql.expression.function.Options; import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.TwoOptionalArguments; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; +import java.time.ZoneId; +import java.time.zone.ZoneRulesException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Locale; +import java.util.Map; +import static java.util.Map.entry; import static org.elasticsearch.common.time.DateFormatter.forPattern; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.THIRD; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; +import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isStringAndExact; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToLong; -public class DateParse extends EsqlScalarFunction implements OptionalArgument { +public class DateParse extends EsqlScalarFunction implements TwoOptionalArguments { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "DateParse", DateParse::new ); + private static final String TIME_ZONE_PARAM_NAME = "time_zone"; + private static final String LOCALE_PARAM_NAME = "locale"; + private final Expression field; private final Expression format; + private final Expression options; @FunctionInfo( returnType = "date", @@ -63,17 +80,51 @@ public DateParse( name = "dateString", type = { "keyword", "text" }, description = "Date expression as a string. If `null` or an empty string, the function returns `null`." - ) Expression second + ) Expression second, + @MapParam( + name = "options", + params = { + @MapParam.MapParamEntry( + name = TIME_ZONE_PARAM_NAME, + type = "keyword", + valueHint = { "standard" }, + description = "Coordinated Universal Time (UTC) offset or IANA time zone used to convert date values in the " + + "query string to UTC." + ), + @MapParam.MapParamEntry( + name = LOCALE_PARAM_NAME, + type = "keyword", + valueHint = { "standard" }, + description = "The locale to use when parsing the date, relevant when parsing month names or week days." + ) }, + description = "(Optional) Additional options for date parsing, specifying time zone and locale " + + "as <>.", + optional = true + ) Expression options ) { - super(source, second != null ? List.of(first, second) : List.of(first)); + super(source, fields(first, second, options)); this.field = second != null ? second : first; this.format = second != null ? first : null; + this.options = options; + } + + private static List fields(Expression field, Expression format, Expression options) { + List list = new ArrayList<>(3); + list.add(field); + if (format != null) { + list.add(format); + } + if (options != null) { + list.add(options); + } + return list; } private DateParse(StreamInput in) throws IOException { this( Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), + in.readOptionalNamedWriteable(Expression.class), in.readOptionalNamedWriteable(Expression.class) ); } @@ -82,7 +133,8 @@ private DateParse(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { source().writeTo(out); out.writeNamedWriteable(children().get(0)); - out.writeOptionalNamedWriteable(children().size() == 2 ? children().get(1) : null); + out.writeOptionalNamedWriteable(children().size() > 1 ? children().get(1) : null); + out.writeOptionalNamedWriteable(children().size() > 2 ? children().get(2) : null); } @Override @@ -132,6 +184,21 @@ static long process(BytesRef val, BytesRef formatter) throws IllegalArgumentExce return dateTimeToLong(val.utf8ToString(), toFormatter(formatter)); } + public static final Map ALLOWED_OPTIONS = Map.ofEntries( + entry(TIME_ZONE_PARAM_NAME, KEYWORD), + entry(LOCALE_PARAM_NAME, KEYWORD) + ); + + private Map parseOptions() throws InvalidArgumentException { + Map matchOptions = new HashMap<>(); + if (this.options == null) { + return matchOptions; + } + + Options.populateMap((MapExpression) this.options, matchOptions, source(), THIRD, ALLOWED_OPTIONS); + return matchOptions; + } + @Override public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { ExpressionEvaluator.Factory fieldEvaluator = toEvaluator.apply(field); @@ -141,9 +208,29 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (DataType.isString(format.dataType()) == false) { throw new IllegalArgumentException("unsupported data type for date_parse [" + format.dataType() + "]"); } + var parsedOptions = this.parseOptions(); + String localeAsString = (String) parsedOptions.get(LOCALE_PARAM_NAME); + Locale locale = localeAsString == null ? null : LocaleUtils.parse(localeAsString); + + String timezoneAsString = (String) parsedOptions.get(TIME_ZONE_PARAM_NAME); + ZoneId timezone = null; + try { + if (timezoneAsString != null) { + timezone = ZoneId.of(timezoneAsString); + } + } catch (ZoneRulesException e) { + throw new IllegalArgumentException("unsupported timezone [" + timezoneAsString + "]"); + } + if (format.foldable()) { try { DateFormatter formatter = toFormatter(format.fold(toEvaluator.foldCtx())); + if (locale != null) { + formatter = formatter.withLocale(locale); + } + if (timezone != null) { + formatter = formatter.withZone(timezone); + } return new DateParseConstantEvaluator.Factory(source(), fieldEvaluator, formatter); } catch (IllegalArgumentException e) { throw new InvalidArgumentException(e, "invalid date pattern for [{}]: {}", sourceText(), e.getMessage()); @@ -159,13 +246,18 @@ private static DateFormatter toFormatter(Object format) { @Override public Expression replaceChildren(List newChildren) { - return new DateParse(source(), newChildren.get(0), newChildren.size() > 1 ? newChildren.get(1) : null); + return new DateParse( + source(), + newChildren.get(0), + newChildren.size() > 1 ? newChildren.get(1) : null, + newChildren.size() > 2 ? newChildren.get(2) : null + ); } @Override protected NodeInfo info() { Expression first = format != null ? format : field; Expression second = format != null ? field : null; - return NodeInfo.create(this, DateParse::new, first, second); + return NodeInfo.create(this, DateParse::new, first, second, options); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseErrorTests.java index 21d9b5fb00537..75b47934b6a04 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseErrorTests.java @@ -27,7 +27,7 @@ protected List cases() { @Override protected Expression build(Source source, List args) { - return new DateParse(source, args.get(0), args.size() > 1 ? args.get(1) : null); + return new DateParse(source, args.get(0), args.size() > 1 ? args.get(1) : null, args.size() == 3 ? args.get(2) : null); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseSerializationTests.java index 79a650c8dd963..dcf47aa82f94d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseSerializationTests.java @@ -19,7 +19,8 @@ protected DateParse createTestInstance() { Source source = randomSource(); Expression first = randomChild(); Expression second = randomBoolean() ? null : randomChild(); - return new DateParse(source, first, second); + Expression options = second != null && randomBoolean() ? randomChild() : null; + return new DateParse(source, first, second, options); } @Override @@ -32,6 +33,11 @@ protected DateParse mutateInstance(DateParse instance) throws IOException { } else { second = randomValueOtherThan(second, () -> randomBoolean() ? null : randomChild()); } - return new DateParse(source, first, second); + Expression options = instance.children().size() == 3 ? instance.children().get(2) : null; + + if (randomBoolean()) { + options = randomValueOtherThan(first, AbstractExpressionSerializationTests::randomChild); + } + return new DateParse(source, first, second, options); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java index 8b53a1b9112b4..c9f5f94a3acdc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.core.InvalidArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.MapExpression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; @@ -156,15 +157,64 @@ public void testInvalidPattern() { new DateParse( Source.EMPTY, new Literal(Source.EMPTY, new BytesRef(pattern), DataType.KEYWORD), - field("str", DataType.KEYWORD) + field("str", DataType.KEYWORD), + null ) ).get(driverContext) ); assertThat(e.getMessage(), startsWith("invalid date pattern for []: Invalid format: [" + pattern + "]")); } + public void testInvalidLocale() { + String pattern = "YYYY"; + String locale = "nonexistinglocale"; + DriverContext driverContext = driverContext(); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> evaluator( + new DateParse( + Source.EMPTY, + new Literal(Source.EMPTY, new BytesRef(pattern), DataType.KEYWORD), + field("str", DataType.KEYWORD), + new MapExpression( + Source.EMPTY, + List.of( + new Literal(Source.EMPTY, new BytesRef("locale"), DataType.KEYWORD), + new Literal(Source.EMPTY, new BytesRef(locale), DataType.KEYWORD) + ) + ) + ) + ).get(driverContext) + ); + assertThat(e.getMessage(), startsWith("Unknown language: " + locale)); + } + + public void testInvalidTimezone() { + String pattern = "YYYY"; + String timezone = "NON-EXISTING-TIMEZONE"; + DriverContext driverContext = driverContext(); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> evaluator( + new DateParse( + Source.EMPTY, + new Literal(Source.EMPTY, new BytesRef(pattern), DataType.KEYWORD), + field("str", DataType.KEYWORD), + new MapExpression( + Source.EMPTY, + List.of( + new Literal(Source.EMPTY, new BytesRef("time_zone"), DataType.KEYWORD), + new Literal(Source.EMPTY, new BytesRef(timezone), DataType.KEYWORD) + ) + ) + ) + ).get(driverContext) + ); + assertThat(e.getMessage(), startsWith("unsupported timezone [" + timezone + "]")); + } + @Override protected Expression build(Source source, List args) { - return new DateParse(source, args.get(0), args.size() > 1 ? args.get(1) : null); + return new DateParse(source, args.get(0), args.size() > 1 ? args.get(1) : null, args.size() == 3 ? args.get(2) : null); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java index ae30dce97ce5a..e45782dcf80ee 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java @@ -97,7 +97,7 @@ public void testBasicNullFolding() { assertNullLiteral(foldNull(new Round(EMPTY, Literal.NULL, null))); assertNullLiteral(foldNull(new Pow(EMPTY, Literal.NULL, Literal.NULL))); assertNullLiteral(foldNull(new DateFormat(EMPTY, Literal.NULL, Literal.NULL, null))); - assertNullLiteral(foldNull(new DateParse(EMPTY, Literal.NULL, Literal.NULL))); + assertNullLiteral(foldNull(new DateParse(EMPTY, Literal.NULL, Literal.NULL, NULL))); assertNullLiteral(foldNull(new DateTrunc(EMPTY, Literal.NULL, Literal.NULL))); assertNullLiteral(foldNull(new Substring(EMPTY, Literal.NULL, Literal.NULL, Literal.NULL))); }