diff --git a/docs/changelog/120143.yaml b/docs/changelog/120143.yaml new file mode 100644 index 0000000000000..7e8cd5a8ceaeb --- /dev/null +++ b/docs/changelog/120143.yaml @@ -0,0 +1,6 @@ +pr: 120143 +summary: Esql - support date nanos in date format function +area: ES|QL +type: enhancement +issues: + - 109994 diff --git a/docs/reference/esql/functions/kibana/definition/date_format.json b/docs/reference/esql/functions/kibana/definition/date_format.json index 629415da30fa2..f6f48e9df82b0 100644 --- a/docs/reference/esql/functions/kibana/definition/date_format.json +++ b/docs/reference/esql/functions/kibana/definition/date_format.json @@ -16,6 +16,18 @@ "variadic" : false, "returnType" : "keyword" }, + { + "params" : [ + { + "name" : "dateFormat", + "type" : "date_nanos", + "optional" : true, + "description" : "Date format (optional). If no format is specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, { "params" : [ { @@ -34,6 +46,24 @@ "variadic" : false, "returnType" : "keyword" }, + { + "params" : [ + { + "name" : "dateFormat", + "type" : "keyword", + "optional" : true, + "description" : "Date format (optional). If no format is specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used. If `null`, the function returns `null`." + }, + { + "name" : "date", + "type" : "date_nanos", + "optional" : false, + "description" : "Date expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, { "params" : [ { @@ -51,6 +81,24 @@ ], "variadic" : false, "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "dateFormat", + "type" : "text", + "optional" : true, + "description" : "Date format (optional). If no format is specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used. If `null`, the function returns `null`." + }, + { + "name" : "date", + "type" : "date_nanos", + "optional" : false, + "description" : "Date expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "keyword" } ], "examples" : [ diff --git a/docs/reference/esql/functions/kibana/definition/match_operator.json b/docs/reference/esql/functions/kibana/definition/match_operator.json index c8cbf1cf9d966..b58f9d5835a2d 100644 --- a/docs/reference/esql/functions/kibana/definition/match_operator.json +++ b/docs/reference/esql/functions/kibana/definition/match_operator.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "operator", "name" : "match_operator", - "description" : "Use `MATCH` to perform a <> on the specified field.\nUsing `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL.\n\nMatch can be used on text fields, as well as other field types like boolean, dates, and numeric types.\n\nFor a simplified syntax, you can use the <> `:` operator instead of `MATCH`.\n\n`MATCH` returns true if the provided query matches the row.", + "description" : "Use `MATCH` to perform a <> on the specified field.\nUsing `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL.\n\nMatch can be used on fields from the text family like <> and <>,\nas well as other field types like keyword, boolean, dates, and numeric types.\n\nFor a simplified syntax, you can use the <> `:` operator instead of `MATCH`.\n\n`MATCH` returns true if the provided query matches the row.", "signatures" : [ { "params" : [ diff --git a/docs/reference/esql/functions/kibana/docs/match_operator.md b/docs/reference/esql/functions/kibana/docs/match_operator.md index 7681c2d1ce231..98f55aacde0b8 100644 --- a/docs/reference/esql/functions/kibana/docs/match_operator.md +++ b/docs/reference/esql/functions/kibana/docs/match_operator.md @@ -6,7 +6,8 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ Use `MATCH` to perform a <> on the specified field. Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL. -Match can be used on text fields, as well as other field types like boolean, dates, and numeric types. +Match can be used on fields from the text family like <> and <>, +as well as other field types like keyword, boolean, dates, and numeric types. For a simplified syntax, you can use the <> `:` operator instead of `MATCH`. diff --git a/docs/reference/esql/functions/types/date_format.asciidoc b/docs/reference/esql/functions/types/date_format.asciidoc index 580094e9be906..c8f4942d98a62 100644 --- a/docs/reference/esql/functions/types/date_format.asciidoc +++ b/docs/reference/esql/functions/types/date_format.asciidoc @@ -6,6 +6,9 @@ |=== dateFormat | date | result date | | keyword +date_nanos | | keyword keyword | date | keyword +keyword | date_nanos | keyword text | date | keyword +text | date_nanos | keyword |=== diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec index 71a65f059528a..f44653171a4f5 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec @@ -1226,3 +1226,25 @@ FROM date_nanos millis:date | nanos:date_nanos | num:long 2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 ; + +Date Nanos Format +required_capability: date_nanos_date_format + +FROM date_nanos +| EVAL sv_nanos = MV_MAX(nanos) +| EVAL a = DATE_FORMAT(sv_nanos), b = DATE_FORMAT("yyyy-MM-dd", sv_nanos), c = DATE_FORMAT("strict_date_optional_time_nanos", sv_nanos) +| KEEP sv_nanos, a, b, c; +ignoreOrder:true + +sv_nanos:date_nanos | a:keyword | b:keyword | c:keyword +2023-10-23T13:55:01.543123456Z | 2023-10-23T13:55:01.543Z | 2023-10-23 | 2023-10-23T13:55:01.543123456Z +2023-10-23T13:53:55.832987654Z | 2023-10-23T13:53:55.832Z | 2023-10-23 | 2023-10-23T13:53:55.832987654Z +2023-10-23T13:52:55.015787878Z | 2023-10-23T13:52:55.015Z | 2023-10-23 | 2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732102837Z | 2023-10-23T13:51:54.732Z | 2023-10-23 | 2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937193000Z | 2023-10-23T13:33:34.937Z | 2023-10-23 | 2023-10-23T13:33:34.937193Z +2023-10-23T12:27:28.948000000Z | 2023-10-23T12:27:28.948Z | 2023-10-23 | 2023-10-23T12:27:28.948Z +2023-10-23T12:15:03.360103847Z | 2023-10-23T12:15:03.360Z | 2023-10-23 | 2023-10-23T12:15:03.360103847Z +2023-10-23T12:15:03.360103847Z | 2023-10-23T12:15:03.360Z | 2023-10-23 | 2023-10-23T12:15:03.360103847Z +2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23 | 2023-03-23T12:15:03.360103847Z +2023-03-23T12:15:03.360103847Z | 2023-03-23T12:15:03.360Z | 2023-03-23 | 2023-03-23T12:15:03.360103847Z +; diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatMillisConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatMillisConstantEvaluator.java new file mode 100644 index 0000000000000..2f41a7440bb06 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatMillisConstantEvaluator.java @@ -0,0 +1,132 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link DateFormat}. + * This class is generated. Do not edit it. + */ +public final class DateFormatMillisConstantEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator val; + + private final DateFormatter formatter; + + private final DriverContext driverContext; + + private Warnings warnings; + + public DateFormatMillisConstantEvaluator(Source source, EvalOperator.ExpressionEvaluator val, + DateFormatter formatter, DriverContext driverContext) { + this.source = source; + this.val = val; + this.formatter = formatter; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock valBlock = (LongBlock) val.eval(page)) { + LongVector valVector = valBlock.asVector(); + if (valVector == null) { + return eval(page.getPositionCount(), valBlock); + } + return eval(page.getPositionCount(), valVector).asBlock(); + } + } + + public BytesRefBlock eval(int positionCount, LongBlock valBlock) { + try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (valBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (valBlock.getValueCount(p) != 1) { + if (valBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendBytesRef(DateFormat.processMillis(valBlock.getLong(valBlock.getFirstValueIndex(p)), this.formatter)); + } + return result.build(); + } + } + + public BytesRefVector eval(int positionCount, LongVector valVector) { + try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + result.appendBytesRef(DateFormat.processMillis(valVector.getLong(p), this.formatter)); + } + return result.build(); + } + } + + @Override + public String toString() { + return "DateFormatMillisConstantEvaluator[" + "val=" + val + ", formatter=" + formatter + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(val); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory val; + + private final DateFormatter formatter; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory val, + DateFormatter formatter) { + this.source = source; + this.val = val; + this.formatter = formatter; + } + + @Override + public DateFormatMillisConstantEvaluator get(DriverContext context) { + return new DateFormatMillisConstantEvaluator(source, val.get(context), formatter, context); + } + + @Override + public String toString() { + return "DateFormatMillisConstantEvaluator[" + "val=" + val + ", formatter=" + formatter + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatMillisEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatMillisEvaluator.java new file mode 100644 index 0000000000000..29da191dbe781 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatMillisEvaluator.java @@ -0,0 +1,159 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import java.util.Locale; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link DateFormat}. + * This class is generated. Do not edit it. + */ +public final class DateFormatMillisEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator val; + + private final EvalOperator.ExpressionEvaluator formatter; + + private final Locale locale; + + private final DriverContext driverContext; + + private Warnings warnings; + + public DateFormatMillisEvaluator(Source source, EvalOperator.ExpressionEvaluator val, + EvalOperator.ExpressionEvaluator formatter, Locale locale, DriverContext driverContext) { + this.source = source; + this.val = val; + this.formatter = formatter; + this.locale = locale; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock valBlock = (LongBlock) val.eval(page)) { + try (BytesRefBlock formatterBlock = (BytesRefBlock) formatter.eval(page)) { + LongVector valVector = valBlock.asVector(); + if (valVector == null) { + return eval(page.getPositionCount(), valBlock, formatterBlock); + } + BytesRefVector formatterVector = formatterBlock.asVector(); + if (formatterVector == null) { + return eval(page.getPositionCount(), valBlock, formatterBlock); + } + return eval(page.getPositionCount(), valVector, formatterVector).asBlock(); + } + } + } + + public BytesRefBlock eval(int positionCount, LongBlock valBlock, BytesRefBlock formatterBlock) { + try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + BytesRef formatterScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (valBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (valBlock.getValueCount(p) != 1) { + if (valBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (formatterBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (formatterBlock.getValueCount(p) != 1) { + if (formatterBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendBytesRef(DateFormat.processMillis(valBlock.getLong(valBlock.getFirstValueIndex(p)), formatterBlock.getBytesRef(formatterBlock.getFirstValueIndex(p), formatterScratch), this.locale)); + } + return result.build(); + } + } + + public BytesRefVector eval(int positionCount, LongVector valVector, + BytesRefVector formatterVector) { + try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) { + BytesRef formatterScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + result.appendBytesRef(DateFormat.processMillis(valVector.getLong(p), formatterVector.getBytesRef(p, formatterScratch), this.locale)); + } + return result.build(); + } + } + + @Override + public String toString() { + return "DateFormatMillisEvaluator[" + "val=" + val + ", formatter=" + formatter + ", locale=" + locale + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(val, formatter); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory val; + + private final EvalOperator.ExpressionEvaluator.Factory formatter; + + private final Locale locale; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory val, + EvalOperator.ExpressionEvaluator.Factory formatter, Locale locale) { + this.source = source; + this.val = val; + this.formatter = formatter; + this.locale = locale; + } + + @Override + public DateFormatMillisEvaluator get(DriverContext context) { + return new DateFormatMillisEvaluator(source, val.get(context), formatter.get(context), locale, context); + } + + @Override + public String toString() { + return "DateFormatMillisEvaluator[" + "val=" + val + ", formatter=" + formatter + ", locale=" + locale + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatConstantEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatNanosConstantEvaluator.java similarity index 83% rename from x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatConstantEvaluator.java rename to x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatNanosConstantEvaluator.java index 25afa6bec360b..1488833227dcb 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatConstantEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatNanosConstantEvaluator.java @@ -24,7 +24,7 @@ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link DateFormat}. * This class is generated. Do not edit it. */ -public final class DateFormatConstantEvaluator implements EvalOperator.ExpressionEvaluator { +public final class DateFormatNanosConstantEvaluator implements EvalOperator.ExpressionEvaluator { private final Source source; private final EvalOperator.ExpressionEvaluator val; @@ -35,7 +35,7 @@ public final class DateFormatConstantEvaluator implements EvalOperator.Expressio private Warnings warnings; - public DateFormatConstantEvaluator(Source source, EvalOperator.ExpressionEvaluator val, + public DateFormatNanosConstantEvaluator(Source source, EvalOperator.ExpressionEvaluator val, DateFormatter formatter, DriverContext driverContext) { this.source = source; this.val = val; @@ -68,7 +68,7 @@ public BytesRefBlock eval(int positionCount, LongBlock valBlock) { result.appendNull(); continue position; } - result.appendBytesRef(DateFormat.process(valBlock.getLong(valBlock.getFirstValueIndex(p)), this.formatter)); + result.appendBytesRef(DateFormat.processNanos(valBlock.getLong(valBlock.getFirstValueIndex(p)), this.formatter)); } return result.build(); } @@ -77,7 +77,7 @@ public BytesRefBlock eval(int positionCount, LongBlock valBlock) { public BytesRefVector eval(int positionCount, LongVector valVector) { try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) { position: for (int p = 0; p < positionCount; p++) { - result.appendBytesRef(DateFormat.process(valVector.getLong(p), this.formatter)); + result.appendBytesRef(DateFormat.processNanos(valVector.getLong(p), this.formatter)); } return result.build(); } @@ -85,7 +85,7 @@ public BytesRefVector eval(int positionCount, LongVector valVector) { @Override public String toString() { - return "DateFormatConstantEvaluator[" + "val=" + val + ", formatter=" + formatter + "]"; + return "DateFormatNanosConstantEvaluator[" + "val=" + val + ", formatter=" + formatter + "]"; } @Override @@ -120,13 +120,13 @@ public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory val, } @Override - public DateFormatConstantEvaluator get(DriverContext context) { - return new DateFormatConstantEvaluator(source, val.get(context), formatter, context); + public DateFormatNanosConstantEvaluator get(DriverContext context) { + return new DateFormatNanosConstantEvaluator(source, val.get(context), formatter, context); } @Override public String toString() { - return "DateFormatConstantEvaluator[" + "val=" + val + ", formatter=" + formatter + "]"; + return "DateFormatNanosConstantEvaluator[" + "val=" + val + ", formatter=" + formatter + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatNanosEvaluator.java similarity index 84% rename from x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatEvaluator.java rename to x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatNanosEvaluator.java index 318ffa5af8f77..a94d522014813 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatNanosEvaluator.java @@ -25,7 +25,7 @@ * {@link EvalOperator.ExpressionEvaluator} implementation for {@link DateFormat}. * This class is generated. Do not edit it. */ -public final class DateFormatEvaluator implements EvalOperator.ExpressionEvaluator { +public final class DateFormatNanosEvaluator implements EvalOperator.ExpressionEvaluator { private final Source source; private final EvalOperator.ExpressionEvaluator val; @@ -38,7 +38,7 @@ public final class DateFormatEvaluator implements EvalOperator.ExpressionEvaluat private Warnings warnings; - public DateFormatEvaluator(Source source, EvalOperator.ExpressionEvaluator val, + public DateFormatNanosEvaluator(Source source, EvalOperator.ExpressionEvaluator val, EvalOperator.ExpressionEvaluator formatter, Locale locale, DriverContext driverContext) { this.source = source; this.val = val; @@ -90,7 +90,7 @@ public BytesRefBlock eval(int positionCount, LongBlock valBlock, BytesRefBlock f result.appendNull(); continue position; } - result.appendBytesRef(DateFormat.process(valBlock.getLong(valBlock.getFirstValueIndex(p)), formatterBlock.getBytesRef(formatterBlock.getFirstValueIndex(p), formatterScratch), this.locale)); + result.appendBytesRef(DateFormat.processNanos(valBlock.getLong(valBlock.getFirstValueIndex(p)), formatterBlock.getBytesRef(formatterBlock.getFirstValueIndex(p), formatterScratch), this.locale)); } return result.build(); } @@ -101,7 +101,7 @@ public BytesRefVector eval(int positionCount, LongVector valVector, try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) { BytesRef formatterScratch = new BytesRef(); position: for (int p = 0; p < positionCount; p++) { - result.appendBytesRef(DateFormat.process(valVector.getLong(p), formatterVector.getBytesRef(p, formatterScratch), this.locale)); + result.appendBytesRef(DateFormat.processNanos(valVector.getLong(p), formatterVector.getBytesRef(p, formatterScratch), this.locale)); } return result.build(); } @@ -109,7 +109,7 @@ public BytesRefVector eval(int positionCount, LongVector valVector, @Override public String toString() { - return "DateFormatEvaluator[" + "val=" + val + ", formatter=" + formatter + ", locale=" + locale + "]"; + return "DateFormatNanosEvaluator[" + "val=" + val + ", formatter=" + formatter + ", locale=" + locale + "]"; } @Override @@ -147,13 +147,13 @@ public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory val, } @Override - public DateFormatEvaluator get(DriverContext context) { - return new DateFormatEvaluator(source, val.get(context), formatter.get(context), locale, context); + public DateFormatNanosEvaluator get(DriverContext context) { + return new DateFormatNanosEvaluator(source, val.get(context), formatter.get(context), locale, context); } @Override public String toString() { - return "DateFormatEvaluator[" + "val=" + val + ", formatter=" + formatter + ", locale=" + locale + "]"; + return "DateFormatNanosEvaluator[" + "val=" + val + ", formatter=" + formatter + ", locale=" + locale + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 4531532fac3c5..6a618414641db 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -384,6 +384,10 @@ public enum Cap { * Support the {@link org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In} operator for date nanos */ DATE_NANOS_IN_OPERATOR(), + /** + * Support running date format function on nanosecond dates + */ + DATE_NANOS_DATE_FORMAT(), /** * DATE_PARSE supports reading timezones diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java index 29648d55cadd8..d30e99794a44e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java @@ -14,8 +14,10 @@ import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; +import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -33,10 +35,11 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isDate; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isStringAndExact; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.nanoTimeToString; public class DateFormat extends EsqlConfigurationFunction implements OptionalArgument { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( @@ -55,10 +58,14 @@ public class DateFormat extends EsqlConfigurationFunction implements OptionalArg ) public DateFormat( Source source, - @Param(optional = true, name = "dateFormat", type = { "keyword", "text", "date" }, description = """ + @Param(optional = true, name = "dateFormat", type = { "keyword", "text", "date", "date_nanos" }, description = """ Date format (optional). If no format is specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used. If `null`, the function returns `null`.""") Expression format, - @Param(name = "date", type = { "date" }, description = "Date expression. If `null`, the function returns `null`.") Expression date, + @Param( + name = "date", + type = { "date", "date_nanos" }, + description = "Date expression. If `null`, the function returns `null`." + ) Expression date, Configuration configuration ) { super(source, date != null ? List.of(format, date) : List.of(format), configuration); @@ -114,7 +121,9 @@ protected TypeResolution resolveType() { } } - resolution = isDate(field, sourceText(), format == null ? FIRST : SECOND); + String operationName = sourceText(); + TypeResolutions.ParamOrdinal paramOrd = format == null ? FIRST : SECOND; + resolution = TypeResolutions.isType(field, DataType::isDate, operationName, paramOrd, "datetime or date_nanos"); if (resolution.unresolved()) { return resolution; } @@ -127,31 +136,63 @@ public boolean foldable() { return field.foldable() && (format == null || format.foldable()); } - @Evaluator(extraName = "Constant") - static BytesRef process(long val, @Fixed DateFormatter formatter) { + @Evaluator(extraName = "MillisConstant") + static BytesRef processMillis(long val, @Fixed DateFormatter formatter) { return new BytesRef(dateTimeToString(val, formatter)); } - @Evaluator - static BytesRef process(long val, BytesRef formatter, @Fixed Locale locale) { + @Evaluator(extraName = "Millis") + static BytesRef processMillis(long val, BytesRef formatter, @Fixed Locale locale) { return new BytesRef(dateTimeToString(val, toFormatter(formatter, locale))); } + @Evaluator(extraName = "NanosConstant") + static BytesRef processNanos(long val, @Fixed DateFormatter formatter) { + return new BytesRef(nanoTimeToString(val, formatter)); + } + + @Evaluator(extraName = "Nanos") + static BytesRef processNanos(long val, BytesRef formatter, @Fixed Locale locale) { + return new BytesRef(nanoTimeToString(val, toFormatter(formatter, locale))); + } + + private ExpressionEvaluator.Factory getConstantEvaluator( + DataType dateType, + EvalOperator.ExpressionEvaluator.Factory fieldEvaluator, + DateFormatter formatter + ) { + if (dateType == DATE_NANOS) { + return new DateFormatNanosConstantEvaluator.Factory(source(), fieldEvaluator, formatter); + } + return new DateFormatMillisConstantEvaluator.Factory(source(), fieldEvaluator, formatter); + } + + private ExpressionEvaluator.Factory getEvaluator( + DataType dateType, + EvalOperator.ExpressionEvaluator.Factory fieldEvaluator, + EvalOperator.ExpressionEvaluator.Factory formatEvaluator + ) { + if (dateType == DATE_NANOS) { + return new DateFormatNanosEvaluator.Factory(source(), fieldEvaluator, formatEvaluator, configuration().locale()); + } + return new DateFormatMillisEvaluator.Factory(source(), fieldEvaluator, formatEvaluator, configuration().locale()); + } + @Override public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var fieldEvaluator = toEvaluator.apply(field); if (format == null) { - return new DateFormatConstantEvaluator.Factory(source(), fieldEvaluator, DEFAULT_DATE_TIME_FORMATTER); + return getConstantEvaluator(field().dataType(), fieldEvaluator, DEFAULT_DATE_TIME_FORMATTER); } if (DataType.isString(format.dataType()) == false) { throw new IllegalArgumentException("unsupported data type for format [" + format.dataType() + "]"); } if (format.foldable()) { DateFormatter formatter = toFormatter(format.fold(toEvaluator.foldCtx()), configuration().locale()); - return new DateFormatConstantEvaluator.Factory(source(), fieldEvaluator, formatter); + return getConstantEvaluator(field.dataType(), fieldEvaluator, formatter); } var formatEvaluator = toEvaluator.apply(format); - return new DateFormatEvaluator.Factory(source(), fieldEvaluator, formatEvaluator, configuration().locale()); + return getEvaluator(field().dataType(), fieldEvaluator, formatEvaluator); } private static DateFormatter toFormatter(Object format, Locale locale) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java index 95ee6ab337bd6..eef0df6b89dd3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -539,6 +539,10 @@ public static String dateTimeToString(long dateTime, DateFormatter formatter) { return formatter == null ? dateTimeToString(dateTime) : formatter.formatMillis(dateTime); } + public static String nanoTimeToString(long dateTime, DateFormatter formatter) { + return formatter == null ? nanoTimeToString(dateTime) : formatter.formatNanos(dateTime); + } + public static BytesRef numericBooleanToString(Object field) { return new BytesRef(String.valueOf(field)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 2df6e30e96081..4ae0784cf7916 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -1198,21 +1198,21 @@ public void testDateFormatOnInt() { verifyUnsupported(""" from test | eval date_format(int) - """, "first argument of [date_format(int)] must be [datetime], found value [int] type [integer]"); + """, "first argument of [date_format(int)] must be [datetime or date_nanos], found value [int] type [integer]"); } public void testDateFormatOnFloat() { verifyUnsupported(""" from test | eval date_format(float) - """, "first argument of [date_format(float)] must be [datetime], found value [float] type [double]"); + """, "first argument of [date_format(float)] must be [datetime or date_nanos], found value [float] type [double]"); } public void testDateFormatOnText() { verifyUnsupported(""" from test | eval date_format(keyword) - """, "first argument of [date_format(keyword)] must be [datetime], found value [keyword] type [keyword]"); + """, "first argument of [date_format(keyword)] must be [datetime or date_nanos], found value [keyword] type [keyword]"); } public void testDateFormatWithNumericFormat() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java index a5e6514b3e02c..c7eebdd82be53 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java @@ -37,7 +37,7 @@ protected Matcher expectedTypeErrorMatcher(List> validPerP String source = sourceForSignature(signature); String name = signature.get(0).typeName(); if (signature.size() == 1) { - return equalTo("first argument of [" + source + "] must be [datetime], found value [] type [" + name + "]"); + return equalTo("first argument of [" + source + "] must be [datetime or date_nanos], found value [] type [" + name + "]"); } // Two argument version // Handle the weird case where we're calling the two argument version with the date first instead of the format. @@ -46,7 +46,7 @@ protected Matcher expectedTypeErrorMatcher(List> validPerP } return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { case 0 -> "string"; - case 1 -> "datetime"; + case 1 -> "datetime or date_nanos"; default -> ""; })); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java index 3dd1f3e629da4..1167b91a81e35 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -44,7 +45,24 @@ public static Iterable parameters() { DataType.KEYWORD, TestCaseSupplier.dateFormatCases(), TestCaseSupplier.dateCases(Instant.parse("1900-01-01T00:00:00.00Z"), Instant.parse("9999-12-31T00:00:00.00Z")), - matchesPattern("DateFormatEvaluator\\[val=Attribute\\[channel=1], formatter=Attribute\\[(channel=0|\\w+)], locale=en_US]"), + matchesPattern( + "DateFormatMillisEvaluator\\[val=Attribute\\[channel=1], formatter=Attribute\\[(channel=0|\\w+)], locale=en_US]" + ), + (lhs, rhs) -> List.of(), + false + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + (format, value) -> new BytesRef( + DateFormatter.forPattern(((BytesRef) format).utf8ToString()).formatNanos(DateUtils.toLong((Instant) value)) + ), + DataType.KEYWORD, + TestCaseSupplier.dateFormatCases(), + TestCaseSupplier.dateNanosCases(), + matchesPattern( + "DateFormatNanosEvaluator\\[val=Attribute\\[channel=1], formatter=Attribute\\[(channel=0|\\w+)], locale=en_US]" + ), (lhs, rhs) -> List.of(), false ) @@ -52,12 +70,20 @@ public static Iterable parameters() { // Default formatter cases TestCaseSupplier.unary( suppliers, - "DateFormatConstantEvaluator[val=Attribute[channel=0], formatter=format[strict_date_optional_time] locale[]]", + "DateFormatMillisConstantEvaluator[val=Attribute[channel=0], formatter=format[strict_date_optional_time] locale[]]", TestCaseSupplier.dateCases(Instant.parse("1900-01-01T00:00:00.00Z"), Instant.parse("9999-12-31T00:00:00.00Z")), DataType.KEYWORD, (value) -> new BytesRef(EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER.formatMillis(((Instant) value).toEpochMilli())), List.of() ); + TestCaseSupplier.unary( + suppliers, + "DateFormatNanosConstantEvaluator[val=Attribute[channel=0], formatter=format[strict_date_optional_time] locale[]]", + TestCaseSupplier.dateNanosCases(), + DataType.KEYWORD, + (value) -> new BytesRef(EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER.formatNanos(DateUtils.toLong((Instant) value))), + List.of() + ); return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); }