diff --git a/docs/changelog/120727.yaml b/docs/changelog/120727.yaml new file mode 100644 index 0000000000000..4d0241e6baad1 --- /dev/null +++ b/docs/changelog/120727.yaml @@ -0,0 +1,6 @@ +pr: 120727 +summary: Esql - Support date nanos in date extract function +area: ES|QL +type: enhancement +issues: + - 110000 diff --git a/docs/reference/esql/functions/kibana/definition/date_extract.json b/docs/reference/esql/functions/kibana/definition/date_extract.json index c6dc6583f324d..0ababf80d9137 100644 --- a/docs/reference/esql/functions/kibana/definition/date_extract.json +++ b/docs/reference/esql/functions/kibana/definition/date_extract.json @@ -22,6 +22,24 @@ "variadic" : false, "returnType" : "long" }, + { + "params" : [ + { + "name" : "datePart", + "type" : "keyword", + "optional" : false, + "description" : "Part of the date to extract. Can be: `aligned_day_of_week_in_month`, `aligned_day_of_week_in_year`, `aligned_week_of_month`, `aligned_week_of_year`, `ampm_of_day`, `clock_hour_of_ampm`, `clock_hour_of_day`, `day_of_month`, `day_of_week`, `day_of_year`, `epoch_day`, `era`, `hour_of_ampm`, `hour_of_day`, `instant_seconds`, `micro_of_day`, `micro_of_second`, `milli_of_day`, `milli_of_second`, `minute_of_day`, `minute_of_hour`, `month_of_year`, `nano_of_day`, `nano_of_second`, `offset_seconds`, `proleptic_month`, `second_of_day`, `second_of_minute`, `year`, or `year_of_era`. Refer to https://docs.oracle.com/javase/8/docs/api/java/time/temporal/ChronoField.html[java.time.temporal.ChronoField] for a description of these values. If `null`, the function returns `null`." + }, + { + "name" : "date", + "type" : "date_nanos", + "optional" : false, + "description" : "Date expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "long" + }, { "params" : [ { @@ -39,6 +57,24 @@ ], "variadic" : false, "returnType" : "long" + }, + { + "params" : [ + { + "name" : "datePart", + "type" : "text", + "optional" : false, + "description" : "Part of the date to extract. Can be: `aligned_day_of_week_in_month`, `aligned_day_of_week_in_year`, `aligned_week_of_month`, `aligned_week_of_year`, `ampm_of_day`, `clock_hour_of_ampm`, `clock_hour_of_day`, `day_of_month`, `day_of_week`, `day_of_year`, `epoch_day`, `era`, `hour_of_ampm`, `hour_of_day`, `instant_seconds`, `micro_of_day`, `micro_of_second`, `milli_of_day`, `milli_of_second`, `minute_of_day`, `minute_of_hour`, `month_of_year`, `nano_of_day`, `nano_of_second`, `offset_seconds`, `proleptic_month`, `second_of_day`, `second_of_minute`, `year`, or `year_of_era`. Refer to https://docs.oracle.com/javase/8/docs/api/java/time/temporal/ChronoField.html[java.time.temporal.ChronoField] for a description of these values. If `null`, the function returns `null`." + }, + { + "name" : "date", + "type" : "date_nanos", + "optional" : false, + "description" : "Date expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "long" } ], "examples" : [ diff --git a/docs/reference/esql/functions/types/date_extract.asciidoc b/docs/reference/esql/functions/types/date_extract.asciidoc index ec9bf70c221cc..207e09b00f786 100644 --- a/docs/reference/esql/functions/types/date_extract.asciidoc +++ b/docs/reference/esql/functions/types/date_extract.asciidoc @@ -6,5 +6,7 @@ |=== datePart | date | result keyword | date | long +keyword | date_nanos | long text | date | long +text | date_nanos | long |=== diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec index 1f4e555bd5d83..ec68e7eecc658 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec @@ -500,6 +500,27 @@ millis:date | nanos:date_nanos | num:long 2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 ; +Date nanos date extract +required_capability: date_nanos_date_extract + +FROM date_nanos +| EVAL nn = MV_MAX(nanos) +| EVAL year = DATE_EXTRACT("year", nn), ns = DATE_EXTRACT("nano_of_second", nn) +| KEEP nn, year, ns; + +nn:date_nanos | year:long | ns:long +2023-10-23T13:55:01.543123456Z | 2023 | 543123456 +2023-10-23T13:53:55.832987654Z | 2023 | 832987654 +2023-10-23T13:52:55.015787878Z | 2023 | 015787878 +2023-10-23T13:51:54.732102837Z | 2023 | 732102837 +2023-10-23T13:33:34.937193000Z | 2023 | 937193000 +2023-10-23T12:27:28.948000000Z | 2023 | 948000000 +2023-10-23T12:15:03.360103847Z | 2023 | 360103847 +2023-10-23T12:15:03.360103847Z | 2023 | 360103847 +2023-03-23T12:15:03.360103847Z | 2023 | 360103847 +2023-03-23T12:15:03.360103847Z | 2023 | 360103847 +; + date nanos to long, index version required_capability: to_date_nanos diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantMillisEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantMillisEvaluator.java new file mode 100644 index 0000000000000..11da518a01ce1 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantMillisEvaluator.java @@ -0,0 +1,137 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import java.time.ZoneId; +import java.time.temporal.ChronoField; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link DateExtract}. + * This class is generated. Do not edit it. + */ +public final class DateExtractConstantMillisEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator value; + + private final ChronoField chronoField; + + private final ZoneId zone; + + private final DriverContext driverContext; + + private Warnings warnings; + + public DateExtractConstantMillisEvaluator(Source source, EvalOperator.ExpressionEvaluator value, + ChronoField chronoField, ZoneId zone, DriverContext driverContext) { + this.source = source; + this.value = value; + this.chronoField = chronoField; + this.zone = zone; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock valueBlock = (LongBlock) value.eval(page)) { + LongVector valueVector = valueBlock.asVector(); + if (valueVector == null) { + return eval(page.getPositionCount(), valueBlock); + } + return eval(page.getPositionCount(), valueVector).asBlock(); + } + } + + public LongBlock eval(int positionCount, LongBlock valueBlock) { + try(LongBlock.Builder result = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (valueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (valueBlock.getValueCount(p) != 1) { + if (valueBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendLong(DateExtract.processMillis(valueBlock.getLong(valueBlock.getFirstValueIndex(p)), this.chronoField, this.zone)); + } + return result.build(); + } + } + + public LongVector eval(int positionCount, LongVector valueVector) { + try(LongVector.FixedBuilder result = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + result.appendLong(p, DateExtract.processMillis(valueVector.getLong(p), this.chronoField, this.zone)); + } + return result.build(); + } + } + + @Override + public String toString() { + return "DateExtractConstantMillisEvaluator[" + "value=" + value + ", chronoField=" + chronoField + ", zone=" + zone + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(value); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory value; + + private final ChronoField chronoField; + + private final ZoneId zone; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory value, + ChronoField chronoField, ZoneId zone) { + this.source = source; + this.value = value; + this.chronoField = chronoField; + this.zone = zone; + } + + @Override + public DateExtractConstantMillisEvaluator get(DriverContext context) { + return new DateExtractConstantMillisEvaluator(source, value.get(context), chronoField, zone, context); + } + + @Override + public String toString() { + return "DateExtractConstantMillisEvaluator[" + "value=" + value + ", chronoField=" + chronoField + ", zone=" + zone + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantNanosEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantNanosEvaluator.java new file mode 100644 index 0000000000000..bbd0a59c87ceb --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantNanosEvaluator.java @@ -0,0 +1,137 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import java.time.ZoneId; +import java.time.temporal.ChronoField; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link DateExtract}. + * This class is generated. Do not edit it. + */ +public final class DateExtractConstantNanosEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator value; + + private final ChronoField chronoField; + + private final ZoneId zone; + + private final DriverContext driverContext; + + private Warnings warnings; + + public DateExtractConstantNanosEvaluator(Source source, EvalOperator.ExpressionEvaluator value, + ChronoField chronoField, ZoneId zone, DriverContext driverContext) { + this.source = source; + this.value = value; + this.chronoField = chronoField; + this.zone = zone; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock valueBlock = (LongBlock) value.eval(page)) { + LongVector valueVector = valueBlock.asVector(); + if (valueVector == null) { + return eval(page.getPositionCount(), valueBlock); + } + return eval(page.getPositionCount(), valueVector).asBlock(); + } + } + + public LongBlock eval(int positionCount, LongBlock valueBlock) { + try(LongBlock.Builder result = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + if (valueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (valueBlock.getValueCount(p) != 1) { + if (valueBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendLong(DateExtract.processNanos(valueBlock.getLong(valueBlock.getFirstValueIndex(p)), this.chronoField, this.zone)); + } + return result.build(); + } + } + + public LongVector eval(int positionCount, LongVector valueVector) { + try(LongVector.FixedBuilder result = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + result.appendLong(p, DateExtract.processNanos(valueVector.getLong(p), this.chronoField, this.zone)); + } + return result.build(); + } + } + + @Override + public String toString() { + return "DateExtractConstantNanosEvaluator[" + "value=" + value + ", chronoField=" + chronoField + ", zone=" + zone + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(value); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory value; + + private final ChronoField chronoField; + + private final ZoneId zone; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory value, + ChronoField chronoField, ZoneId zone) { + this.source = source; + this.value = value; + this.chronoField = chronoField; + this.zone = zone; + } + + @Override + public DateExtractConstantNanosEvaluator get(DriverContext context) { + return new DateExtractConstantNanosEvaluator(source, value.get(context), chronoField, zone, context); + } + + @Override + public String toString() { + return "DateExtractConstantNanosEvaluator[" + "value=" + value + ", chronoField=" + chronoField + ", zone=" + zone + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractMillisEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractMillisEvaluator.java new file mode 100644 index 0000000000000..edc0b2cb0f0ce --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractMillisEvaluator.java @@ -0,0 +1,169 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import java.time.ZoneId; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link DateExtract}. + * This class is generated. Do not edit it. + */ +public final class DateExtractMillisEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator value; + + private final EvalOperator.ExpressionEvaluator chronoField; + + private final ZoneId zone; + + private final DriverContext driverContext; + + private Warnings warnings; + + public DateExtractMillisEvaluator(Source source, EvalOperator.ExpressionEvaluator value, + EvalOperator.ExpressionEvaluator chronoField, ZoneId zone, DriverContext driverContext) { + this.source = source; + this.value = value; + this.chronoField = chronoField; + this.zone = zone; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock valueBlock = (LongBlock) value.eval(page)) { + try (BytesRefBlock chronoFieldBlock = (BytesRefBlock) chronoField.eval(page)) { + LongVector valueVector = valueBlock.asVector(); + if (valueVector == null) { + return eval(page.getPositionCount(), valueBlock, chronoFieldBlock); + } + BytesRefVector chronoFieldVector = chronoFieldBlock.asVector(); + if (chronoFieldVector == null) { + return eval(page.getPositionCount(), valueBlock, chronoFieldBlock); + } + return eval(page.getPositionCount(), valueVector, chronoFieldVector); + } + } + } + + public LongBlock eval(int positionCount, LongBlock valueBlock, BytesRefBlock chronoFieldBlock) { + try(LongBlock.Builder result = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + BytesRef chronoFieldScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (valueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (valueBlock.getValueCount(p) != 1) { + if (valueBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (chronoFieldBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (chronoFieldBlock.getValueCount(p) != 1) { + if (chronoFieldBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + try { + result.appendLong(DateExtract.processMillis(valueBlock.getLong(valueBlock.getFirstValueIndex(p)), chronoFieldBlock.getBytesRef(chronoFieldBlock.getFirstValueIndex(p), chronoFieldScratch), this.zone)); + } catch (IllegalArgumentException e) { + warnings().registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + public LongBlock eval(int positionCount, LongVector valueVector, + BytesRefVector chronoFieldVector) { + try(LongBlock.Builder result = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + BytesRef chronoFieldScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + try { + result.appendLong(DateExtract.processMillis(valueVector.getLong(p), chronoFieldVector.getBytesRef(p, chronoFieldScratch), this.zone)); + } catch (IllegalArgumentException e) { + warnings().registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + @Override + public String toString() { + return "DateExtractMillisEvaluator[" + "value=" + value + ", chronoField=" + chronoField + ", zone=" + zone + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(value, chronoField); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory value; + + private final EvalOperator.ExpressionEvaluator.Factory chronoField; + + private final ZoneId zone; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory value, + EvalOperator.ExpressionEvaluator.Factory chronoField, ZoneId zone) { + this.source = source; + this.value = value; + this.chronoField = chronoField; + this.zone = zone; + } + + @Override + public DateExtractMillisEvaluator get(DriverContext context) { + return new DateExtractMillisEvaluator(source, value.get(context), chronoField.get(context), zone, context); + } + + @Override + public String toString() { + return "DateExtractMillisEvaluator[" + "value=" + value + ", chronoField=" + chronoField + ", zone=" + zone + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractNanosEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractNanosEvaluator.java new file mode 100644 index 0000000000000..97a04f0d06a74 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractNanosEvaluator.java @@ -0,0 +1,169 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import java.time.ZoneId; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link DateExtract}. + * This class is generated. Do not edit it. + */ +public final class DateExtractNanosEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator value; + + private final EvalOperator.ExpressionEvaluator chronoField; + + private final ZoneId zone; + + private final DriverContext driverContext; + + private Warnings warnings; + + public DateExtractNanosEvaluator(Source source, EvalOperator.ExpressionEvaluator value, + EvalOperator.ExpressionEvaluator chronoField, ZoneId zone, DriverContext driverContext) { + this.source = source; + this.value = value; + this.chronoField = chronoField; + this.zone = zone; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock valueBlock = (LongBlock) value.eval(page)) { + try (BytesRefBlock chronoFieldBlock = (BytesRefBlock) chronoField.eval(page)) { + LongVector valueVector = valueBlock.asVector(); + if (valueVector == null) { + return eval(page.getPositionCount(), valueBlock, chronoFieldBlock); + } + BytesRefVector chronoFieldVector = chronoFieldBlock.asVector(); + if (chronoFieldVector == null) { + return eval(page.getPositionCount(), valueBlock, chronoFieldBlock); + } + return eval(page.getPositionCount(), valueVector, chronoFieldVector); + } + } + } + + public LongBlock eval(int positionCount, LongBlock valueBlock, BytesRefBlock chronoFieldBlock) { + try(LongBlock.Builder result = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + BytesRef chronoFieldScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (valueBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (valueBlock.getValueCount(p) != 1) { + if (valueBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (chronoFieldBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (chronoFieldBlock.getValueCount(p) != 1) { + if (chronoFieldBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + try { + result.appendLong(DateExtract.processNanos(valueBlock.getLong(valueBlock.getFirstValueIndex(p)), chronoFieldBlock.getBytesRef(chronoFieldBlock.getFirstValueIndex(p), chronoFieldScratch), this.zone)); + } catch (IllegalArgumentException e) { + warnings().registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + public LongBlock eval(int positionCount, LongVector valueVector, + BytesRefVector chronoFieldVector) { + try(LongBlock.Builder result = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + BytesRef chronoFieldScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + try { + result.appendLong(DateExtract.processNanos(valueVector.getLong(p), chronoFieldVector.getBytesRef(p, chronoFieldScratch), this.zone)); + } catch (IllegalArgumentException e) { + warnings().registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + @Override + public String toString() { + return "DateExtractNanosEvaluator[" + "value=" + value + ", chronoField=" + chronoField + ", zone=" + zone + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(value, chronoField); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory value; + + private final EvalOperator.ExpressionEvaluator.Factory chronoField; + + private final ZoneId zone; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory value, + EvalOperator.ExpressionEvaluator.Factory chronoField, ZoneId zone) { + this.source = source; + this.value = value; + this.chronoField = chronoField; + this.zone = zone; + } + + @Override + public DateExtractNanosEvaluator get(DriverContext context) { + return new DateExtractNanosEvaluator(source, value.get(context), chronoField.get(context), zone, context); + } + + @Override + public String toString() { + return "DateExtractNanosEvaluator[" + "value=" + value + ", chronoField=" + chronoField + ", zone=" + zone + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 182328b54c4c5..ef46d71ac1de1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -475,7 +475,10 @@ public enum Cap { * Support Least and Greatest functions on Date Nanos type */ LEAST_GREATEST_FOR_DATENANOS(), - + /** + * support date extract function for date nanos + */ + DATE_NANOS_DATE_EXTRACT(), /** * Support add and subtract on date nanos */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java index 7fc5d82441802..20ff398803854 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java @@ -33,10 +33,10 @@ import java.time.temporal.ChronoField; import java.util.List; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isDate; import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isStringAndExact; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.EsqlConverter.STRING_TO_CHRONO_FIELD; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.chronoToLong; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.chronoToLongNanos; public class DateExtract extends EsqlConfigurationFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( @@ -72,7 +72,11 @@ public DateExtract( Refer to https://docs.oracle.com/javase/8/docs/api/java/time/temporal/ChronoField.html[java.time.temporal.ChronoField] for a description of these values.\n If `null`, the function returns `null`.""") Expression chronoFieldExp, - @Param(name = "date", type = "date", description = "Date expression. If `null`, the function returns `null`.") Expression field, + @Param( + name = "date", + type = { "date", "date_nanos" }, + description = "Date expression. If `null`, the function returns `null`." + ) Expression field, Configuration configuration ) { super(source, List.of(chronoFieldExp, field), configuration); @@ -109,17 +113,42 @@ public String getWriteableName() { @Override public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { - var fieldEvaluator = toEvaluator.apply(children().get(1)); + boolean isNanos = switch (field().dataType()) { + case DataType.DATETIME -> false; + case DataType.DATE_NANOS -> true; + default -> throw new UnsupportedOperationException( + "Unsupported field type [" + + field().dataType().name() + + "]. " + + "If you're seeing this, there's a bug in DateExtract.resolveType" + ); + }; + + ExpressionEvaluator.Factory fieldEvaluator = toEvaluator.apply(children().get(1)); + + // Constant chrono field if (children().get(0).foldable()) { ChronoField chrono = chronoField(toEvaluator.foldCtx()); if (chrono == null) { BytesRef field = (BytesRef) children().get(0).fold(toEvaluator.foldCtx()); throw new InvalidArgumentException("invalid date field for [{}]: {}", sourceText(), field.utf8ToString()); } - return new DateExtractConstantEvaluator.Factory(source(), fieldEvaluator, chrono, configuration().zoneId()); + + if (isNanos) { + return new DateExtractConstantNanosEvaluator.Factory(source(), fieldEvaluator, chrono, configuration().zoneId()); + } else { + return new DateExtractConstantMillisEvaluator.Factory(source(), fieldEvaluator, chrono, configuration().zoneId()); + } } + var chronoEvaluator = toEvaluator.apply(children().get(0)); - return new DateExtractEvaluator.Factory(source(), fieldEvaluator, chronoEvaluator, configuration().zoneId()); + + if (isNanos) { + return new DateExtractNanosEvaluator.Factory(source(), fieldEvaluator, chronoEvaluator, configuration().zoneId()); + } else { + return new DateExtractMillisEvaluator.Factory(source(), fieldEvaluator, chronoEvaluator, configuration().zoneId()); + } + } private ChronoField chronoField(FoldContext ctx) { @@ -138,16 +167,26 @@ private ChronoField chronoField(FoldContext ctx) { return chronoField; } - @Evaluator(warnExceptions = { IllegalArgumentException.class }) - static long process(long value, BytesRef chronoField, @Fixed ZoneId zone) { + @Evaluator(extraName = "Millis", warnExceptions = { IllegalArgumentException.class }) + static long processMillis(long value, BytesRef chronoField, @Fixed ZoneId zone) { return chronoToLong(value, chronoField, zone); } - @Evaluator(extraName = "Constant") - static long process(long value, @Fixed ChronoField chronoField, @Fixed ZoneId zone) { + @Evaluator(extraName = "ConstantMillis") + static long processMillis(long value, @Fixed ChronoField chronoField, @Fixed ZoneId zone) { return chronoToLong(value, chronoField, zone); } + @Evaluator(extraName = "Nanos", warnExceptions = { IllegalArgumentException.class }) + static long processNanos(long value, BytesRef chronoField, @Fixed ZoneId zone) { + return chronoToLongNanos(value, chronoField, zone); + } + + @Evaluator(extraName = "ConstantNanos") + static long processNanos(long value, @Fixed ChronoField chronoField, @Fixed ZoneId zone) { + return chronoToLongNanos(value, chronoField, zone); + } + @Override public Expression replaceChildren(List newChildren) { return new DateExtract(source(), newChildren.get(0), newChildren.get(1), configuration()); @@ -168,8 +207,15 @@ protected TypeResolution resolveType() { if (childrenResolved() == false) { return new TypeResolution("Unresolved children"); } + String operationName = sourceText(); return isStringAndExact(children().get(0), sourceText(), TypeResolutions.ParamOrdinal.FIRST).and( - isDate(children().get(1), sourceText(), TypeResolutions.ParamOrdinal.SECOND) + TypeResolutions.isType( + children().get(1), + DataType::isDate, + operationName, + TypeResolutions.ParamOrdinal.SECOND, + "datetime or date_nanos" + ) ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java index eef0df6b89dd3..4259de7347abd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -464,13 +464,36 @@ private static ChronoField stringToChrono(Object field) { public static long chronoToLong(long dateTime, BytesRef chronoField, ZoneId zone) { ChronoField chrono = ChronoField.valueOf(chronoField.utf8ToString().toUpperCase(Locale.ROOT)); - return Instant.ofEpochMilli(dateTime).atZone(zone).getLong(chrono); + return chronoToLong(dateTime, chrono, zone); } public static long chronoToLong(long dateTime, ChronoField chronoField, ZoneId zone) { return Instant.ofEpochMilli(dateTime).atZone(zone).getLong(chronoField); } + /** + * Extract the given {@link ChronoField} value from a date specified as a long number of nanoseconds since epoch + * @param dateNanos - long nanoseconds since epoch + * @param chronoField - The field to extract + * @param zone - Timezone for the given date + * @return - long representing the given ChronoField value + */ + public static long chronoToLongNanos(long dateNanos, BytesRef chronoField, ZoneId zone) { + ChronoField chrono = ChronoField.valueOf(chronoField.utf8ToString().toUpperCase(Locale.ROOT)); + return chronoToLongNanos(dateNanos, chrono, zone); + } + + /** + * Extract the given {@link ChronoField} value from a date specified as a long number of nanoseconds since epoch + * @param dateNanos - long nanoseconds since epoch + * @param chronoField - The field to extract + * @param zone - Timezone for the given date + * @return - long representing the given ChronoField value + */ + public static long chronoToLongNanos(long dateNanos, ChronoField chronoField, ZoneId zone) { + return DateUtils.toInstant(dateNanos).atZone(zone).getLong(chronoField); + } + /** * The following conversions are between String and other data types. */ diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractErrorTests.java index d5b9a06c8738e..feee1dd06f30f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractErrorTests.java @@ -35,7 +35,7 @@ protected Expression build(Source source, List args) { protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { case 0 -> "string"; - case 1 -> "datetime"; + case 1 -> "datetime or date_nanos"; default -> ""; })); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java index cd27ce511b317..01a84d7885ed3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java @@ -26,6 +26,7 @@ import java.time.Instant; import java.time.ZonedDateTime; import java.time.temporal.ChronoField; +import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; @@ -40,53 +41,70 @@ public DateExtractTests(@Name("TestCase") Supplier te @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors( - true, - List.of( - new TestCaseSupplier( - List.of(DataType.KEYWORD, DataType.DATETIME), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("YeAr"), DataType.KEYWORD, "chrono"), - new TestCaseSupplier.TypedData(1687944333000L, DataType.DATETIME, "date") - ), - "DateExtractEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]", - DataType.LONG, - equalTo(2023L) - ) - ), - new TestCaseSupplier( - List.of(DataType.TEXT, DataType.DATETIME), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("YeAr"), DataType.TEXT, "chrono"), - new TestCaseSupplier.TypedData(1687944333000L, DataType.DATETIME, "date") - ), - "DateExtractEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]", - DataType.LONG, - equalTo(2023L) - ) - ), - new TestCaseSupplier( - List.of(DataType.KEYWORD, DataType.DATETIME), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("not a unit"), DataType.KEYWORD, "chrono"), - new TestCaseSupplier.TypedData(0L, DataType.DATETIME, "date") + var suppliers = new ArrayList(); - ), - "DateExtractEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]", - DataType.LONG, - is(nullValue()) - ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") - .withWarning( - "Line -1:-1: java.lang.IllegalArgumentException: " - + "No enum constant java.time.temporal.ChronoField.NOT A UNIT" + for (var stringType : DataType.stringTypes()) { + suppliers.addAll( + List.of( + new TestCaseSupplier( + List.of(stringType, DataType.DATETIME), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef("YeAr"), stringType, "chrono"), + new TestCaseSupplier.TypedData(1687944333000L, DataType.DATETIME, "date") + ), + "DateExtractMillisEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]", + DataType.LONG, + equalTo(2023L) + ) + ), + new TestCaseSupplier( + List.of(stringType, DataType.DATE_NANOS), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef("YeAr"), stringType, "chrono"), + new TestCaseSupplier.TypedData(1687944333000000000L, DataType.DATE_NANOS, "date") + ), + "DateExtractNanosEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]", + DataType.LONG, + equalTo(2023L) ) - .withFoldingException(InvalidArgumentException.class, "invalid date field for []: not a unit") + ), + new TestCaseSupplier( + List.of(stringType, DataType.DATE_NANOS), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef("nano_of_second"), stringType, "chrono"), + new TestCaseSupplier.TypedData(1687944333000123456L, DataType.DATE_NANOS, "date") + ), + "DateExtractNanosEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]", + DataType.LONG, + equalTo(123456L) + ) + ), + new TestCaseSupplier( + List.of(stringType, DataType.DATETIME), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef("not a unit"), stringType, "chrono"), + new TestCaseSupplier.TypedData(0L, DataType.DATETIME, "date") + + ), + "DateExtractMillisEvaluator[value=Attribute[channel=1], chronoField=Attribute[channel=0], zone=Z]", + DataType.LONG, + is(nullValue()) + ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") + .withWarning( + "Line -1:-1: java.lang.IllegalArgumentException: " + + "No enum constant java.time.temporal.ChronoField.NOT A UNIT" + ) + .withFoldingException(InvalidArgumentException.class, "invalid date field for []: not a unit") + ) ) - ) - ); + ); + } + + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } public void testAllChronoFields() { @@ -102,7 +120,7 @@ public void testAllChronoFields() { assertThat(instance.fold(FoldContext.small()), is(date.getLong(value))); assertThat( - DateExtract.process(epochMilli, new BytesRef(value.name()), EsqlTestUtils.TEST_CFG.zoneId()), + DateExtract.processMillis(epochMilli, new BytesRef(value.name()), EsqlTestUtils.TEST_CFG.zoneId()), is(date.getLong(value)) ); }