diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/first_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/types/first_over_time.md index 18df111586e0a..621b49b68739c 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/first_over_time.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/first_over_time.md @@ -4,6 +4,9 @@ | field | result | | --- | --- | +| counter_double | double | +| counter_integer | integer | +| counter_long | long | | double | double | | integer | integer | | long | long | diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/last_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/types/last_over_time.md index 18df111586e0a..621b49b68739c 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/last_over_time.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/last_over_time.md @@ -4,6 +4,9 @@ | field | result | | --- | --- | +| counter_double | double | +| counter_integer | integer | +| counter_long | long | | double | double | | integer | integer | | long | long | diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/first_over_time.json b/docs/reference/query-languages/esql/kibana/definition/functions/first_over_time.json index e2cc52285e8e9..33761fcf1e13a 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/first_over_time.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/first_over_time.json @@ -4,6 +4,42 @@ "name" : "first_over_time", "description" : "Calculates the earliest value of a field, where recency determined by the `@timestamp` field.", "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "counter_double", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "field", + "type" : "counter_integer", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "integer" + }, + { + "params" : [ + { + "name" : "field", + "type" : "counter_long", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "long" + }, { "params" : [ { diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/last_over_time.json b/docs/reference/query-languages/esql/kibana/definition/functions/last_over_time.json index 48f96880e0001..62f2afda7b34d 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/last_over_time.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/last_over_time.json @@ -4,6 +4,42 @@ "name" : "last_over_time", "description" : "Calculates the latest value of a field, where recency determined by the `@timestamp` field.", "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "counter_double", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "field", + "type" : "counter_integer", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "integer" + }, + { + "params" : [ + { + "name" : "field", + "type" : "counter_long", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "long" + }, { "params" : [ { diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 618f941a0214a..00f1e4dd75fc2 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -802,6 +802,15 @@ public DataType noText() { return isString(this) ? KEYWORD : this; } + public DataType noCounter() { + return switch (this) { + case COUNTER_DOUBLE -> DOUBLE; + case COUNTER_INTEGER -> INTEGER; + case COUNTER_LONG -> LONG; + default -> this; + }; + } + public boolean isDate() { return switch (this) { case DATETIME, DATE_NANOS -> true; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/EsqlQueryGenerator.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/EsqlQueryGenerator.java index 2e56fd9243a2f..47e47fa4de1ab 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/EsqlQueryGenerator.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/generator/EsqlQueryGenerator.java @@ -295,12 +295,15 @@ yield switch ((randomIntBetween(0, 6))) { if (counterField == null) { yield null; } - yield "rate(" + counterField + ")"; + yield switch ((randomIntBetween(0, 2))) { + case 0 -> "rate(" + counterField + ")"; + case 1 -> "first_over_time(" + counterField + ")"; + default -> "last_over_time(" + counterField + ")"; + }; } case 2 -> { // numerics except aggregate_metric_double // TODO: add to case 0 when support for aggregate_metric_double is added to these functions - // TODO: add to case 1 when support for counters is added String numericFieldName = randomNumericField(previousOutput); if (numericFieldName == null) { yield null; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries-first-over-time.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries-first-over-time.csv-spec index 40a5d38f3e2a7..e871415630189 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries-first-over-time.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries-first-over-time.csv-spec @@ -126,3 +126,42 @@ events:long | pod:keyword | time_bucket:datetime 6 | two | 2024-05-10T00:20:00.000Z ; +first_over_time_counter_double +required_capability: ts_command_v0 +required_capability: first_last_over_time_counter_support +TS k8s +| STATS sum = sum(first_over_time(network.total_cost)) by pod, time_bucket = bucket(@timestamp, 10minute) +| SORT time_bucket, pod +; + +sum:double | pod:keyword | time_bucket:datetime +35.75 | one | 2024-05-10T00:00:00.000Z +34.375 | three | 2024-05-10T00:00:00.000Z +30.375 | two | 2024-05-10T00:00:00.000Z +96.5 | one | 2024-05-10T00:10:00.000Z +191.75 | three | 2024-05-10T00:10:00.000Z +163.25 | two | 2024-05-10T00:10:00.000Z +224.875 | one | 2024-05-10T00:20:00.000Z +142.875 | three | 2024-05-10T00:20:00.000Z +163.625 | two | 2024-05-10T00:20:00.000Z +; + +first_over_time_counter_long +required_capability: ts_command_v0 +required_capability: first_last_over_time_counter_support +TS k8s +| STATS max = max(first_over_time(network.total_bytes_in)) by pod, time_bucket = bucket(@timestamp, 10minute) +| SORT time_bucket, pod +; + +max:long | pod:keyword | time_bucket:datetime +1103 | one | 2024-05-10T00:00:00.000Z +1441 | three | 2024-05-10T00:00:00.000Z +1395 | two | 2024-05-10T00:00:00.000Z +6077 | one | 2024-05-10T00:10:00.000Z +4200 | three | 2024-05-10T00:10:00.000Z +3478 | two | 2024-05-10T00:10:00.000Z +10207 | one | 2024-05-10T00:20:00.000Z +9969 | three | 2024-05-10T00:20:00.000Z +8709 | two | 2024-05-10T00:20:00.000Z +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries-last-over-time.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries-last-over-time.csv-spec index adeb3894e16f5..b675cf30fc31f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries-last-over-time.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries-last-over-time.csv-spec @@ -276,3 +276,42 @@ events:long | pod:keyword | time_bucket:datetime 5 | two | 2024-05-10T00:20:00.000Z ; +last_over_time_counter_double +required_capability: ts_command_v0 +required_capability: first_last_over_time_counter_support +TS k8s +| STATS sum = sum(last_over_time(network.total_cost)) by pod, time_bucket = bucket(@timestamp, 10minute) +| SORT time_bucket, pod +; + +sum:double | pod:keyword | time_bucket:datetime +190.125 | one | 2024-05-10T00:00:00.000Z +158.75 | three | 2024-05-10T00:00:00.000Z +148.25 | two | 2024-05-10T00:00:00.000Z +220.125 | one | 2024-05-10T00:10:00.000Z +271.75 | three | 2024-05-10T00:10:00.000Z +160.75 | two | 2024-05-10T00:10:00.000Z +190.625 | one | 2024-05-10T00:20:00.000Z +180.375 | three | 2024-05-10T00:20:00.000Z +191.0 | two | 2024-05-10T00:20:00.000Z +; + +last_over_time_counter_long +required_capability: ts_command_v0 +required_capability: first_last_over_time_counter_support +TS k8s +| STATS max = max(last_over_time(network.total_bytes_in)) by pod, time_bucket = bucket(@timestamp, 10minute) +| SORT time_bucket, pod +; + +max:long | pod:keyword | time_bucket:datetime +5963 | one | 2024-05-10T00:00:00.000Z +4169 | three | 2024-05-10T00:00:00.000Z +7900 | two | 2024-05-10T00:00:00.000Z +9254 | one | 2024-05-10T00:10:00.000Z +9195 | three | 2024-05-10T00:10:00.000Z +8031 | two | 2024-05-10T00:10:00.000Z +7286 | one | 2024-05-10T00:20:00.000Z +10797 | three | 2024-05-10T00:20:00.000Z +10277 | two | 2024-05-10T00:20:00.000Z +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 483da630bb4c3..d085bdec02b06 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1549,6 +1549,11 @@ public enum Cap { */ TS_COMMAND_V0(), + /** + * Add support for counter doubles, ints, and longs in first_ and last_over_time + */ + FIRST_LAST_OVER_TIME_COUNTER_SUPPORT, + FIX_ALIAS_ID_WHEN_DROP_ALL_AGGREGATES, /** diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTime.java index ca76d521142c7..9270c8d301b4e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTime.java @@ -56,7 +56,10 @@ public class FirstOverTime extends TimeSeriesAggregateFunction implements Option preview = true, examples = { @Example(file = "k8s-timeseries", tag = "first_over_time") } ) - public FirstOverTime(Source source, @Param(name = "field", type = { "long", "integer", "double" }) Expression field) { + public FirstOverTime( + Source source, + @Param(name = "field", type = { "counter_long", "counter_integer", "counter_double", "long", "integer", "double" }) Expression field + ) { this(source, field, new UnresolvedAttribute(source, "@timestamp")); } @@ -109,21 +112,20 @@ public FirstOverTime withFilter(Expression filter) { @Override public DataType dataType() { - return field().dataType(); + return field().dataType().noCounter(); } @Override protected TypeResolution resolveType() { - return isType(field(), dt -> dt.isNumeric() && dt != DataType.UNSIGNED_LONG, sourceText(), DEFAULT, "numeric except unsigned_long") - .and( - isType( - timestamp, - dt -> dt == DataType.DATETIME || dt == DataType.DATE_NANOS, - sourceText(), - SECOND, - "date_nanos or datetime" - ) - ); + return isType( + field(), + dt -> (dt.noCounter().isNumeric() && dt != DataType.UNSIGNED_LONG) || dt == DataType.AGGREGATE_METRIC_DOUBLE, + sourceText(), + DEFAULT, + "numeric except unsigned_long" + ).and( + isType(timestamp, dt -> dt == DataType.DATETIME || dt == DataType.DATE_NANOS, sourceText(), SECOND, "date_nanos or datetime") + ); } @Override @@ -132,9 +134,9 @@ public AggregatorFunctionSupplier supplier() { // we can read the first encountered value for each group of `_tsid` and time bucket. final DataType type = field().dataType(); return switch (type) { - case LONG -> new FirstLongByTimestampAggregatorFunctionSupplier(); - case INTEGER -> new FirstIntByTimestampAggregatorFunctionSupplier(); - case DOUBLE -> new FirstDoubleByTimestampAggregatorFunctionSupplier(); + case LONG, COUNTER_LONG -> new FirstLongByTimestampAggregatorFunctionSupplier(); + case INTEGER, COUNTER_INTEGER -> new FirstIntByTimestampAggregatorFunctionSupplier(); + case DOUBLE, COUNTER_DOUBLE -> new FirstDoubleByTimestampAggregatorFunctionSupplier(); case FLOAT -> new FirstFloatByTimestampAggregatorFunctionSupplier(); default -> throw EsqlIllegalArgumentException.illegalDataType(type); }; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java index ccb9695339891..e754c83dd9d90 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTime.java @@ -57,7 +57,13 @@ public class LastOverTime extends TimeSeriesAggregateFunction implements Optiona preview = true, examples = { @Example(file = "k8s-timeseries", tag = "last_over_time") } ) - public LastOverTime(Source source, @Param(name = "field", type = { "long", "integer", "double", "_tsid" }) Expression field) { + public LastOverTime( + Source source, + @Param( + name = "field", + type = { "counter_long", "counter_integer", "counter_double", "long", "integer", "double", "_tsid" } + ) Expression field + ) { this(source, field, new UnresolvedAttribute(source, "@timestamp")); } @@ -110,14 +116,14 @@ public LastOverTime withFilter(Expression filter) { @Override public DataType dataType() { - return field().dataType(); + return field().dataType().noCounter(); } @Override protected TypeResolution resolveType() { return isType( field(), - dt -> (dt.isNumeric() && dt != DataType.UNSIGNED_LONG) || dt == DataType.TSID_DATA_TYPE, + dt -> (dt.noCounter().isNumeric() && dt != DataType.UNSIGNED_LONG) || dt == DataType.TSID_DATA_TYPE, sourceText(), DEFAULT, "numeric except unsigned_long" @@ -132,9 +138,9 @@ public AggregatorFunctionSupplier supplier() { // we can read the first encountered value for each group of `_tsid` and time bucket. final DataType type = field().dataType(); return switch (type) { - case LONG -> new LastLongByTimestampAggregatorFunctionSupplier(); - case INTEGER -> new LastIntByTimestampAggregatorFunctionSupplier(); - case DOUBLE -> new LastDoubleByTimestampAggregatorFunctionSupplier(); + case LONG, COUNTER_LONG -> new LastLongByTimestampAggregatorFunctionSupplier(); + case INTEGER, COUNTER_INTEGER -> new LastIntByTimestampAggregatorFunctionSupplier(); + case DOUBLE, COUNTER_DOUBLE -> new LastDoubleByTimestampAggregatorFunctionSupplier(); case FLOAT -> new LastFloatByTimestampAggregatorFunctionSupplier(); case TSID_DATA_TYPE -> new LastBytesRefByTimestampAggregatorFunctionSupplier(); default -> throw EsqlIllegalArgumentException.illegalDataType(type); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTimeTests.java index 49a50d697a4b6..7f8dc0d8475a7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTimeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstOverTimeTests.java @@ -43,8 +43,13 @@ public static Iterable parameters() { ); for (List valuesSupplier : valuesSuppliers) { for (TestCaseSupplier.TypedDataSupplier fieldSupplier : valuesSupplier) { - TestCaseSupplier testCaseSupplier = makeSupplier(fieldSupplier); - suppliers.add(testCaseSupplier); + DataType type = fieldSupplier.type(); + suppliers.add(makeSupplier(fieldSupplier, type)); + switch (type) { + case LONG -> suppliers.add(makeSupplier(fieldSupplier, DataType.COUNTER_LONG)); + case DOUBLE -> suppliers.add(makeSupplier(fieldSupplier, DataType.COUNTER_DOUBLE)); + case INTEGER -> suppliers.add(makeSupplier(fieldSupplier, DataType.COUNTER_INTEGER)); + } } } return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(suppliers); @@ -65,11 +70,11 @@ public void testAggregateIntermediate() { assumeTrue("time-series aggregation doesn't support ungrouped", false); } - private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier) { - DataType type = fieldSupplier.type(); + private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier, DataType type) { return new TestCaseSupplier(fieldSupplier.name(), List.of(type, DataType.DATETIME), () -> { TestCaseSupplier.TypedData fieldTypedData = fieldSupplier.get(); List dataRows = fieldTypedData.multiRowData(); + fieldTypedData = TestCaseSupplier.TypedData.multiRow(dataRows, type, fieldTypedData.name()); List timestamps = IntStream.range(0, dataRows.size()).mapToLong(unused -> randomNonNegativeLong()).boxed().toList(); TestCaseSupplier.TypedData timestampsField = TestCaseSupplier.TypedData.multiRow(timestamps, DataType.DATETIME, "timestamps"); Object expected = null; @@ -85,8 +90,8 @@ private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier } return new TestCaseSupplier.TestCase( List.of(fieldTypedData, timestampsField), - standardAggregatorName("First", fieldSupplier.type()) + "ByTimestamp", - type, + standardAggregatorName("First", type) + "ByTimestamp", + fieldSupplier.type(), equalTo(expected) ); }); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java index 6d04983f38fcc..91162f7bf8c33 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/LastOverTimeTests.java @@ -43,8 +43,13 @@ public static Iterable parameters() { ); for (List valuesSupplier : valuesSuppliers) { for (TestCaseSupplier.TypedDataSupplier fieldSupplier : valuesSupplier) { - TestCaseSupplier testCaseSupplier = makeSupplier(fieldSupplier); - suppliers.add(testCaseSupplier); + DataType type = fieldSupplier.type(); + suppliers.add(makeSupplier(fieldSupplier, type)); + switch (type) { + case LONG -> suppliers.add(makeSupplier(fieldSupplier, DataType.COUNTER_LONG)); + case DOUBLE -> suppliers.add(makeSupplier(fieldSupplier, DataType.COUNTER_DOUBLE)); + case INTEGER -> suppliers.add(makeSupplier(fieldSupplier, DataType.COUNTER_INTEGER)); + } } } return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(suppliers); @@ -65,11 +70,11 @@ public void testAggregateIntermediate() { assumeTrue("time-series aggregation doesn't support ungrouped", false); } - private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier) { - DataType type = fieldSupplier.type(); + private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier, DataType type) { return new TestCaseSupplier(fieldSupplier.name(), List.of(type, DataType.DATETIME), () -> { TestCaseSupplier.TypedData fieldTypedData = fieldSupplier.get(); List dataRows = fieldTypedData.multiRowData(); + fieldTypedData = TestCaseSupplier.TypedData.multiRow(dataRows, type, fieldTypedData.name()); List timestamps = IntStream.range(0, dataRows.size()).mapToLong(unused -> randomNonNegativeLong()).boxed().toList(); TestCaseSupplier.TypedData timestampsField = TestCaseSupplier.TypedData.multiRow(timestamps, DataType.DATETIME, "timestamps"); Object expected = null; @@ -85,8 +90,8 @@ private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier } return new TestCaseSupplier.TestCase( List.of(fieldTypedData, timestampsField), - standardAggregatorName("Last", fieldSupplier.type()) + "ByTimestamp", - type, + standardAggregatorName("Last", type) + "ByTimestamp", + fieldSupplier.type(), equalTo(expected) ); });