diff --git a/muted-tests.yml b/muted-tests.yml index 54efde6a77ef0..b833bb00db460 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -429,9 +429,6 @@ tests: - class: org.elasticsearch.xpack.search.CrossClusterAsyncSearchIT method: testCancelViaExpirationOnRemoteResultsWithMinimizeRoundtrips issue: https://github.com/elastic/elasticsearch/issues/127302 -- class: org.elasticsearch.xpack.test.rest.XPackRestIT - method: test {p0=esql/60_usage/*} - issue: https://github.com/elastic/elasticsearch/issues/133449 - class: org.elasticsearch.xpack.esql.action.CrossClusterQueryWithFiltersIT method: testFilterWithUnavailableRemote issue: https://github.com/elastic/elasticsearch/issues/133450 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java index 465631afe5af2..34dabe3d03782 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.telemetry; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.ChangePoint; @@ -59,7 +60,8 @@ public enum FeatureMetric { MV_EXPAND(MvExpand.class::isInstance), SHOW(ShowInfo.class::isInstance), ROW(Row.class::isInstance), - FROM(EsRelation.class::isInstance), + FROM(x -> x instanceof EsRelation relation && relation.indexMode() != IndexMode.TIME_SERIES), + TS(x -> x instanceof EsRelation relation && relation.indexMode() == IndexMode.TIME_SERIES), DROP(Drop.class::isInstance), KEEP(Keep.class::isInstance), RENAME(Rename.class::isInstance), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java index 8abb9f982551f..05e0f70b949f0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java @@ -41,6 +41,7 @@ import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.SHOW; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.SORT; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.STATS; +import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.TS; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.WHERE; import static org.elasticsearch.xpack.esql.telemetry.Metrics.FEATURES_PREFIX; import static org.elasticsearch.xpack.esql.telemetry.Metrics.FUNC_PREFIX; @@ -63,6 +64,7 @@ public void testDissectQuery() { assertEquals(0, show(c)); assertEquals(0, row(c)); assertEquals(1L, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); @@ -86,6 +88,7 @@ public void testEvalQuery() { assertEquals(0, show(c)); assertEquals(0, row(c)); assertEquals(1L, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); @@ -109,6 +112,7 @@ public void testGrokQuery() { assertEquals(0, show(c)); assertEquals(0, row(c)); assertEquals(1L, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); @@ -132,6 +136,7 @@ public void testLimitQuery() { assertEquals(0, show(c)); assertEquals(0, row(c)); assertEquals(1L, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); @@ -154,6 +159,7 @@ public void testSortQuery() { assertEquals(0, show(c)); assertEquals(0, row(c)); assertEquals(1L, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); @@ -176,6 +182,7 @@ public void testStatsQuery() { assertEquals(0, show(c)); assertEquals(0, row(c)); assertEquals(1L, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); @@ -199,6 +206,7 @@ public void testWhereQuery() { assertEquals(0, show(c)); assertEquals(0, row(c)); assertEquals(1L, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); @@ -221,6 +229,7 @@ public void testTwoWhereQuery() { assertEquals(0, show(c)); assertEquals(0, row(c)); assertEquals(1L, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); @@ -263,6 +272,7 @@ public void testTwoQueriesExecuted() { assertEquals(0, show(c)); assertEquals(0, row(c)); assertEquals(2L, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); @@ -349,6 +359,7 @@ public void testEnrich() { assertEquals(0, show(c)); assertEquals(0, row(c)); assertEquals(1L, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(1L, keep(c)); assertEquals(0, rename(c)); @@ -381,6 +392,7 @@ public void testMvExpand() { assertEquals(0, show(c)); assertEquals(0, row(c)); assertEquals(1L, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(1L, keep(c)); assertEquals(0, rename(c)); @@ -403,6 +415,7 @@ public void testShowInfo() { assertEquals(1L, show(c)); assertEquals(0, row(c)); assertEquals(0, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); @@ -426,6 +439,7 @@ public void testRow() { assertEquals(0, show(c)); assertEquals(1L, row(c)); assertEquals(0, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); @@ -448,6 +462,7 @@ public void testDropAndRename() { assertEquals(0, show(c)); assertEquals(0, row(c)); assertEquals(1L, from(c)); + assertEquals(0, ts(c)); assertEquals(1L, drop(c)); assertEquals(0, keep(c)); assertEquals(1L, rename(c)); @@ -476,6 +491,7 @@ public void testKeep() { assertEquals(0, show(c)); assertEquals(0, row(c)); assertEquals(1L, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(1L, keep(c)); assertEquals(0, rename(c)); @@ -502,6 +518,7 @@ public void testCategorize() { assertEquals(0, show(c)); assertEquals(0, row(c)); assertEquals(1L, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(1L, keep(c)); assertEquals(0, rename(c)); @@ -530,6 +547,7 @@ public void testInlineStatsStandalone() { assertEquals(0, show(c)); assertEquals(0, row(c)); assertEquals(1L, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); @@ -558,6 +576,7 @@ public void testInlineStatsWithOtherStats() { assertEquals(0, show(c)); assertEquals(0, row(c)); assertEquals(1L, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); @@ -585,6 +604,7 @@ public void testBinaryPlanAfterStats() { assertEquals(0, show(c)); assertEquals(0, row(c)); assertEquals(1L, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); @@ -617,6 +637,7 @@ public void testBinaryPlanAfterStatsExpressionJoin() { assertEquals(0, show(c)); assertEquals(0, row(c)); assertEquals(1L, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(1L, rename(c)); @@ -645,6 +666,7 @@ public void testBinaryPlanAfterInlineStats() { assertEquals(0, show(c)); assertEquals(0, row(c)); assertEquals(1L, from(c)); + assertEquals(0, ts(c)); assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); @@ -654,6 +676,60 @@ public void testBinaryPlanAfterInlineStats() { assertEquals(1, function("max", c)); } + public void testTimeSeriesAggregate() { + assumeTrue("TS required", EsqlCapabilities.Cap.TS_COMMAND_V0.isEnabled()); + Counters c = esql(""" + TS metrics + | STATS sum(avg_over_time(salary))"""); + assertEquals(0, dissect(c)); + assertEquals(0, eval(c)); + assertEquals(0, grok(c)); + assertEquals(0, limit(c)); + assertEquals(0, sort(c)); + assertEquals(1L, stats(c)); + assertEquals(0, where(c)); + assertEquals(0, enrich(c)); + assertEquals(0, mvExpand(c)); + assertEquals(0, show(c)); + assertEquals(0, row(c)); + assertEquals(0, from(c)); + assertEquals(1L, ts(c)); + assertEquals(0, drop(c)); + assertEquals(0, keep(c)); + assertEquals(0, rename(c)); + assertEquals(0, inlineStats(c)); + assertEquals(0, lookupJoinOnFields(c)); + assertEquals(0, lookupJoinOnExpression(c)); + assertEquals(1, function("sum", c)); + assertEquals(1, function("avg_over_time", c)); + } + + public void testTimeSeriesNoAggregate() { + assumeTrue("TS required", EsqlCapabilities.Cap.TS_COMMAND_V0.isEnabled()); + Counters c = esql(""" + TS metrics + | KEEP salary"""); + assertEquals(0, dissect(c)); + assertEquals(0, eval(c)); + assertEquals(0, grok(c)); + assertEquals(0, limit(c)); + assertEquals(0, sort(c)); + assertEquals(0, stats(c)); + assertEquals(0, where(c)); + assertEquals(0, enrich(c)); + assertEquals(0, mvExpand(c)); + assertEquals(0, show(c)); + assertEquals(0, row(c)); + assertEquals(0, from(c)); + assertEquals(1L, ts(c)); + assertEquals(0, drop(c)); + assertEquals(1L, keep(c)); + assertEquals(0, rename(c)); + assertEquals(0, inlineStats(c)); + assertEquals(0, lookupJoinOnFields(c)); + assertEquals(0, lookupJoinOnExpression(c)); + } + private long dissect(Counters c) { return c.get(FEATURES_PREFIX + DISSECT); } @@ -702,6 +778,10 @@ private long from(Counters c) { return c.get(FEATURES_PREFIX + FROM); } + private long ts(Counters c) { + return c.get(FEATURES_PREFIX + TS); + } + private long drop(Counters c) { return c.get(FEATURES_PREFIX + DROP); } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index 3a7df3b62764f..c4d5783b810ea 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -24,6 +24,24 @@ setup: message: type: keyword + - do: + indices.create: + index: test-ts + body: + settings: + number_of_shards: 1 + index.mode: time_series + index.routing_path: [ host ] + mappings: + properties: + "@timestamp": + type: date + count: + type: integer + host: + type: keyword + time_series_dimension: true + --- "Basic ESQL usage output (telemetry) snapshot version": - requires: @@ -50,12 +68,13 @@ setup: - do: { xpack.usage: { } } - match: { esql.available: true } - match: { esql.enabled: true } - - length: { esql.features: 27 } + - length: { esql.features: 28 } - set: { esql.features.dissect: dissect_counter } - set: { esql.features.drop: drop_counter } - set: { esql.features.eval: eval_counter } - set: { esql.features.enrich: enrich_counter } - set: { esql.features.from: from_counter } + - set: { esql.features.ts: ts_counter } - set: { esql.features.grok: grok_counter } - set: { esql.features.keep: keep_counter } - set: { esql.features.limit: limit_counter } @@ -122,6 +141,7 @@ setup: - match: { esql.features.fuse: $fuse_counter } - match: { esql.features.completion: $completion_counter } - match: { esql.features.sample: $sample_counter } + - match: { esql.features.ts: $ts_counter } - gt: { esql.queries.rest.total: $rest_total_counter } - match: { esql.queries.rest.failed: $rest_failed_counter } - match: { esql.queries.kibana.total: $kibana_total_counter } @@ -136,6 +156,53 @@ setup: - gt: { esql.functions.categorize: $functions_categorize } - match: {esql.functions.text_embedding: $functions_text_embedding} + - set: { esql.features.limit: limit_counter } + - set: { esql.features.sort: sort_counter } + - set: { esql.features.stats: stats_counter } + - set: { esql.features.where: where_counter } + - set: { esql.features.ts: ts_counter } + - set: { esql.queries.rest.total: rest_total_counter } + - set: { esql.queries._all.total: all_total_counter } + - set: { esql.functions.min: functions_min } + - set: { esql.functions.avg_over_time: functions_avg_over_time } + + - do: + esql.query: + body: + query: 'TS test-ts + | stats min(avg_over_time(count))' + + - do: { xpack.usage: { } } + - match: { esql.available: true } + - match: { esql.enabled: true } + - match: { esql.features.dissect: $dissect_counter } + - match: { esql.features.eval: $eval_counter } + - match: { esql.features.grok: $grok_counter } + - match: { esql.features.limit: $limit_counter } + - match: { esql.features.sort: $sort_counter } + - gt: { esql.features.stats: $stats_counter } + - match: { esql.features.where: $where_counter } + - match: { esql.features.lookup_join: $lookup_join_counter } + - match: { esql.features.lookup_join_on_expression: $lookup_join_on_expression_counter } + - match: { esql.features.lookup: $lookup_counter } + - match: { esql.features.change_point: $change_point_counter } + - match: { esql.features.inline_stats: $inline_stats_counter } + - match: { esql.features.rerank: $rerank_counter } + - match: { esql.features.insist: $insist_counter } + - match: { esql.features.fork: $fork_counter } + - match: { esql.features.fuse: $fuse_counter } + - match: { esql.features.completion: $completion_counter } + - match: { esql.features.sample: $sample_counter } + - gt: { esql.features.ts: $ts_counter } + - gt: { esql.queries.rest.total: $rest_total_counter } + - match: { esql.queries.rest.failed: $rest_failed_counter } + - match: { esql.queries.kibana.total: $kibana_total_counter } + - match: { esql.queries.kibana.failed: $kibana_failed_counter } + - gt: { esql.queries._all.total: $all_total_counter } + - match: { esql.queries._all.failed: $all_failed_counter } + - gt: { esql.functions.min: $functions_min } + - gt: { esql.functions.min: $functions_avg_over_time } + # There's one of these per function but that's a ton of things to check. So we just spot check that a few exist. - exists: esql.functions.delay - exists: esql.functions.idelta @@ -156,12 +223,13 @@ setup: - do: { xpack.usage: { } } - match: { esql.available: true } - match: { esql.enabled: true } - - length: { esql.features: 27 } + - length: { esql.features: 28 } - set: { esql.features.dissect: dissect_counter } - set: { esql.features.drop: drop_counter } - set: { esql.features.eval: eval_counter } - set: { esql.features.enrich: enrich_counter } - set: { esql.features.from: from_counter } + - set: { esql.features.ts: ts_counter } - set: { esql.features.grok: grok_counter } - set: { esql.features.keep: keep_counter } - set: { esql.features.limit: limit_counter } @@ -226,6 +294,7 @@ setup: - match: { esql.features.fork: $fork_counter } - match: { esql.features.fuse: $fuse_counter } - match: { esql.features.completion: $completion_counter } + - match: { esql.features.ts: $ts_counter } - gt: { esql.queries.rest.total: $rest_total_counter } - match: { esql.queries.rest.failed: $rest_failed_counter } - match: { esql.queries.kibana.total: $kibana_total_counter } @@ -239,6 +308,53 @@ setup: - match: { esql.functions.coalesce: $functions_coalesce } - gt: { esql.functions.categorize: $functions_categorize } + - set: { esql.features.limit: limit_counter } + - set: { esql.features.sort: sort_counter } + - set: { esql.features.stats: stats_counter } + - set: { esql.features.where: where_counter } + - set: { esql.features.ts: ts_counter } + - set: { esql.queries.rest.total: rest_total_counter } + - set: { esql.queries._all.total: all_total_counter } + - set: { esql.functions.min: functions_min } + - set: { esql.functions.avg_over_time: functions_avg_over_time } + + - do: + esql.query: + body: + query: 'TS test-ts + | stats min(avg_over_time(count))' + + - do: { xpack.usage: { } } + - match: { esql.available: true } + - match: { esql.enabled: true } + - match: { esql.features.dissect: $dissect_counter } + - match: { esql.features.eval: $eval_counter } + - match: { esql.features.grok: $grok_counter } + - match: { esql.features.limit: $limit_counter } + - match: { esql.features.sort: $sort_counter } + - gt: { esql.features.stats: $stats_counter } + - match: { esql.features.where: $where_counter } + - match: { esql.features.lookup_join: $lookup_join_counter } + - match: { esql.features.lookup_join_on_expression: $lookup_join_on_expression_counter } + - match: { esql.features.lookup: $lookup_counter } + - match: { esql.features.change_point: $change_point_counter } + - match: { esql.features.inline_stats: $inline_stats_counter } + - match: { esql.features.rerank: $rerank_counter } + - match: { esql.features.insist: $insist_counter } + - match: { esql.features.fork: $fork_counter } + - match: { esql.features.fuse: $fuse_counter } + - match: { esql.features.completion: $completion_counter } + - match: { esql.features.sample: $sample_counter } + - gt: { esql.features.ts: $ts_counter } + - gt: { esql.queries.rest.total: $rest_total_counter } + - match: { esql.queries.rest.failed: $rest_failed_counter } + - match: { esql.queries.kibana.total: $kibana_total_counter } + - match: { esql.queries.kibana.failed: $kibana_failed_counter } + - gt: { esql.queries._all.total: $all_total_counter } + - match: { esql.queries._all.failed: $all_failed_counter } + - gt: { esql.functions.min: $functions_min } + - gt: { esql.functions.min: $functions_avg_over_time } + # There's one of these per function but that's a ton of things to check. So we just spot check that a few exist. - not_exists: esql.functions.delay - exists: esql.functions.idelta