From 02c8a00cef75e60349ee391d7011457dd25d60e1 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 5 Sep 2025 17:04:57 +0200 Subject: [PATCH 01/10] Add relevant attributes to search took time APM metrics We already record the took time of a search request via took metric. We'd like to be able to slice such latencies based on some recurring categories of the request: - does it have agg or hit only? - is it sorted by field or by score? - does it have a time range filter? - does it target user data or internal indices? This commit introduces introspection for a search request and stores the extracted attributes together with the search took time metric. --- .../search/SearchRequestIntrospector.java | 245 ++++++++++++++ .../action/search/TransportSearchAction.java | 15 +- .../search/TransportSearchScrollAction.java | 2 +- .../action/search/SearchResponseMetrics.java | 12 +- .../SearchRequestIntrospectorTests.java | 298 ++++++++++++++++++ 5 files changed, 565 insertions(+), 7 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/action/search/SearchRequestIntrospector.java create mode 100644 server/src/test/java/org/elasticsearch/action/search/SearchRequestIntrospectorTests.java diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestIntrospector.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestIntrospector.java new file mode 100644 index 0000000000000..8862269ae47a0 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestIntrospector.java @@ -0,0 +1,245 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.search; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.BoostingQueryBuilder; +import org.elasticsearch.index.query.ConstantScoreQueryBuilder; +import org.elasticsearch.index.query.NestedQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.search.SearchService; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.vectors.KnnVectorQueryBuilder; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Used to introspect a search request and extract metadata from it around the features it uses. + */ +public class SearchRequestIntrospector { + + /** + * Introspects the provided search request and extracts metadata from it about some of its characteristics. + * + */ + public static QueryMetadata introspectSearchRequest(SearchRequest searchRequest) { + + String target = introspectIndices(searchRequest.indices()); + + String pitOrScroll = null; + if (searchRequest.scroll() != null) { + pitOrScroll = SCROLL; + } + + SearchSourceBuilder searchSourceBuilder = searchRequest.source(); + if (searchSourceBuilder == null) { + return new QueryMetadata(target, ScoreSortBuilder.NAME, HITS_ONLY, false, Strings.EMPTY_ARRAY, pitOrScroll); + } + + if (searchSourceBuilder.pointInTimeBuilder() != null) { + pitOrScroll = PIT; + } + + final String primarySort; + if (searchSourceBuilder.sorts() == null || searchSourceBuilder.sorts().isEmpty()) { + primarySort = ScoreSortBuilder.NAME; + } else { + primarySort = introspectPrimarySort(searchSourceBuilder.sorts().getFirst()); + } + + final String queryType = introspectQueryType(searchSourceBuilder); + + QueryMetadataBuilder queryMetadataBuilder = new QueryMetadataBuilder(); + if (searchSourceBuilder.query() != null) { + introspectQueryBuilder(searchSourceBuilder.query(), queryMetadataBuilder); + } + + final boolean hasKnn = searchSourceBuilder.knnSearch().isEmpty() == false || queryMetadataBuilder.knnQuery; + + return new QueryMetadata( + target, + primarySort, + queryType, + hasKnn, + queryMetadataBuilder.rangeFields.toArray(new String[0]), + pitOrScroll + ); + } + + private static final class QueryMetadataBuilder { + private boolean knnQuery = false; + private final List rangeFields = new ArrayList<>(); + } + + public record QueryMetadata( + String target, + String primarySort, + String queryType, + boolean knn, + String[] rangeFields, + String pitOrScroll + ) { + + public Map toAttributesMap() { + Map attributes = new HashMap<>(); + attributes.put(TARGET_ATTRIBUTE, target); + attributes.put(SORT_ATTRIBUTE, primarySort); + if (pitOrScroll == null) { + attributes.put(QUERY_TYPE_ATTRIBUTE, queryType); + } else { + attributes.put(QUERY_TYPE_ATTRIBUTE, Arrays.asList(queryType, pitOrScroll)); + } + + attributes.put(KNN_ATTRIBUTE, knn); + attributes.put(RANGES_ATTRIBUTE, rangeFields); + return attributes; + } + } + + private static final String TARGET_ATTRIBUTE = "target"; + private static final String SORT_ATTRIBUTE = "sort"; + private static final String QUERY_TYPE_ATTRIBUTE = "query_type"; + private static final String KNN_ATTRIBUTE = "knn"; + private static final String RANGES_ATTRIBUTE = "ranges"; + + private static final String TARGET_KIBANA = ".kibana"; + private static final String TARGET_ML = ".ml"; + private static final String TARGET_FLEET = ".fleet"; + private static final String TARGET_SLO = ".slo"; + private static final String TARGET_ALERTS = ".alerts"; + private static final String TARGET_ELASTIC = ".elastic"; + private static final String TARGET_DS = ".ds-"; + private static final String TARGET_OTHERS = ".others"; + private static final String TARGET_USER = "user"; + + static String introspectIndices(String[] indices) { + // Note that indices are expected to be resolved, meaning wildcards are not handled on purpose + // If indices resolve to data streams, the name of the data stream is returned as opposed to its backing indices + if (indices.length == 1) { + String index = indices[0]; + if (index.startsWith(".")) { + if (index.startsWith(TARGET_KIBANA)) { + return TARGET_KIBANA; + } + if (index.startsWith(TARGET_ML)) { + return TARGET_ML; + } + if (index.startsWith(TARGET_FLEET)) { + return TARGET_FLEET; + } + if (index.startsWith(TARGET_SLO)) { + return TARGET_SLO; + } + if (index.startsWith(TARGET_ALERTS)) { + return TARGET_ALERTS; + } + if (index.startsWith(TARGET_ELASTIC)) { + return TARGET_ELASTIC; + } + // this happens only when data streams backing indices are searched explicitly + if (index.startsWith(TARGET_DS)) { + return TARGET_DS; + } + return TARGET_OTHERS; + } + } + return TARGET_USER; + } + + private static final String TIMESTAMP = "@timestamp"; + private static final String EVENT_INGESTED = "event.ingested"; + private static final String _DOC = "_doc"; + private static final String FIELD = "field"; + + static String introspectPrimarySort(SortBuilder primarySortBuilder) { + if (primarySortBuilder instanceof FieldSortBuilder fieldSort) { + return switch (fieldSort.getFieldName()) { + case TIMESTAMP -> TIMESTAMP; + case EVENT_INGESTED -> EVENT_INGESTED; + case _DOC -> _DOC; + default -> FIELD; + }; + } + return primarySortBuilder.getWriteableName(); + } + + private static final String HITS_AND_AGGS = "hits_and_aggs"; + private static final String HITS_ONLY = "hits_only"; + private static final String AGGS_ONLY = "aggs_only"; + private static final String COUNT_ONLY = "count_only"; + private static final String PIT = "pit"; + private static final String SCROLL = "scroll"; + + public static final Map SEARCH_SCROLL_ATTRIBUTES = Map.of(QUERY_TYPE_ATTRIBUTE, SCROLL); + + static String introspectQueryType(SearchSourceBuilder searchSourceBuilder) { + int size = searchSourceBuilder.size(); + if (size == -1) { + size = SearchService.DEFAULT_SIZE; + } + if (searchSourceBuilder.aggregations() != null && size > 0) { + return HITS_AND_AGGS; + } + if (searchSourceBuilder.aggregations() != null) { + return AGGS_ONLY; + } + if (size > 0) { + return HITS_ONLY; + } + return COUNT_ONLY; + } + + static void introspectQueryBuilder(QueryBuilder queryBuilder, QueryMetadataBuilder queryMetadataBuilder) { + switch (queryBuilder) { + case BoolQueryBuilder bool: + for (QueryBuilder must : bool.must()) { + introspectQueryBuilder(must, queryMetadataBuilder); + } + for (QueryBuilder filter : bool.filter()) { + introspectQueryBuilder(filter, queryMetadataBuilder); + } + if (bool.must().isEmpty() && bool.filter().isEmpty() && bool.mustNot().isEmpty() && bool.should().size() == 1) { + introspectQueryBuilder(bool.should().getFirst(), queryMetadataBuilder); + } + // Note that should clauses are ignored unless there's only one that becomes mandatory + // must_not clauses are also ignored for now + break; + case ConstantScoreQueryBuilder constantScore: + introspectQueryBuilder(constantScore.innerQuery(), queryMetadataBuilder); + break; + case BoostingQueryBuilder boosting: + introspectQueryBuilder(boosting.positiveQuery(), queryMetadataBuilder); + break; + case NestedQueryBuilder nested: + introspectQueryBuilder(nested.query(), queryMetadataBuilder); + break; + case RangeQueryBuilder range: + switch (range.fieldName()) { + case TIMESTAMP -> queryMetadataBuilder.rangeFields.add(TIMESTAMP); + case EVENT_INGESTED -> queryMetadataBuilder.rangeFields.add(EVENT_INGESTED); + default -> queryMetadataBuilder.rangeFields.add(FIELD); + } + break; + case KnnVectorQueryBuilder knn: + queryMetadataBuilder.knnQuery = true; + break; + default: + } + } +} diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 0233597033180..89f8c543a2bee 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -333,7 +333,12 @@ public long buildTookInMillis() { @Override protected void doExecute(Task task, SearchRequest searchRequest, ActionListener listener) { - executeRequest((SearchTask) task, searchRequest, new SearchResponseActionListener(listener), AsyncSearchActionProvider::new); + executeRequest( + (SearchTask) task, + searchRequest, + new SearchResponseActionListener(listener, searchRequest), + AsyncSearchActionProvider::new + ); } void executeRequest( @@ -526,7 +531,7 @@ void executeRequest( // We set the keep alive to -1 to indicate that we don't need the pit id in the response. // This is needed since we delete the pit prior to sending the response so the id doesn't exist anymore. source.pointInTimeBuilder(new PointInTimeBuilder(resp.getPointInTimeId()).setKeepAlive(TimeValue.MINUS_ONE)); - var pitListener = new SearchResponseActionListener(delegate) { + var pitListener = new SearchResponseActionListener(delegate, original) { @Override public void onResponse(SearchResponse response) { // we need to close the PIT first so we delay the release of the response to after the closing @@ -2012,9 +2017,11 @@ private class SearchResponseActionListener extends DelegatingActionListener listener) { + SearchResponseActionListener(ActionListener listener, SearchRequest searchRequest) { super(listener); + this.searchRequest = searchRequest; if (listener instanceof SearchResponseActionListener srListener) { usageBuilder = srListener.usageBuilder; } else { @@ -2046,7 +2053,7 @@ public void setClient(Task task) { @Override public void onResponse(SearchResponse searchResponse) { try { - searchResponseMetrics.recordTookTime(searchResponse.getTookInMillis()); + searchResponseMetrics.recordTookTime(searchResponse.getTookInMillis(), searchRequest); SearchResponseMetrics.ResponseCountTotalStatus responseCountTotalStatus = SearchResponseMetrics.ResponseCountTotalStatus.SUCCESS; if (searchResponse.getShardFailures() != null && searchResponse.getShardFailures().length > 0) { diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java index b232cd16ba65e..c83a72f55059e 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java @@ -60,7 +60,7 @@ protected void doExecute(Task task, SearchScrollRequest request, ActionListener< @Override public void onResponse(SearchResponse searchResponse) { try { - searchResponseMetrics.recordTookTime(searchResponse.getTookInMillis()); + searchResponseMetrics.recordTookTimeForSearchScroll(searchResponse.getTookInMillis()); SearchResponseMetrics.ResponseCountTotalStatus responseCountTotalStatus = SearchResponseMetrics.ResponseCountTotalStatus.SUCCESS; if (searchResponse.getShardFailures() != null && searchResponse.getShardFailures().length > 0) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java index bec0ddb17f6ab..c9ed898cbc52f 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java @@ -9,6 +9,8 @@ package org.elasticsearch.rest.action.search; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchRequestIntrospector; import org.elasticsearch.telemetry.metric.LongCounter; import org.elasticsearch.telemetry.metric.LongHistogram; import org.elasticsearch.telemetry.metric.MeterRegistry; @@ -66,8 +68,14 @@ private SearchResponseMetrics(LongHistogram tookDurationTotalMillisHistogram, Lo this.responseCountTotalCounter = responseCountTotalCounter; } - public long recordTookTime(long tookTime) { - tookDurationTotalMillisHistogram.record(tookTime); + public long recordTookTimeForSearchScroll(long tookTime) { + tookDurationTotalMillisHistogram.record(tookTime, SearchRequestIntrospector.SEARCH_SCROLL_ATTRIBUTES); + return tookTime; + } + + public long recordTookTime(long tookTime, SearchRequest searchRequest) { + SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); + tookDurationTotalMillisHistogram.record(tookTime, queryMetadata.toAttributesMap()); return tookTime; } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestIntrospectorTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestIntrospectorTests.java new file mode 100644 index 0000000000000..d03d7607633a3 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestIntrospectorTests.java @@ -0,0 +1,298 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.search; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.BoostingQueryBuilder; +import org.elasticsearch.index.query.ConstantScoreQueryBuilder; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.script.Script; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.builder.PointInTimeBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.GeoDistanceSortBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; +import org.elasticsearch.search.sort.ScriptSortBuilder; +import org.elasticsearch.test.ESTestCase; + +public class SearchRequestIntrospectorTests extends ESTestCase { + + public void testIndexIntrospectionSingleIndex() { + assertEquals(".kibana", SearchRequestIntrospector.introspectIndices(new String[] { ".kibana_task_manager" })); + assertEquals(".kibana", SearchRequestIntrospector.introspectIndices(new String[] { ".kibana_9_2_0" })); + assertEquals(".kibana", SearchRequestIntrospector.introspectIndices(new String[] { ".kibana_ingest_9_2_0" })); + assertEquals(".kibana", SearchRequestIntrospector.introspectIndices(new String[] { ".kibana_security_solution" })); + assertEquals(".fleet", SearchRequestIntrospector.introspectIndices(new String[] { ".fleet-agents" })); + assertEquals(".ml", SearchRequestIntrospector.introspectIndices(new String[] { ".ml-anomalies" })); + assertEquals(".ml", SearchRequestIntrospector.introspectIndices(new String[] { ".ml-notifications" })); + assertEquals(".slo", SearchRequestIntrospector.introspectIndices(new String[] { ".slo" })); + assertEquals(".alerts", SearchRequestIntrospector.introspectIndices(new String[] { ".alerts" })); + assertEquals(".elastic", SearchRequestIntrospector.introspectIndices(new String[] { ".elastic" })); + assertEquals(".ds-", SearchRequestIntrospector.introspectIndices(new String[] { ".ds-test1" })); + assertEquals(".ds-", SearchRequestIntrospector.introspectIndices(new String[] { ".ds-test2" })); + assertEquals(".ds-", SearchRequestIntrospector.introspectIndices(new String[] { ".ds-test3" })); + + assertEquals(".others", SearchRequestIntrospector.introspectIndices(new String[] { ".a" })); + assertEquals(".others", SearchRequestIntrospector.introspectIndices(new String[] { ".abcde" })); + + assertEquals("user", SearchRequestIntrospector.introspectIndices(new String[] { "a" })); + assertEquals("user", SearchRequestIntrospector.introspectIndices(new String[] { "ab" })); + assertEquals("user", SearchRequestIntrospector.introspectIndices(new String[] { "abc" })); + assertEquals("user", SearchRequestIntrospector.introspectIndices(new String[] { "abcd" })); + + String indexName = "a" + randomAlphaOfLengthBetween(3, 10); + assertEquals("user", SearchRequestIntrospector.introspectIndices(new String[] { indexName })); + } + + public void testIndexIntrospectionMultipleIndices() { + int length = randomIntBetween(2, 10); + String[] indices = new String[length]; + for (int i = 0; i < length; i++) { + indices[i] = randomAlphaOfLengthBetween(3, 10); + } + assertEquals("user", SearchRequestIntrospector.introspectIndices(indices)); + } + + public void testPrimarySortIntrospection() { + assertEquals("@timestamp", SearchRequestIntrospector.introspectPrimarySort(new FieldSortBuilder("@timestamp"))); + assertEquals("event.ingested", SearchRequestIntrospector.introspectPrimarySort(new FieldSortBuilder("event.ingested"))); + assertEquals("_doc", SearchRequestIntrospector.introspectPrimarySort(new FieldSortBuilder("_doc"))); + assertEquals("field", SearchRequestIntrospector.introspectPrimarySort(new FieldSortBuilder(randomAlphaOfLengthBetween(3, 10)))); + assertEquals("_score", SearchRequestIntrospector.introspectPrimarySort(new ScoreSortBuilder())); + assertEquals( + "_geo_distance", + SearchRequestIntrospector.introspectPrimarySort(new GeoDistanceSortBuilder(randomAlphaOfLengthBetween(3, 10), 1d, 1d)) + ); + assertEquals( + "_script", + SearchRequestIntrospector.introspectPrimarySort( + new ScriptSortBuilder(new Script("id"), randomFrom(ScriptSortBuilder.ScriptSortType.values())) + ) + ); + } + + public void testQueryTypeIntrospection() { + { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + assertEquals("hits_only", SearchRequestIntrospector.introspectQueryType(searchSourceBuilder)); + } + { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.size(randomIntBetween(1, 100)); + assertEquals("hits_only", SearchRequestIntrospector.introspectQueryType(searchSourceBuilder)); + } + { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.size(0); + assertEquals("count_only", SearchRequestIntrospector.introspectQueryType(searchSourceBuilder)); + } + { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.aggregation(new TermsAggregationBuilder("test")); + assertEquals("hits_and_aggs", SearchRequestIntrospector.introspectQueryType(searchSourceBuilder)); + } + { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.size(randomIntBetween(1, 100)); + searchSourceBuilder.aggregation(new TermsAggregationBuilder("test")); + assertEquals("hits_and_aggs", SearchRequestIntrospector.introspectQueryType(searchSourceBuilder)); + } + { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.size(0); + searchSourceBuilder.aggregation(new TermsAggregationBuilder("test")); + assertEquals("aggs_only", SearchRequestIntrospector.introspectQueryType(searchSourceBuilder)); + } + } + + public void testIntrospectSearchRequest() { + { + SearchRequest searchRequest = new SearchRequest(); + SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); + assertEquals("user", queryMetadata.target()); + assertEquals("hits_only", queryMetadata.queryType()); + assertFalse(queryMetadata.knn()); + assertEquals("_score", queryMetadata.primarySort()); + assertEquals(0, queryMetadata.rangeFields().length); + assertNull(queryMetadata.pitOrScroll()); + } + { + SearchRequest searchRequest = new SearchRequest(); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.pointInTimeBuilder(new PointInTimeBuilder(BytesArray.EMPTY)); + SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); + assertEquals("user", queryMetadata.target()); + assertEquals("hits_only", queryMetadata.queryType()); + assertFalse(queryMetadata.knn()); + assertEquals("_score", queryMetadata.primarySort()); + assertEquals(0, queryMetadata.rangeFields().length); + assertEquals("pit", queryMetadata.pitOrScroll()); + } + { + SearchRequest searchRequest = new SearchRequest(); + searchRequest.scroll(new TimeValue(randomIntBetween(1, 10))); + SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); + assertEquals("user", queryMetadata.target()); + assertEquals("hits_only", queryMetadata.queryType()); + assertFalse(queryMetadata.knn()); + assertEquals("_score", queryMetadata.primarySort()); + assertEquals(0, queryMetadata.rangeFields().length); + assertEquals("scroll", queryMetadata.pitOrScroll()); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + searchSourceBuilder.query(new RangeQueryBuilder("@timestamp")); + SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); + assertEquals("user", queryMetadata.target()); + assertEquals("hits_only", queryMetadata.queryType()); + assertFalse(queryMetadata.knn()); + assertEquals("@timestamp", queryMetadata.primarySort()); + assertArrayEquals(new String[] { "@timestamp" }, queryMetadata.rangeFields()); + assertNull(queryMetadata.pitOrScroll()); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + int numBool = randomIntBetween(2, 10); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + for (int i = 0; i < numBool; i++) { + BoolQueryBuilder boolQueryBuilderNew = new BoolQueryBuilder(); + boolQueryBuilder.must(boolQueryBuilderNew); + boolQueryBuilder = boolQueryBuilderNew; + } + boolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); + searchSourceBuilder.query(boolQueryBuilder); + if (randomBoolean()) { + boolQueryBuilder.should(new RangeQueryBuilder("event.ingested")); + } + SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); + assertEquals("user", queryMetadata.target()); + assertEquals("hits_only", queryMetadata.queryType()); + assertFalse(queryMetadata.knn()); + assertEquals("@timestamp", queryMetadata.primarySort()); + assertArrayEquals(new String[] { "@timestamp" }, queryMetadata.rangeFields()); + assertNull(queryMetadata.pitOrScroll()); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + int numBool = randomIntBetween(2, 10); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + for (int i = 0; i < numBool; i++) { + BoolQueryBuilder boolQueryBuilderNew = new BoolQueryBuilder(); + boolQueryBuilder.filter(boolQueryBuilderNew); + boolQueryBuilder = boolQueryBuilderNew; + } + if (randomBoolean()) { + boolQueryBuilder.should(new RangeQueryBuilder("event.ingested")); + } + + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp")); + searchSourceBuilder.query(boolQueryBuilder); + SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); + assertEquals("user", queryMetadata.target()); + assertEquals("hits_only", queryMetadata.queryType()); + assertFalse(queryMetadata.knn()); + assertEquals("@timestamp", queryMetadata.primarySort()); + assertArrayEquals(new String[] { "@timestamp" }, queryMetadata.rangeFields()); + assertNull(queryMetadata.pitOrScroll()); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); + boolQueryBuilder.must(new RangeQueryBuilder("event.ingested")); + boolQueryBuilder.must(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10))); + searchSourceBuilder.query(boolQueryBuilder); + SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); + assertEquals("user", queryMetadata.target()); + assertEquals("hits_only", queryMetadata.queryType()); + assertFalse(queryMetadata.knn()); + assertEquals("@timestamp", queryMetadata.primarySort()); + assertArrayEquals(new String[] { "@timestamp", "event.ingested", "field" }, queryMetadata.rangeFields()); + assertNull(queryMetadata.pitOrScroll()); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.should(new RangeQueryBuilder("@timestamp")); + searchSourceBuilder.query(boolQueryBuilder); + SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); + assertEquals("user", queryMetadata.target()); + assertEquals("hits_only", queryMetadata.queryType()); + assertFalse(queryMetadata.knn()); + assertEquals("@timestamp", queryMetadata.primarySort()); + assertArrayEquals(new String[] { "@timestamp" }, queryMetadata.rangeFields()); + assertNull(queryMetadata.pitOrScroll()); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.should(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10))); + searchSourceBuilder.query(boolQueryBuilder); + SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); + assertEquals("user", queryMetadata.target()); + assertEquals("hits_only", queryMetadata.queryType()); + assertFalse(queryMetadata.knn()); + assertEquals("@timestamp", queryMetadata.primarySort()); + assertArrayEquals(new String[] { "field" }, queryMetadata.rangeFields()); + assertNull(queryMetadata.pitOrScroll()); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + searchSourceBuilder.query(new ConstantScoreQueryBuilder(new RangeQueryBuilder("@timestamp"))); + SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); + assertEquals("user", queryMetadata.target()); + assertEquals("hits_only", queryMetadata.queryType()); + assertFalse(queryMetadata.knn()); + assertEquals("@timestamp", queryMetadata.primarySort()); + assertArrayEquals(new String[] { "@timestamp" }, queryMetadata.rangeFields()); + assertNull(queryMetadata.pitOrScroll()); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + searchSourceBuilder.query(new BoostingQueryBuilder(new RangeQueryBuilder("@timestamp"), new MatchAllQueryBuilder())); + SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); + assertEquals("user", queryMetadata.target()); + assertEquals("hits_only", queryMetadata.queryType()); + assertFalse(queryMetadata.knn()); + assertEquals("@timestamp", queryMetadata.primarySort()); + assertArrayEquals(new String[] { "@timestamp" }, queryMetadata.rangeFields()); + assertNull(queryMetadata.pitOrScroll()); + } + } +} From 91c763fbad113f2d071ddf4cce01aa9a34a047ff Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 5 Sep 2025 17:45:58 +0200 Subject: [PATCH 02/10] Update docs/changelog/134232.yaml --- docs/changelog/134232.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/134232.yaml diff --git a/docs/changelog/134232.yaml b/docs/changelog/134232.yaml new file mode 100644 index 0000000000000..5e51297e7ff95 --- /dev/null +++ b/docs/changelog/134232.yaml @@ -0,0 +1,5 @@ +pr: 134232 +summary: Add relevant attributes to search took time APM metrics +area: Search +type: enhancement +issues: [] From 3c54484519a1551960fefaf86ddca176e2905369 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 12 Sep 2025 11:34:38 +0200 Subject: [PATCH 03/10] iter --- ... => SearchRequestAttributesExtractor.java} | 210 ++++++------ .../action/search/SearchResponseMetrics.java | 8 +- ...SearchRequestAttributesExtractorTests.java | 280 ++++++++++++++++ .../SearchRequestIntrospectorTests.java | 298 ------------------ .../SearchTookTimeTelemetryTests.java | 166 +++++++++- .../AsyncSearchTookTimeTelemetryTests.java | 19 +- 6 files changed, 566 insertions(+), 415 deletions(-) rename server/src/main/java/org/elasticsearch/action/search/{SearchRequestIntrospector.java => SearchRequestAttributesExtractor.java} (53%) create mode 100644 server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java delete mode 100644 server/src/test/java/org/elasticsearch/action/search/SearchRequestIntrospectorTests.java diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestIntrospector.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java similarity index 53% rename from server/src/main/java/org/elasticsearch/action/search/SearchRequestIntrospector.java rename to server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java index 8862269ae47a0..2bc99e68c2f68 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestIntrospector.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java @@ -9,7 +9,8 @@ package org.elasticsearch.action.search; -import org.elasticsearch.common.Strings; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.BoostingQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; @@ -24,23 +25,26 @@ import org.elasticsearch.search.vectors.KnnVectorQueryBuilder; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; /** * Used to introspect a search request and extract metadata from it around the features it uses. + * Given that the purpose of this class is to extract metrics attributes, it should do its best + * to extract the minimum set of needed information without hurting performance, and without + * ever breaking: if something goes wrong around extracting attributes, it should skip extracting + * them as opposed to failing the search. */ -public class SearchRequestIntrospector { +public class SearchRequestAttributesExtractor { + private static final Logger logger = LogManager.getLogger(SearchRequestAttributesExtractor.class); /** * Introspects the provided search request and extracts metadata from it about some of its characteristics. * */ - public static QueryMetadata introspectSearchRequest(SearchRequest searchRequest) { - - String target = introspectIndices(searchRequest.indices()); + public static Map extractAttributes(SearchRequest searchRequest) { + String target = extractIndices(searchRequest.indices()); String pitOrScroll = null; if (searchRequest.scroll() != null) { @@ -49,7 +53,7 @@ public static QueryMetadata introspectSearchRequest(SearchRequest searchRequest) SearchSourceBuilder searchSourceBuilder = searchRequest.source(); if (searchSourceBuilder == null) { - return new QueryMetadata(target, ScoreSortBuilder.NAME, HITS_ONLY, false, Strings.EMPTY_ARRAY, pitOrScroll); + return buildAttributesMap(target, ScoreSortBuilder.NAME, HITS_ONLY, false, null, pitOrScroll); } if (searchSourceBuilder.pointInTimeBuilder() != null) { @@ -60,34 +64,25 @@ public static QueryMetadata introspectSearchRequest(SearchRequest searchRequest) if (searchSourceBuilder.sorts() == null || searchSourceBuilder.sorts().isEmpty()) { primarySort = ScoreSortBuilder.NAME; } else { - primarySort = introspectPrimarySort(searchSourceBuilder.sorts().getFirst()); + primarySort = extractPrimarySort(searchSourceBuilder.sorts().getFirst()); } - final String queryType = introspectQueryType(searchSourceBuilder); + final String queryType = extractQueryType(searchSourceBuilder); QueryMetadataBuilder queryMetadataBuilder = new QueryMetadataBuilder(); if (searchSourceBuilder.query() != null) { - introspectQueryBuilder(searchSourceBuilder.query(), queryMetadataBuilder); + try { + introspectQueryBuilder(searchSourceBuilder.query(), queryMetadataBuilder); + } catch (Exception e) { + logger.error("Failed to extract query attribute", e); + } } final boolean hasKnn = searchSourceBuilder.knnSearch().isEmpty() == false || queryMetadataBuilder.knnQuery; - - return new QueryMetadata( - target, - primarySort, - queryType, - hasKnn, - queryMetadataBuilder.rangeFields.toArray(new String[0]), - pitOrScroll - ); + return buildAttributesMap(target, primarySort, queryType, hasKnn, queryMetadataBuilder.getRangeFields(), pitOrScroll); } - private static final class QueryMetadataBuilder { - private boolean knnQuery = false; - private final List rangeFields = new ArrayList<>(); - } - - public record QueryMetadata( + private static Map buildAttributesMap( String target, String primarySort, String queryType, @@ -95,28 +90,39 @@ public record QueryMetadata( String[] rangeFields, String pitOrScroll ) { + Map attributes = new HashMap<>(5, 1.0f); + attributes.put(TARGET_ATTRIBUTE, target); + attributes.put(SORT_ATTRIBUTE, primarySort); + if (pitOrScroll == null) { + attributes.put(QUERY_TYPE_ATTRIBUTE, queryType); + } else { + attributes.put(QUERY_TYPE_ATTRIBUTE, new String[] { queryType, pitOrScroll }); + } - public Map toAttributesMap() { - Map attributes = new HashMap<>(); - attributes.put(TARGET_ATTRIBUTE, target); - attributes.put(SORT_ATTRIBUTE, primarySort); - if (pitOrScroll == null) { - attributes.put(QUERY_TYPE_ATTRIBUTE, queryType); - } else { - attributes.put(QUERY_TYPE_ATTRIBUTE, Arrays.asList(queryType, pitOrScroll)); - } - - attributes.put(KNN_ATTRIBUTE, knn); + attributes.put(KNN_ATTRIBUTE, knn); + if (rangeFields != null) { attributes.put(RANGES_ATTRIBUTE, rangeFields); - return attributes; + } + return attributes; + } + + private static final class QueryMetadataBuilder { + private boolean knnQuery = false; + private final List rangeFields = new ArrayList<>(); + + String[] getRangeFields() { + if (rangeFields.isEmpty()) { + return null; + } + return rangeFields.toArray(new String[0]); } } - private static final String TARGET_ATTRIBUTE = "target"; - private static final String SORT_ATTRIBUTE = "sort"; - private static final String QUERY_TYPE_ATTRIBUTE = "query_type"; - private static final String KNN_ATTRIBUTE = "knn"; - private static final String RANGES_ATTRIBUTE = "ranges"; + static final String TARGET_ATTRIBUTE = "target"; + static final String SORT_ATTRIBUTE = "sort"; + static final String QUERY_TYPE_ATTRIBUTE = "query_type"; + static final String KNN_ATTRIBUTE = "knn"; + static final String RANGES_ATTRIBUTE = "ranges"; private static final String TARGET_KIBANA = ".kibana"; private static final String TARGET_ML = ".ml"; @@ -127,39 +133,45 @@ public Map toAttributesMap() { private static final String TARGET_DS = ".ds-"; private static final String TARGET_OTHERS = ".others"; private static final String TARGET_USER = "user"; + private static final String ERROR = "error"; - static String introspectIndices(String[] indices) { - // Note that indices are expected to be resolved, meaning wildcards are not handled on purpose - // If indices resolve to data streams, the name of the data stream is returned as opposed to its backing indices - if (indices.length == 1) { - String index = indices[0]; - if (index.startsWith(".")) { - if (index.startsWith(TARGET_KIBANA)) { - return TARGET_KIBANA; - } - if (index.startsWith(TARGET_ML)) { - return TARGET_ML; - } - if (index.startsWith(TARGET_FLEET)) { - return TARGET_FLEET; - } - if (index.startsWith(TARGET_SLO)) { - return TARGET_SLO; - } - if (index.startsWith(TARGET_ALERTS)) { - return TARGET_ALERTS; - } - if (index.startsWith(TARGET_ELASTIC)) { - return TARGET_ELASTIC; - } - // this happens only when data streams backing indices are searched explicitly - if (index.startsWith(TARGET_DS)) { - return TARGET_DS; + static String extractIndices(String[] indices) { + try { + // Note that indices are expected to be resolved, meaning wildcards are not handled on purpose + // If indices resolve to data streams, the name of the data stream is returned as opposed to its backing indices + if (indices.length == 1) { + String index = indices[0]; + if (index.startsWith(".")) { + if (index.startsWith(TARGET_KIBANA)) { + return TARGET_KIBANA; + } + if (index.startsWith(TARGET_ML)) { + return TARGET_ML; + } + if (index.startsWith(TARGET_FLEET)) { + return TARGET_FLEET; + } + if (index.startsWith(TARGET_SLO)) { + return TARGET_SLO; + } + if (index.startsWith(TARGET_ALERTS)) { + return TARGET_ALERTS; + } + if (index.startsWith(TARGET_ELASTIC)) { + return TARGET_ELASTIC; + } + // this happens only when data streams backing indices are searched explicitly + if (index.startsWith(TARGET_DS)) { + return TARGET_DS; + } + return TARGET_OTHERS; } - return TARGET_OTHERS; } + return TARGET_USER; + } catch (Exception e) { + logger.error("Failed to extract indices attribute", e); + return ERROR; } - return TARGET_USER; } private static final String TIMESTAMP = "@timestamp"; @@ -167,16 +179,21 @@ static String introspectIndices(String[] indices) { private static final String _DOC = "_doc"; private static final String FIELD = "field"; - static String introspectPrimarySort(SortBuilder primarySortBuilder) { - if (primarySortBuilder instanceof FieldSortBuilder fieldSort) { - return switch (fieldSort.getFieldName()) { - case TIMESTAMP -> TIMESTAMP; - case EVENT_INGESTED -> EVENT_INGESTED; - case _DOC -> _DOC; - default -> FIELD; - }; + static String extractPrimarySort(SortBuilder primarySortBuilder) { + try { + if (primarySortBuilder instanceof FieldSortBuilder fieldSort) { + return switch (fieldSort.getFieldName()) { + case TIMESTAMP -> TIMESTAMP; + case EVENT_INGESTED -> EVENT_INGESTED; + case _DOC -> _DOC; + default -> FIELD; + }; + } + return primarySortBuilder.getWriteableName(); + } catch (Exception e) { + logger.error("Failed to extract primary sort attribute", e); + return ERROR; } - return primarySortBuilder.getWriteableName(); } private static final String HITS_AND_AGGS = "hits_and_aggs"; @@ -188,21 +205,26 @@ static String introspectPrimarySort(SortBuilder primarySortBuilder) { public static final Map SEARCH_SCROLL_ATTRIBUTES = Map.of(QUERY_TYPE_ATTRIBUTE, SCROLL); - static String introspectQueryType(SearchSourceBuilder searchSourceBuilder) { - int size = searchSourceBuilder.size(); - if (size == -1) { - size = SearchService.DEFAULT_SIZE; - } - if (searchSourceBuilder.aggregations() != null && size > 0) { - return HITS_AND_AGGS; - } - if (searchSourceBuilder.aggregations() != null) { - return AGGS_ONLY; - } - if (size > 0) { - return HITS_ONLY; + static String extractQueryType(SearchSourceBuilder searchSourceBuilder) { + try { + int size = searchSourceBuilder.size(); + if (size == -1) { + size = SearchService.DEFAULT_SIZE; + } + if (searchSourceBuilder.aggregations() != null && size > 0) { + return HITS_AND_AGGS; + } + if (searchSourceBuilder.aggregations() != null) { + return AGGS_ONLY; + } + if (size > 0) { + return HITS_ONLY; + } + return COUNT_ONLY; + } catch (Exception e) { + logger.error("Failed to extract query type attribute", e); + return ERROR; } - return COUNT_ONLY; } static void introspectQueryBuilder(QueryBuilder queryBuilder, QueryMetadataBuilder queryMetadataBuilder) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java index c9ed898cbc52f..e86f5e89bc8a6 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java @@ -10,7 +10,7 @@ package org.elasticsearch.rest.action.search; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchRequestIntrospector; +import org.elasticsearch.action.search.SearchRequestAttributesExtractor; import org.elasticsearch.telemetry.metric.LongCounter; import org.elasticsearch.telemetry.metric.LongHistogram; import org.elasticsearch.telemetry.metric.MeterRegistry; @@ -69,13 +69,13 @@ private SearchResponseMetrics(LongHistogram tookDurationTotalMillisHistogram, Lo } public long recordTookTimeForSearchScroll(long tookTime) { - tookDurationTotalMillisHistogram.record(tookTime, SearchRequestIntrospector.SEARCH_SCROLL_ATTRIBUTES); + tookDurationTotalMillisHistogram.record(tookTime, SearchRequestAttributesExtractor.SEARCH_SCROLL_ATTRIBUTES); return tookTime; } public long recordTookTime(long tookTime, SearchRequest searchRequest) { - SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); - tookDurationTotalMillisHistogram.record(tookTime, queryMetadata.toAttributesMap()); + Map attributes = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + tookDurationTotalMillisHistogram.record(tookTime, attributes); return tookTime; } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java new file mode 100644 index 0000000000000..31a5a3f82661b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java @@ -0,0 +1,280 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.search; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.BoostingQueryBuilder; +import org.elasticsearch.index.query.ConstantScoreQueryBuilder; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.script.Script; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.builder.PointInTimeBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.GeoDistanceSortBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; +import org.elasticsearch.search.sort.ScriptSortBuilder; +import org.elasticsearch.test.ESTestCase; + +import java.util.Map; + +public class SearchRequestAttributesExtractorTests extends ESTestCase { + + public void testIndexIntrospectionSingleIndex() { + assertEquals(".kibana", SearchRequestAttributesExtractor.extractIndices(new String[] { ".kibana_task_manager" })); + assertEquals(".kibana", SearchRequestAttributesExtractor.extractIndices(new String[] { ".kibana_9_2_0" })); + assertEquals(".kibana", SearchRequestAttributesExtractor.extractIndices(new String[] { ".kibana_ingest_9_2_0" })); + assertEquals(".kibana", SearchRequestAttributesExtractor.extractIndices(new String[] { ".kibana_security_solution" })); + assertEquals(".fleet", SearchRequestAttributesExtractor.extractIndices(new String[] { ".fleet-agents" })); + assertEquals(".ml", SearchRequestAttributesExtractor.extractIndices(new String[] { ".ml-anomalies" })); + assertEquals(".ml", SearchRequestAttributesExtractor.extractIndices(new String[] { ".ml-notifications" })); + assertEquals(".slo", SearchRequestAttributesExtractor.extractIndices(new String[] { ".slo" })); + assertEquals(".alerts", SearchRequestAttributesExtractor.extractIndices(new String[] { ".alerts" })); + assertEquals(".elastic", SearchRequestAttributesExtractor.extractIndices(new String[] { ".elastic" })); + assertEquals(".ds-", SearchRequestAttributesExtractor.extractIndices(new String[] { ".ds-test1" })); + assertEquals(".ds-", SearchRequestAttributesExtractor.extractIndices(new String[] { ".ds-test2" })); + assertEquals(".ds-", SearchRequestAttributesExtractor.extractIndices(new String[] { ".ds-test3" })); + + assertEquals(".others", SearchRequestAttributesExtractor.extractIndices(new String[] { ".a" })); + assertEquals(".others", SearchRequestAttributesExtractor.extractIndices(new String[] { ".abcde" })); + + assertEquals("user", SearchRequestAttributesExtractor.extractIndices(new String[] { "a" })); + assertEquals("user", SearchRequestAttributesExtractor.extractIndices(new String[] { "ab" })); + assertEquals("user", SearchRequestAttributesExtractor.extractIndices(new String[] { "abc" })); + assertEquals("user", SearchRequestAttributesExtractor.extractIndices(new String[] { "abcd" })); + + String indexName = "a" + randomAlphaOfLengthBetween(3, 10); + assertEquals("user", SearchRequestAttributesExtractor.extractIndices(new String[] { indexName })); + } + + public void testIndexIntrospectionMultipleIndices() { + int length = randomIntBetween(2, 10); + String[] indices = new String[length]; + for (int i = 0; i < length; i++) { + indices[i] = randomAlphaOfLengthBetween(3, 10); + } + assertEquals("user", SearchRequestAttributesExtractor.extractIndices(indices)); + } + + public void testPrimarySortIntrospection() { + assertEquals("@timestamp", SearchRequestAttributesExtractor.extractPrimarySort(new FieldSortBuilder("@timestamp"))); + assertEquals("event.ingested", SearchRequestAttributesExtractor.extractPrimarySort(new FieldSortBuilder("event.ingested"))); + assertEquals("_doc", SearchRequestAttributesExtractor.extractPrimarySort(new FieldSortBuilder("_doc"))); + assertEquals("field", SearchRequestAttributesExtractor.extractPrimarySort(new FieldSortBuilder(randomAlphaOfLengthBetween(3, 10)))); + assertEquals("_score", SearchRequestAttributesExtractor.extractPrimarySort(new ScoreSortBuilder())); + assertEquals( + "_geo_distance", + SearchRequestAttributesExtractor.extractPrimarySort(new GeoDistanceSortBuilder(randomAlphaOfLengthBetween(3, 10), 1d, 1d)) + ); + assertEquals( + "_script", + SearchRequestAttributesExtractor.extractPrimarySort( + new ScriptSortBuilder(new Script("id"), randomFrom(ScriptSortBuilder.ScriptSortType.values())) + ) + ); + } + + public void testQueryTypeIntrospection() { + { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + assertEquals("hits_only", SearchRequestAttributesExtractor.extractQueryType(searchSourceBuilder)); + } + { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.size(randomIntBetween(1, 100)); + assertEquals("hits_only", SearchRequestAttributesExtractor.extractQueryType(searchSourceBuilder)); + } + { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.size(0); + assertEquals("count_only", SearchRequestAttributesExtractor.extractQueryType(searchSourceBuilder)); + } + { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.aggregation(new TermsAggregationBuilder("test")); + assertEquals("hits_and_aggs", SearchRequestAttributesExtractor.extractQueryType(searchSourceBuilder)); + } + { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.size(randomIntBetween(1, 100)); + searchSourceBuilder.aggregation(new TermsAggregationBuilder("test")); + assertEquals("hits_and_aggs", SearchRequestAttributesExtractor.extractQueryType(searchSourceBuilder)); + } + { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.size(0); + searchSourceBuilder.aggregation(new TermsAggregationBuilder("test")); + assertEquals("aggs_only", SearchRequestAttributesExtractor.extractQueryType(searchSourceBuilder)); + } + } + + private static void assertAttributes( + Map attributes, + String target, + String primarySort, + String queryType, + boolean knn, + String[] rangeFields, + String pitOrScroll + ) { + + assertEquals(target, attributes.get(SearchRequestAttributesExtractor.TARGET_ATTRIBUTE)); + assertEquals(primarySort, attributes.get(SearchRequestAttributesExtractor.SORT_ATTRIBUTE)); + if (pitOrScroll == null) { + assertEquals(queryType, attributes.get(SearchRequestAttributesExtractor.QUERY_TYPE_ATTRIBUTE)); + } else { + String[] queryTypes = (String[]) attributes.get(SearchRequestAttributesExtractor.QUERY_TYPE_ATTRIBUTE); + assertEquals(queryType, queryTypes[0]); + assertEquals(pitOrScroll, queryTypes[1]); + } + assertEquals(knn, attributes.get(SearchRequestAttributesExtractor.KNN_ATTRIBUTE)); + if (rangeFields == null) { + assertFalse(attributes.containsKey(SearchRequestAttributesExtractor.RANGES_ATTRIBUTE)); + } else { + assertArrayEquals(rangeFields, (String[]) attributes.get(SearchRequestAttributesExtractor.RANGES_ATTRIBUTE)); + } + } + + public void testExtractAttributes() { + { + SearchRequest searchRequest = new SearchRequest(); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, null, null); + } + { + SearchRequest searchRequest = new SearchRequest(); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.pointInTimeBuilder(new PointInTimeBuilder(BytesArray.EMPTY)); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, null, "pit"); + } + { + SearchRequest searchRequest = new SearchRequest(); + searchRequest.scroll(new TimeValue(randomIntBetween(1, 10))); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, null, "scroll"); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + searchSourceBuilder.query(new RangeQueryBuilder("@timestamp")); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, new String[] { "@timestamp" }, null); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + int numBool = randomIntBetween(2, 10); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + for (int i = 0; i < numBool; i++) { + BoolQueryBuilder boolQueryBuilderNew = new BoolQueryBuilder(); + boolQueryBuilder.must(boolQueryBuilderNew); + boolQueryBuilder = boolQueryBuilderNew; + } + boolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); + searchSourceBuilder.query(boolQueryBuilder); + if (randomBoolean()) { + boolQueryBuilder.should(new RangeQueryBuilder("event.ingested")); + } + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, new String[] { "@timestamp" }, null); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + int numBool = randomIntBetween(2, 10); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + for (int i = 0; i < numBool; i++) { + BoolQueryBuilder boolQueryBuilderNew = new BoolQueryBuilder(); + boolQueryBuilder.filter(boolQueryBuilderNew); + boolQueryBuilder = boolQueryBuilderNew; + } + if (randomBoolean()) { + boolQueryBuilder.should(new RangeQueryBuilder("event.ingested")); + } + + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp")); + searchSourceBuilder.query(boolQueryBuilder); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, new String[] { "@timestamp" }, null); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); + boolQueryBuilder.must(new RangeQueryBuilder("event.ingested")); + boolQueryBuilder.must(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10))); + searchSourceBuilder.query(boolQueryBuilder); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + assertAttributes( + stringObjectMap, + "user", + "@timestamp", + "hits_only", + false, + new String[] { "@timestamp", "event.ingested", "field" }, + null + ); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.should(new RangeQueryBuilder("@timestamp")); + searchSourceBuilder.query(boolQueryBuilder); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, new String[] { "@timestamp" }, null); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.should(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10))); + searchSourceBuilder.query(boolQueryBuilder); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, new String[] { "field" }, null); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + searchSourceBuilder.query(new ConstantScoreQueryBuilder(new RangeQueryBuilder("@timestamp"))); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, new String[] { "@timestamp" }, null); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + searchSourceBuilder.query(new BoostingQueryBuilder(new RangeQueryBuilder("@timestamp"), new MatchAllQueryBuilder())); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, new String[] { "@timestamp" }, null); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestIntrospectorTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestIntrospectorTests.java deleted file mode 100644 index d03d7607633a3..0000000000000 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestIntrospectorTests.java +++ /dev/null @@ -1,298 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.action.search; - -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.BoostingQueryBuilder; -import org.elasticsearch.index.query.ConstantScoreQueryBuilder; -import org.elasticsearch.index.query.MatchAllQueryBuilder; -import org.elasticsearch.index.query.RangeQueryBuilder; -import org.elasticsearch.script.Script; -import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.builder.PointInTimeBuilder; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.elasticsearch.search.sort.GeoDistanceSortBuilder; -import org.elasticsearch.search.sort.ScoreSortBuilder; -import org.elasticsearch.search.sort.ScriptSortBuilder; -import org.elasticsearch.test.ESTestCase; - -public class SearchRequestIntrospectorTests extends ESTestCase { - - public void testIndexIntrospectionSingleIndex() { - assertEquals(".kibana", SearchRequestIntrospector.introspectIndices(new String[] { ".kibana_task_manager" })); - assertEquals(".kibana", SearchRequestIntrospector.introspectIndices(new String[] { ".kibana_9_2_0" })); - assertEquals(".kibana", SearchRequestIntrospector.introspectIndices(new String[] { ".kibana_ingest_9_2_0" })); - assertEquals(".kibana", SearchRequestIntrospector.introspectIndices(new String[] { ".kibana_security_solution" })); - assertEquals(".fleet", SearchRequestIntrospector.introspectIndices(new String[] { ".fleet-agents" })); - assertEquals(".ml", SearchRequestIntrospector.introspectIndices(new String[] { ".ml-anomalies" })); - assertEquals(".ml", SearchRequestIntrospector.introspectIndices(new String[] { ".ml-notifications" })); - assertEquals(".slo", SearchRequestIntrospector.introspectIndices(new String[] { ".slo" })); - assertEquals(".alerts", SearchRequestIntrospector.introspectIndices(new String[] { ".alerts" })); - assertEquals(".elastic", SearchRequestIntrospector.introspectIndices(new String[] { ".elastic" })); - assertEquals(".ds-", SearchRequestIntrospector.introspectIndices(new String[] { ".ds-test1" })); - assertEquals(".ds-", SearchRequestIntrospector.introspectIndices(new String[] { ".ds-test2" })); - assertEquals(".ds-", SearchRequestIntrospector.introspectIndices(new String[] { ".ds-test3" })); - - assertEquals(".others", SearchRequestIntrospector.introspectIndices(new String[] { ".a" })); - assertEquals(".others", SearchRequestIntrospector.introspectIndices(new String[] { ".abcde" })); - - assertEquals("user", SearchRequestIntrospector.introspectIndices(new String[] { "a" })); - assertEquals("user", SearchRequestIntrospector.introspectIndices(new String[] { "ab" })); - assertEquals("user", SearchRequestIntrospector.introspectIndices(new String[] { "abc" })); - assertEquals("user", SearchRequestIntrospector.introspectIndices(new String[] { "abcd" })); - - String indexName = "a" + randomAlphaOfLengthBetween(3, 10); - assertEquals("user", SearchRequestIntrospector.introspectIndices(new String[] { indexName })); - } - - public void testIndexIntrospectionMultipleIndices() { - int length = randomIntBetween(2, 10); - String[] indices = new String[length]; - for (int i = 0; i < length; i++) { - indices[i] = randomAlphaOfLengthBetween(3, 10); - } - assertEquals("user", SearchRequestIntrospector.introspectIndices(indices)); - } - - public void testPrimarySortIntrospection() { - assertEquals("@timestamp", SearchRequestIntrospector.introspectPrimarySort(new FieldSortBuilder("@timestamp"))); - assertEquals("event.ingested", SearchRequestIntrospector.introspectPrimarySort(new FieldSortBuilder("event.ingested"))); - assertEquals("_doc", SearchRequestIntrospector.introspectPrimarySort(new FieldSortBuilder("_doc"))); - assertEquals("field", SearchRequestIntrospector.introspectPrimarySort(new FieldSortBuilder(randomAlphaOfLengthBetween(3, 10)))); - assertEquals("_score", SearchRequestIntrospector.introspectPrimarySort(new ScoreSortBuilder())); - assertEquals( - "_geo_distance", - SearchRequestIntrospector.introspectPrimarySort(new GeoDistanceSortBuilder(randomAlphaOfLengthBetween(3, 10), 1d, 1d)) - ); - assertEquals( - "_script", - SearchRequestIntrospector.introspectPrimarySort( - new ScriptSortBuilder(new Script("id"), randomFrom(ScriptSortBuilder.ScriptSortType.values())) - ) - ); - } - - public void testQueryTypeIntrospection() { - { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - assertEquals("hits_only", SearchRequestIntrospector.introspectQueryType(searchSourceBuilder)); - } - { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.size(randomIntBetween(1, 100)); - assertEquals("hits_only", SearchRequestIntrospector.introspectQueryType(searchSourceBuilder)); - } - { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.size(0); - assertEquals("count_only", SearchRequestIntrospector.introspectQueryType(searchSourceBuilder)); - } - { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.aggregation(new TermsAggregationBuilder("test")); - assertEquals("hits_and_aggs", SearchRequestIntrospector.introspectQueryType(searchSourceBuilder)); - } - { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.size(randomIntBetween(1, 100)); - searchSourceBuilder.aggregation(new TermsAggregationBuilder("test")); - assertEquals("hits_and_aggs", SearchRequestIntrospector.introspectQueryType(searchSourceBuilder)); - } - { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.size(0); - searchSourceBuilder.aggregation(new TermsAggregationBuilder("test")); - assertEquals("aggs_only", SearchRequestIntrospector.introspectQueryType(searchSourceBuilder)); - } - } - - public void testIntrospectSearchRequest() { - { - SearchRequest searchRequest = new SearchRequest(); - SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); - assertEquals("user", queryMetadata.target()); - assertEquals("hits_only", queryMetadata.queryType()); - assertFalse(queryMetadata.knn()); - assertEquals("_score", queryMetadata.primarySort()); - assertEquals(0, queryMetadata.rangeFields().length); - assertNull(queryMetadata.pitOrScroll()); - } - { - SearchRequest searchRequest = new SearchRequest(); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchRequest.source(searchSourceBuilder); - searchSourceBuilder.pointInTimeBuilder(new PointInTimeBuilder(BytesArray.EMPTY)); - SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); - assertEquals("user", queryMetadata.target()); - assertEquals("hits_only", queryMetadata.queryType()); - assertFalse(queryMetadata.knn()); - assertEquals("_score", queryMetadata.primarySort()); - assertEquals(0, queryMetadata.rangeFields().length); - assertEquals("pit", queryMetadata.pitOrScroll()); - } - { - SearchRequest searchRequest = new SearchRequest(); - searchRequest.scroll(new TimeValue(randomIntBetween(1, 10))); - SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); - assertEquals("user", queryMetadata.target()); - assertEquals("hits_only", queryMetadata.queryType()); - assertFalse(queryMetadata.knn()); - assertEquals("_score", queryMetadata.primarySort()); - assertEquals(0, queryMetadata.rangeFields().length); - assertEquals("scroll", queryMetadata.pitOrScroll()); - } - { - SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchRequest.source(searchSourceBuilder); - searchSourceBuilder.sort("@timestamp"); - searchSourceBuilder.query(new RangeQueryBuilder("@timestamp")); - SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); - assertEquals("user", queryMetadata.target()); - assertEquals("hits_only", queryMetadata.queryType()); - assertFalse(queryMetadata.knn()); - assertEquals("@timestamp", queryMetadata.primarySort()); - assertArrayEquals(new String[] { "@timestamp" }, queryMetadata.rangeFields()); - assertNull(queryMetadata.pitOrScroll()); - } - { - SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchRequest.source(searchSourceBuilder); - searchSourceBuilder.sort("@timestamp"); - int numBool = randomIntBetween(2, 10); - BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - for (int i = 0; i < numBool; i++) { - BoolQueryBuilder boolQueryBuilderNew = new BoolQueryBuilder(); - boolQueryBuilder.must(boolQueryBuilderNew); - boolQueryBuilder = boolQueryBuilderNew; - } - boolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); - searchSourceBuilder.query(boolQueryBuilder); - if (randomBoolean()) { - boolQueryBuilder.should(new RangeQueryBuilder("event.ingested")); - } - SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); - assertEquals("user", queryMetadata.target()); - assertEquals("hits_only", queryMetadata.queryType()); - assertFalse(queryMetadata.knn()); - assertEquals("@timestamp", queryMetadata.primarySort()); - assertArrayEquals(new String[] { "@timestamp" }, queryMetadata.rangeFields()); - assertNull(queryMetadata.pitOrScroll()); - } - { - SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchRequest.source(searchSourceBuilder); - searchSourceBuilder.sort("@timestamp"); - int numBool = randomIntBetween(2, 10); - BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - for (int i = 0; i < numBool; i++) { - BoolQueryBuilder boolQueryBuilderNew = new BoolQueryBuilder(); - boolQueryBuilder.filter(boolQueryBuilderNew); - boolQueryBuilder = boolQueryBuilderNew; - } - if (randomBoolean()) { - boolQueryBuilder.should(new RangeQueryBuilder("event.ingested")); - } - - boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp")); - searchSourceBuilder.query(boolQueryBuilder); - SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); - assertEquals("user", queryMetadata.target()); - assertEquals("hits_only", queryMetadata.queryType()); - assertFalse(queryMetadata.knn()); - assertEquals("@timestamp", queryMetadata.primarySort()); - assertArrayEquals(new String[] { "@timestamp" }, queryMetadata.rangeFields()); - assertNull(queryMetadata.pitOrScroll()); - } - { - SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchRequest.source(searchSourceBuilder); - searchSourceBuilder.sort("@timestamp"); - BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - boolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); - boolQueryBuilder.must(new RangeQueryBuilder("event.ingested")); - boolQueryBuilder.must(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10))); - searchSourceBuilder.query(boolQueryBuilder); - SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); - assertEquals("user", queryMetadata.target()); - assertEquals("hits_only", queryMetadata.queryType()); - assertFalse(queryMetadata.knn()); - assertEquals("@timestamp", queryMetadata.primarySort()); - assertArrayEquals(new String[] { "@timestamp", "event.ingested", "field" }, queryMetadata.rangeFields()); - assertNull(queryMetadata.pitOrScroll()); - } - { - SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchRequest.source(searchSourceBuilder); - searchSourceBuilder.sort("@timestamp"); - BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - boolQueryBuilder.should(new RangeQueryBuilder("@timestamp")); - searchSourceBuilder.query(boolQueryBuilder); - SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); - assertEquals("user", queryMetadata.target()); - assertEquals("hits_only", queryMetadata.queryType()); - assertFalse(queryMetadata.knn()); - assertEquals("@timestamp", queryMetadata.primarySort()); - assertArrayEquals(new String[] { "@timestamp" }, queryMetadata.rangeFields()); - assertNull(queryMetadata.pitOrScroll()); - } - { - SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchRequest.source(searchSourceBuilder); - searchSourceBuilder.sort("@timestamp"); - BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - boolQueryBuilder.should(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10))); - searchSourceBuilder.query(boolQueryBuilder); - SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); - assertEquals("user", queryMetadata.target()); - assertEquals("hits_only", queryMetadata.queryType()); - assertFalse(queryMetadata.knn()); - assertEquals("@timestamp", queryMetadata.primarySort()); - assertArrayEquals(new String[] { "field" }, queryMetadata.rangeFields()); - assertNull(queryMetadata.pitOrScroll()); - } - { - SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchRequest.source(searchSourceBuilder); - searchSourceBuilder.sort("@timestamp"); - searchSourceBuilder.query(new ConstantScoreQueryBuilder(new RangeQueryBuilder("@timestamp"))); - SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); - assertEquals("user", queryMetadata.target()); - assertEquals("hits_only", queryMetadata.queryType()); - assertFalse(queryMetadata.knn()); - assertEquals("@timestamp", queryMetadata.primarySort()); - assertArrayEquals(new String[] { "@timestamp" }, queryMetadata.rangeFields()); - assertNull(queryMetadata.pitOrScroll()); - } - { - SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchRequest.source(searchSourceBuilder); - searchSourceBuilder.sort("@timestamp"); - searchSourceBuilder.query(new BoostingQueryBuilder(new RangeQueryBuilder("@timestamp"), new MatchAllQueryBuilder())); - SearchRequestIntrospector.QueryMetadata queryMetadata = SearchRequestIntrospector.introspectSearchRequest(searchRequest); - assertEquals("user", queryMetadata.target()); - assertEquals("hits_only", queryMetadata.queryType()); - assertFalse(queryMetadata.knn()); - assertEquals("@timestamp", queryMetadata.primarySort()); - assertArrayEquals(new String[] { "@timestamp" }, queryMetadata.rangeFields()); - assertNull(queryMetadata.pitOrScroll()); - } - } -} diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java index 2c000314c7dad..35760efe6928b 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java @@ -9,15 +9,18 @@ package org.elasticsearch.search.TelemetryMetrics; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.search.MultiSearchRequestBuilder; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.rest.RestUtils; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.telemetry.Measurement; import org.elasticsearch.telemetry.TestTelemetryPlugin; @@ -29,6 +32,8 @@ import java.util.Collection; import java.util.Comparator; import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery; @@ -47,18 +52,8 @@ protected boolean resetNodeAfterTest() { @Before public void setUpIndex() { - var num_primaries = randomIntBetween(1, 4); - createIndex( - indexName, - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, num_primaries) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .build() - ); - ensureGreen(indexName); - - prepareIndex(indexName).setId("1").setSource("body", "foo").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex(indexName).setId("2").setSource("body", "foo").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(indexName).setId("1").setSource("body", "foo", "@timestamp", "2024-11-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(indexName).setId("2").setSource("body", "foo", "@timestamp", "2024-12-01").setRefreshPolicy(IMMEDIATE).get(); } @After @@ -82,7 +77,57 @@ public void testSimpleQuery() { List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); assertEquals(1, measurements.size()); - assertEquals(searchResponse.getTook().millis(), measurements.getFirst().getLong()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertSimpleQueryAttributes(measurement.attributes()); + } + + public void testSimpleQueryAgainstWildcardExpression() { + SearchResponse searchResponse = client().prepareSearch("*").setQuery(simpleQueryStringQuery("foo")).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1", "2"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertSimpleQueryAttributes(measurement.attributes()); + } + + public void testSimpleQueryAgainstAlias() { + IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest( + RestUtils.REST_MASTER_TIMEOUT_DEFAULT, + new TimeValue(30, TimeUnit.SECONDS) + ); + indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add().indices(indexName).alias("alias")); + IndicesAliasesResponse indicesAliasesResponse = client().admin().indices().aliases(indicesAliasesRequest).actionGet(); + assertFalse(indicesAliasesResponse.hasErrors()); + SearchResponse searchResponse = client().prepareSearch("alias").setQuery(simpleQueryStringQuery("foo")).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1", "2"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertSimpleQueryAttributes(measurement.attributes()); + + } + + private static void assertSimpleQueryAttributes(Map attributes) { + assertEquals(4, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals(false, attributes.get("knn")); } public void testMultiSearch() { @@ -113,6 +158,7 @@ public void testMultiSearch() { int i = 0; for (Measurement measurement : measurements) { assertEquals(tookTimes.get(i++).longValue(), measurement.getLong()); + assertSimpleQueryAttributes(measurement.attributes()); } } @@ -123,17 +169,105 @@ public void testScroll() { client().prepareSearch(indexName).setSize(1).setQuery(simpleQueryStringQuery("foo")), 2, (respNum, response) -> { - if (respNum <= 2) { + if (respNum == 1) { + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement( + TOOK_DURATION_TOTAL_HISTOGRAM_NAME + ); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + Map attributes = measurement.attributes(); + assertEquals(4, attributes.size()); + assertEquals("user", attributes.get("target")); + assertArrayEquals(new String[] { "hits_only", "scroll" }, (String[]) attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals(false, attributes.get("knn")); + } else { List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement( TOOK_DURATION_TOTAL_HISTOGRAM_NAME ); assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + Map attributes = measurement.attributes(); + assertEquals(1, attributes.size()); + assertEquals("scroll", attributes.get("query_type")); } resetMeter(); } ); } + /** + * Make sure that despite can match and query rewrite, we see the time range filter and record its corresponding attribute + */ + public void testTimeRangeFilterNoResults() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from("2025-01-01")); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(indexName).setPreFilterShardSize(1).setQuery(boolQueryBuilder).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertTimeRangeAttributes(measurement.attributes()); + } + + /** + * Make sure that despite can match and query rewrite, we see the time range filter and record its corresponding attribute + */ + public void testTimeRangeFilterAllResults() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from("2024-10-01")); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(indexName).setPreFilterShardSize(1).setQuery(boolQueryBuilder).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1", "2"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertTimeRangeAttributes(measurement.attributes()); + } + + public void testTimeRangeFilterOneResults() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from("2024-12-01")); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(indexName).setPreFilterShardSize(1).setQuery(boolQueryBuilder).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "2"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertTimeRangeAttributes(measurement.attributes()); + } + + private static void assertTimeRangeAttributes(Map attributes) { + assertEquals(5, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals(false, attributes.get("knn")); + assertArrayEquals(new String[] { "@timestamp" }, (String[]) attributes.get("ranges")); + } + private void resetMeter() { getTestTelemetryPlugin().resetMeter(); } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java index f3dbc9ac67202..faca5e8dbec2a 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java @@ -30,6 +30,7 @@ import java.util.Collection; import java.util.List; +import java.util.Map; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery; @@ -95,7 +96,8 @@ public void testAsyncForegroundQuery() { } final List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); assertEquals(1, measurements.size()); - assertEquals(tookInMillis, measurements.getFirst().getLong()); + Measurement measurement = measurements.getFirst(); + assertEquals(tookInMillis, measurement.getLong()); for (int i = 0; i < randomIntBetween(3, 10); i++) { AsyncSearchResponse asyncSearchResponse2 = client().execute(GetAsyncSearchAction.INSTANCE, new GetAsyncResultRequest(id)) @@ -106,7 +108,8 @@ public void testAsyncForegroundQuery() { asyncSearchResponse2.decRef(); } assertEquals(1, measurements.size()); - assertEquals(tookInMillis, measurements.getFirst().getLong()); + assertEquals(tookInMillis, measurement.getLong()); + assertAttributes(measurement.attributes()); } } @@ -142,7 +145,9 @@ public void testAsyncBackgroundQuery() throws Exception { SearchResponse searchResponse = asyncSearchResponse2.getSearchResponse(); try { assertSearchHits(searchResponse, "1", "2"); - assertEquals(searchResponse.getTook().millis(), measurements.getFirst().getLong()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertAttributes(measurement.attributes()); } finally { asyncSearchResponse2.decRef(); } @@ -152,4 +157,12 @@ public void testAsyncBackgroundQuery() throws Exception { private TestTelemetryPlugin getTestTelemetryPlugin() { return getInstanceFromNode(PluginsService.class).filterPlugins(TestTelemetryPlugin.class).toList().getFirst(); } + + private static void assertAttributes(Map attributes) { + assertEquals(4, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals(false, attributes.get("knn")); + } } From 07838b2916dee7cab4f9a30c0b4307590ac82cbd Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 12 Sep 2025 12:18:48 +0200 Subject: [PATCH 04/10] iter --- .../SearchRequestAttributesExtractor.java | 19 +++++----- ...SearchRequestAttributesExtractorTests.java | 35 +++++++++++++++++++ 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java index 2bc99e68c2f68..ccf9ef682fd76 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java @@ -72,7 +72,7 @@ public static Map extractAttributes(SearchRequest searchRequest) QueryMetadataBuilder queryMetadataBuilder = new QueryMetadataBuilder(); if (searchSourceBuilder.query() != null) { try { - introspectQueryBuilder(searchSourceBuilder.query(), queryMetadataBuilder); + introspectQueryBuilder(searchSourceBuilder.query(), queryMetadataBuilder, 0); } catch (Exception e) { logger.error("Failed to extract query attribute", e); } @@ -227,29 +227,32 @@ static String extractQueryType(SearchSourceBuilder searchSourceBuilder) { } } - static void introspectQueryBuilder(QueryBuilder queryBuilder, QueryMetadataBuilder queryMetadataBuilder) { + private static void introspectQueryBuilder(QueryBuilder queryBuilder, QueryMetadataBuilder queryMetadataBuilder, int level) { + if (level > 20) { + return; + } switch (queryBuilder) { case BoolQueryBuilder bool: for (QueryBuilder must : bool.must()) { - introspectQueryBuilder(must, queryMetadataBuilder); + introspectQueryBuilder(must, queryMetadataBuilder, ++level); } for (QueryBuilder filter : bool.filter()) { - introspectQueryBuilder(filter, queryMetadataBuilder); + introspectQueryBuilder(filter, queryMetadataBuilder, ++level); } if (bool.must().isEmpty() && bool.filter().isEmpty() && bool.mustNot().isEmpty() && bool.should().size() == 1) { - introspectQueryBuilder(bool.should().getFirst(), queryMetadataBuilder); + introspectQueryBuilder(bool.should().getFirst(), queryMetadataBuilder, ++level); } // Note that should clauses are ignored unless there's only one that becomes mandatory // must_not clauses are also ignored for now break; case ConstantScoreQueryBuilder constantScore: - introspectQueryBuilder(constantScore.innerQuery(), queryMetadataBuilder); + introspectQueryBuilder(constantScore.innerQuery(), queryMetadataBuilder, ++level); break; case BoostingQueryBuilder boosting: - introspectQueryBuilder(boosting.positiveQuery(), queryMetadataBuilder); + introspectQueryBuilder(boosting.positiveQuery(), queryMetadataBuilder, ++level); break; case NestedQueryBuilder nested: - introspectQueryBuilder(nested.query(), queryMetadataBuilder); + introspectQueryBuilder(nested.query(), queryMetadataBuilder, ++level); break; case RangeQueryBuilder range: switch (range.fieldName()) { diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java index 31a5a3f82661b..33b0d4154256e 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java @@ -277,4 +277,39 @@ public void testExtractAttributes() { assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, new String[] { "@timestamp" }, null); } } + + public void testDepthLimit() { + { + SearchRequest searchRequest = new SearchRequest("index"); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + searchRequest.source(new SearchSourceBuilder().query(boolQueryBuilder)); + BoolQueryBuilder newBoolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.must(newBoolQueryBuilder); + int depth = randomIntBetween(5, 18); + for (int i = 0; i < depth; i++) { + BoolQueryBuilder innerBoolQueryBuilder = new BoolQueryBuilder(); + newBoolQueryBuilder.must(innerBoolQueryBuilder); + newBoolQueryBuilder = innerBoolQueryBuilder; + } + newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, new String[]{"@timestamp"}, null); + } + { + SearchRequest searchRequest = new SearchRequest("index"); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + searchRequest.source(new SearchSourceBuilder().query(boolQueryBuilder)); + BoolQueryBuilder newBoolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.must(newBoolQueryBuilder); + int depth = randomIntBetween(19, 50); + for (int i = 0; i < depth; i++) { + BoolQueryBuilder innerBoolQueryBuilder = new BoolQueryBuilder(); + newBoolQueryBuilder.must(innerBoolQueryBuilder); + newBoolQueryBuilder = innerBoolQueryBuilder; + } + newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, null, null); + } + } } From 98d5a5adc11c064ea5dcd88b2ed75648249c8cb0 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 12 Sep 2025 17:56:02 +0200 Subject: [PATCH 05/10] iter --- .../SearchRequestAttributesExtractor.java | 55 +++++++++-------- ...SearchRequestAttributesExtractorTests.java | 60 +++++++++---------- .../SearchTookTimeTelemetryTests.java | 13 ++-- 3 files changed, 63 insertions(+), 65 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java index ccf9ef682fd76..c6b697590bceb 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java @@ -24,9 +24,7 @@ import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.vectors.KnnVectorQueryBuilder; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; /** @@ -53,7 +51,7 @@ public static Map extractAttributes(SearchRequest searchRequest) SearchSourceBuilder searchSourceBuilder = searchRequest.source(); if (searchSourceBuilder == null) { - return buildAttributesMap(target, ScoreSortBuilder.NAME, HITS_ONLY, false, null, pitOrScroll); + return buildAttributesMap(target, ScoreSortBuilder.NAME, HITS_ONLY, false, false, false, pitOrScroll); } if (searchSourceBuilder.pointInTimeBuilder() != null) { @@ -79,7 +77,15 @@ public static Map extractAttributes(SearchRequest searchRequest) } final boolean hasKnn = searchSourceBuilder.knnSearch().isEmpty() == false || queryMetadataBuilder.knnQuery; - return buildAttributesMap(target, primarySort, queryType, hasKnn, queryMetadataBuilder.getRangeFields(), pitOrScroll); + return buildAttributesMap( + target, + primarySort, + queryType, + hasKnn, + queryMetadataBuilder.rangeOnTimestamp, + queryMetadataBuilder.rangeOnEventIngested, + pitOrScroll + ); } private static Map buildAttributesMap( @@ -87,42 +93,42 @@ private static Map buildAttributesMap( String primarySort, String queryType, boolean knn, - String[] rangeFields, + boolean rangeOnTimestamp, + boolean rangeOnEventIngested, String pitOrScroll ) { Map attributes = new HashMap<>(5, 1.0f); attributes.put(TARGET_ATTRIBUTE, target); attributes.put(SORT_ATTRIBUTE, primarySort); - if (pitOrScroll == null) { - attributes.put(QUERY_TYPE_ATTRIBUTE, queryType); - } else { - attributes.put(QUERY_TYPE_ATTRIBUTE, new String[] { queryType, pitOrScroll }); + attributes.put(QUERY_TYPE_ATTRIBUTE, queryType); + if (pitOrScroll != null) { + attributes.put(PIT_SCROLL_ATTRIBUTE, pitOrScroll); } - - attributes.put(KNN_ATTRIBUTE, knn); - if (rangeFields != null) { - attributes.put(RANGES_ATTRIBUTE, rangeFields); + if (knn) { + attributes.put(KNN_ATTRIBUTE, knn); + } + if (rangeOnTimestamp) { + attributes.put(RANGE_TIMESTAMP_ATTRIBUTE, rangeOnTimestamp); + } + if (rangeOnEventIngested) { + attributes.put(RANGE_EVENT_INGESTED_ATTRIBUTE, rangeOnEventIngested); } return attributes; } private static final class QueryMetadataBuilder { private boolean knnQuery = false; - private final List rangeFields = new ArrayList<>(); - - String[] getRangeFields() { - if (rangeFields.isEmpty()) { - return null; - } - return rangeFields.toArray(new String[0]); - } + private boolean rangeOnTimestamp = false; + private boolean rangeOnEventIngested = false; } static final String TARGET_ATTRIBUTE = "target"; static final String SORT_ATTRIBUTE = "sort"; static final String QUERY_TYPE_ATTRIBUTE = "query_type"; + static final String PIT_SCROLL_ATTRIBUTE = "pit_scroll"; static final String KNN_ATTRIBUTE = "knn"; - static final String RANGES_ATTRIBUTE = "ranges"; + static final String RANGE_TIMESTAMP_ATTRIBUTE = "range_timestamp"; + static final String RANGE_EVENT_INGESTED_ATTRIBUTE = "range_event_ingested"; private static final String TARGET_KIBANA = ".kibana"; private static final String TARGET_ML = ".ml"; @@ -256,9 +262,8 @@ private static void introspectQueryBuilder(QueryBuilder queryBuilder, QueryMetad break; case RangeQueryBuilder range: switch (range.fieldName()) { - case TIMESTAMP -> queryMetadataBuilder.rangeFields.add(TIMESTAMP); - case EVENT_INGESTED -> queryMetadataBuilder.rangeFields.add(EVENT_INGESTED); - default -> queryMetadataBuilder.rangeFields.add(FIELD); + case TIMESTAMP -> queryMetadataBuilder.rangeOnTimestamp = true; + case EVENT_INGESTED -> queryMetadataBuilder.rangeOnEventIngested = true; } break; case KnnVectorQueryBuilder knn: diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java index 33b0d4154256e..5c92112d7e56c 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java @@ -124,24 +124,28 @@ private static void assertAttributes( String primarySort, String queryType, boolean knn, - String[] rangeFields, + boolean rangeOnTimestamp, + boolean rangeOnEventIngested, String pitOrScroll ) { - assertEquals(target, attributes.get(SearchRequestAttributesExtractor.TARGET_ATTRIBUTE)); assertEquals(primarySort, attributes.get(SearchRequestAttributesExtractor.SORT_ATTRIBUTE)); - if (pitOrScroll == null) { - assertEquals(queryType, attributes.get(SearchRequestAttributesExtractor.QUERY_TYPE_ATTRIBUTE)); + assertEquals(queryType, attributes.get(SearchRequestAttributesExtractor.QUERY_TYPE_ATTRIBUTE)); + assertEquals(pitOrScroll, attributes.get(SearchRequestAttributesExtractor.PIT_SCROLL_ATTRIBUTE)); + if (knn) { + assertEquals(knn, attributes.get(SearchRequestAttributesExtractor.KNN_ATTRIBUTE)); + } else { + assertNull(attributes.get(SearchRequestAttributesExtractor.KNN_ATTRIBUTE)); + } + if (rangeOnTimestamp) { + assertEquals(rangeOnTimestamp, attributes.get(SearchRequestAttributesExtractor.RANGE_TIMESTAMP_ATTRIBUTE)); } else { - String[] queryTypes = (String[]) attributes.get(SearchRequestAttributesExtractor.QUERY_TYPE_ATTRIBUTE); - assertEquals(queryType, queryTypes[0]); - assertEquals(pitOrScroll, queryTypes[1]); + assertNull(attributes.get(SearchRequestAttributesExtractor.RANGE_TIMESTAMP_ATTRIBUTE)); } - assertEquals(knn, attributes.get(SearchRequestAttributesExtractor.KNN_ATTRIBUTE)); - if (rangeFields == null) { - assertFalse(attributes.containsKey(SearchRequestAttributesExtractor.RANGES_ATTRIBUTE)); + if (rangeOnEventIngested) { + assertEquals(rangeOnEventIngested, attributes.get(SearchRequestAttributesExtractor.RANGE_EVENT_INGESTED_ATTRIBUTE)); } else { - assertArrayEquals(rangeFields, (String[]) attributes.get(SearchRequestAttributesExtractor.RANGES_ATTRIBUTE)); + assertNull(attributes.get(SearchRequestAttributesExtractor.RANGE_EVENT_INGESTED_ATTRIBUTE)); } } @@ -149,7 +153,7 @@ public void testExtractAttributes() { { SearchRequest searchRequest = new SearchRequest(); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); - assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, null, null); + assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, false, false, null); } { SearchRequest searchRequest = new SearchRequest(); @@ -157,13 +161,13 @@ public void testExtractAttributes() { searchRequest.source(searchSourceBuilder); searchSourceBuilder.pointInTimeBuilder(new PointInTimeBuilder(BytesArray.EMPTY)); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); - assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, null, "pit"); + assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, false, false, "pit"); } { SearchRequest searchRequest = new SearchRequest(); searchRequest.scroll(new TimeValue(randomIntBetween(1, 10))); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); - assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, null, "scroll"); + assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, false, false, "scroll"); } { SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); @@ -172,7 +176,7 @@ public void testExtractAttributes() { searchSourceBuilder.sort("@timestamp"); searchSourceBuilder.query(new RangeQueryBuilder("@timestamp")); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); - assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, new String[] { "@timestamp" }, null); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null); } { SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); @@ -192,7 +196,7 @@ public void testExtractAttributes() { boolQueryBuilder.should(new RangeQueryBuilder("event.ingested")); } Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); - assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, new String[] { "@timestamp" }, null); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null); } { SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); @@ -213,7 +217,7 @@ public void testExtractAttributes() { boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp")); searchSourceBuilder.query(boolQueryBuilder); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); - assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, new String[] { "@timestamp" }, null); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null); } { SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); @@ -226,15 +230,7 @@ public void testExtractAttributes() { boolQueryBuilder.must(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10))); searchSourceBuilder.query(boolQueryBuilder); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); - assertAttributes( - stringObjectMap, - "user", - "@timestamp", - "hits_only", - false, - new String[] { "@timestamp", "event.ingested", "field" }, - null - ); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, true, null); } { SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); @@ -245,7 +241,7 @@ public void testExtractAttributes() { boolQueryBuilder.should(new RangeQueryBuilder("@timestamp")); searchSourceBuilder.query(boolQueryBuilder); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); - assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, new String[] { "@timestamp" }, null); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null); } { SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); @@ -256,7 +252,7 @@ public void testExtractAttributes() { boolQueryBuilder.should(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10))); searchSourceBuilder.query(boolQueryBuilder); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); - assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, new String[] { "field" }, null); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, false, false, null); } { SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); @@ -265,7 +261,7 @@ public void testExtractAttributes() { searchSourceBuilder.sort("@timestamp"); searchSourceBuilder.query(new ConstantScoreQueryBuilder(new RangeQueryBuilder("@timestamp"))); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); - assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, new String[] { "@timestamp" }, null); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null); } { SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); @@ -274,7 +270,7 @@ public void testExtractAttributes() { searchSourceBuilder.sort("@timestamp"); searchSourceBuilder.query(new BoostingQueryBuilder(new RangeQueryBuilder("@timestamp"), new MatchAllQueryBuilder())); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); - assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, new String[] { "@timestamp" }, null); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null); } } @@ -293,7 +289,7 @@ public void testDepthLimit() { } newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); - assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, new String[]{"@timestamp"}, null); + assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, true, false, null); } { SearchRequest searchRequest = new SearchRequest("index"); @@ -309,7 +305,7 @@ public void testDepthLimit() { } newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); - assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, null, null); + assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, false, false, null); } } } diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java index 35760efe6928b..770f1fbc4d0f0 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java @@ -119,15 +119,13 @@ public void testSimpleQueryAgainstAlias() { Measurement measurement = measurements.getFirst(); assertEquals(searchResponse.getTook().millis(), measurement.getLong()); assertSimpleQueryAttributes(measurement.attributes()); - } private static void assertSimpleQueryAttributes(Map attributes) { - assertEquals(4, attributes.size()); + assertEquals(3, attributes.size()); assertEquals("user", attributes.get("target")); assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); - assertEquals(false, attributes.get("knn")); } public void testMultiSearch() { @@ -178,9 +176,9 @@ public void testScroll() { Map attributes = measurement.attributes(); assertEquals(4, attributes.size()); assertEquals("user", attributes.get("target")); - assertArrayEquals(new String[] { "hits_only", "scroll" }, (String[]) attributes.get("query_type")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("scroll", attributes.get("pit_scroll")); assertEquals("_score", attributes.get("sort")); - assertEquals(false, attributes.get("knn")); } else { List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement( TOOK_DURATION_TOTAL_HISTOGRAM_NAME @@ -260,12 +258,11 @@ public void testTimeRangeFilterOneResults() { } private static void assertTimeRangeAttributes(Map attributes) { - assertEquals(5, attributes.size()); + assertEquals(4, attributes.size()); assertEquals("user", attributes.get("target")); assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); - assertEquals(false, attributes.get("knn")); - assertArrayEquals(new String[] { "@timestamp" }, (String[]) attributes.get("ranges")); + assertEquals(true, attributes.get("range_timestamp")); } private void resetMeter() { From c0271a34ce8847756c4c7e91824443642bfd5933 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 12 Sep 2025 20:27:40 +0200 Subject: [PATCH 06/10] iter --- .../TelemetryMetrics/SearchTookTimeTelemetryTests.java | 8 ++++++++ .../xpack/search/AsyncSearchTookTimeTelemetryTests.java | 3 +-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java index 770f1fbc4d0f0..82527817349d6 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java @@ -194,6 +194,14 @@ public void testScroll() { ); } + //TODO test PIT (e.g. open PIT does not record took time + + //TODO test resolve indices with data stream + + //TODO test resolved indices (and alias) with security enabled + + //TODO do we actually want the concrete indices? We should be fine either way with the starts with logic + /** * Make sure that despite can match and query rewrite, we see the time range filter and record its corresponding attribute */ diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java index faca5e8dbec2a..e0d6189d15a8a 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java @@ -159,10 +159,9 @@ private TestTelemetryPlugin getTestTelemetryPlugin() { } private static void assertAttributes(Map attributes) { - assertEquals(4, attributes.size()); + assertEquals(3, attributes.size()); assertEquals("user", attributes.get("target")); assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); - assertEquals(false, attributes.get("knn")); } } From 814b2107b8adee3c2922a6dd9145132cceed641d Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 12 Sep 2025 18:35:44 +0000 Subject: [PATCH 07/10] [CI] Auto commit changes from spotless --- .../TelemetryMetrics/SearchTookTimeTelemetryTests.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java index 82527817349d6..76563e1d91104 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java @@ -194,13 +194,13 @@ public void testScroll() { ); } - //TODO test PIT (e.g. open PIT does not record took time + // TODO test PIT (e.g. open PIT does not record took time - //TODO test resolve indices with data stream + // TODO test resolve indices with data stream - //TODO test resolved indices (and alias) with security enabled + // TODO test resolved indices (and alias) with security enabled - //TODO do we actually want the concrete indices? We should be fine either way with the starts with logic + // TODO do we actually want the concrete indices? We should be fine either way with the starts with logic /** * Make sure that despite can match and query rewrite, we see the time range filter and record its corresponding attribute From 53dcc9fffdc49428c9164fadc5142c0b566a2eb8 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 15 Sep 2025 12:23:25 +0200 Subject: [PATCH 08/10] wip --- .../SearchTookTimeTelemetryTests.java | 51 +++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java index 76563e1d91104..4f0ae12332f86 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java @@ -11,10 +11,16 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; +import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.datastreams.CreateDataStreamAction; import org.elasticsearch.action.search.MultiSearchRequestBuilder; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; @@ -22,6 +28,8 @@ import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.rest.RestUtils; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.telemetry.Measurement; import org.elasticsearch.telemetry.TestTelemetryPlugin; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -38,6 +46,7 @@ import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery; import static org.elasticsearch.rest.action.search.SearchResponseMetrics.TOOK_DURATION_TOTAL_HISTOGRAM_NAME; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertScrollResponsesAndHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; @@ -194,14 +203,50 @@ public void testScroll() { ); } - // TODO test PIT (e.g. open PIT does not record took time - - // TODO test resolve indices with data stream + //TODO test PIT (e.g. open PIT does not record took time // TODO test resolved indices (and alias) with security enabled // TODO do we actually want the concrete indices? We should be fine either way with the starts with logic + public void testDataStream() { + TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("id"); + request.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of("foo-*")) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build() + ); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + "foo" + ); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet()); + + prepareIndex("foo").setId("1").setSource("body", "foo", "@timestamp", "2024-11-01").setRefreshPolicy(IMMEDIATE).get(); + + SearchResponse searchResponse = client().prepareSearch("foo").addSort(new FieldSortBuilder("@timestamp")).setQuery(new RangeQueryBuilder("@timestamp").from("2024-01-01")).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1", "2"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + Map attributes = measurement.attributes(); + assertEquals(4, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("@timestamp", attributes.get("sort")); + assertEquals(true, attributes.get("range_timestamp")); + } + /** * Make sure that despite can match and query rewrite, we see the time range filter and record its corresponding attribute */ From 83378b906c740fad969fcad045dafc4df8d9fd0d Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 15 Sep 2025 20:33:19 +0200 Subject: [PATCH 09/10] iter --- .../action/search/TransportSearchAction.java | 14 +++- .../action/search/SearchResponseMetrics.java | 1 - .../SearchTookTimeTelemetryTests.java | 67 ++++--------------- 3 files changed, 26 insertions(+), 56 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 3358c32a748dc..c87e2af1039c5 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -451,7 +451,13 @@ public void onFailure(Exception e) { if (shouldMinimizeRoundtrips(rewritten)) { usageBuilder.setFeature(CCSUsageTelemetry.MRT_FEATURE); } - searchResponseActionListener = new SearchTelemetryListener(delegate, searchResponseMetrics, searchRequestAttributes, usageService, usageBuilder); + searchResponseActionListener = new SearchTelemetryListener( + delegate, + searchResponseMetrics, + searchRequestAttributes, + usageService, + usageBuilder + ); } } else { searchResponseActionListener = delegate; @@ -2053,7 +2059,11 @@ private static class SearchTelemetryListener extends DelegatingActionListener listener, SearchResponseMetrics searchResponseMetrics, Map searchRequestAttributes) { + SearchTelemetryListener( + ActionListener listener, + SearchResponseMetrics searchResponseMetrics, + Map searchRequestAttributes + ) { super(listener); this.searchResponseMetrics = searchResponseMetrics; this.searchRequestAttributes = searchRequestAttributes; diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java index 99897cdbbd791..5df6153f766df 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java @@ -9,7 +9,6 @@ package org.elasticsearch.rest.action.search; -import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestAttributesExtractor; import org.elasticsearch.telemetry.metric.LongCounter; import org.elasticsearch.telemetry.metric.LongHistogram; diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java index b7964ceda0274..5a8a4bd10d08f 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java @@ -11,13 +11,12 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; -import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; -import org.elasticsearch.action.datastreams.CreateDataStreamAction; import org.elasticsearch.action.search.MultiSearchRequestBuilder; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; @@ -29,7 +28,6 @@ import org.elasticsearch.search.rescore.QueryRescorerBuilder; import org.elasticsearch.search.retriever.RescorerRetrieverBuilder; import org.elasticsearch.search.retriever.StandardRetrieverBuilder; -import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.telemetry.Measurement; import org.elasticsearch.telemetry.TestTelemetryPlugin; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -47,7 +45,6 @@ import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery; import static org.elasticsearch.rest.action.search.SearchResponseMetrics.TOOK_DURATION_TOTAL_HISTOGRAM_NAME; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertScrollResponsesAndHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; @@ -62,6 +59,15 @@ protected boolean resetNodeAfterTest() { @Before public void setUpIndex() { + var num_primaries = randomIntBetween(2, 4); + createIndex( + indexName, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, num_primaries) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build() + ); + ensureGreen(indexName); prepareIndex(indexName).setId("1").setSource("body", "foo", "@timestamp", "2024-11-01").setRefreshPolicy(IMMEDIATE).get(); prepareIndex(indexName).setId("2").setSource("body", "foo", "@timestamp", "2024-12-01").setRefreshPolicy(IMMEDIATE).get(); } @@ -227,53 +233,6 @@ public void testScroll() { ); } - // TODO test PIT (e.g. open PIT does not record took time - - // TODO test resolved indices (and alias) with security enabled - - // TODO do we actually want the concrete indices? We should be fine either way with the starts with logic - - public void testDataStream() { - TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("id"); - request.indexTemplate( - ComposableIndexTemplate.builder() - .indexPatterns(List.of("foo-*")) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) - .build() - ); - client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); - CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request( - TEST_REQUEST_TIMEOUT, - TEST_REQUEST_TIMEOUT, - "foo" - ); - assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet()); - - prepareIndex("foo").setId("1").setSource("body", "foo", "@timestamp", "2024-11-01").setRefreshPolicy(IMMEDIATE).get(); - - SearchResponse searchResponse = client().prepareSearch("foo") - .addSort(new FieldSortBuilder("@timestamp")) - .setQuery(new RangeQueryBuilder("@timestamp").from("2024-01-01")) - .get(); - try { - assertNoFailures(searchResponse); - assertSearchHits(searchResponse, "1", "2"); - } finally { - searchResponse.decRef(); - } - - List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); - assertEquals(1, measurements.size()); - Measurement measurement = measurements.getFirst(); - assertEquals(searchResponse.getTook().millis(), measurement.getLong()); - Map attributes = measurement.attributes(); - assertEquals(4, attributes.size()); - assertEquals("user", attributes.get("target")); - assertEquals("hits_only", attributes.get("query_type")); - assertEquals("@timestamp", attributes.get("sort")); - assertEquals(true, attributes.get("range_timestamp")); - } - /** * Make sure that despite can match and query rewrite, we see the time range filter and record its corresponding attribute */ @@ -285,6 +244,8 @@ public void testTimeRangeFilterNoResults() { try { assertNoFailures(searchResponse); assertSearchHits(searchResponse); + // can match kicked in, query got rewritten to match_none, yet we extracted the time range before rewrite + assertEquals(searchResponse.getSkippedShards(), searchResponse.getTotalShards()); } finally { searchResponse.decRef(); } @@ -318,7 +279,7 @@ public void testTimeRangeFilterAllResults() { assertTimeRangeAttributes(measurement.attributes()); } - public void testTimeRangeFilterOneResults() { + public void testTimeRangeFilterOneResult() { BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from("2024-12-01")); boolQueryBuilder.must(simpleQueryStringQuery("foo")); From 44ffe87e8da4cc2e154a801359d6f4566939276d Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 15 Sep 2025 21:09:56 +0200 Subject: [PATCH 10/10] iter --- .../SearchRequestAttributesExtractor.java | 10 +- .../action/search/TransportSearchAction.java | 5 +- ...SearchRequestAttributesExtractorTests.java | 65 +++++++-- .../SearchTookTimeTelemetryTests.java | 132 +++++++++++++++++- 4 files changed, 193 insertions(+), 19 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java index c6b697590bceb..b0f5aeeb0e0f1 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.BoostingQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; @@ -34,15 +35,17 @@ * ever breaking: if something goes wrong around extracting attributes, it should skip extracting * them as opposed to failing the search. */ -public class SearchRequestAttributesExtractor { +public final class SearchRequestAttributesExtractor { private static final Logger logger = LogManager.getLogger(SearchRequestAttributesExtractor.class); + private SearchRequestAttributesExtractor() {} + /** * Introspects the provided search request and extracts metadata from it about some of its characteristics. * */ - public static Map extractAttributes(SearchRequest searchRequest) { - String target = extractIndices(searchRequest.indices()); + public static Map extractAttributes(SearchRequest searchRequest, String[] localIndices) { + String target = extractIndices(localIndices); String pitOrScroll = null; if (searchRequest.scroll() != null) { @@ -147,6 +150,7 @@ static String extractIndices(String[] indices) { // If indices resolve to data streams, the name of the data stream is returned as opposed to its backing indices if (indices.length == 1) { String index = indices[0]; + assert Regex.isSimpleMatchPattern(index) == false; if (index.startsWith(".")) { if (index.startsWith(TARGET_KIBANA)) { return TARGET_KIBANA; diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index c87e2af1039c5..6366916c0a64f 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -424,7 +424,10 @@ public void onFailure(Exception e) { final ActionListener searchResponseActionListener; if (collectSearchTelemetry) { - Map searchRequestAttributes = SearchRequestAttributesExtractor.extractAttributes(original); + Map searchRequestAttributes = SearchRequestAttributesExtractor.extractAttributes( + original, + Arrays.stream(resolvedIndices.getConcreteLocalIndices()).map(Index::getName).toArray(String[]::new) + ); if (collectCCSTelemetry == false || resolvedIndices.getRemoteClusterIndices().isEmpty()) { searchResponseActionListener = new SearchTelemetryListener(delegate, searchResponseMetrics, searchRequestAttributes); } else { diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java index 5c92112d7e56c..ceb7e8cc9e8de 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java @@ -152,7 +152,10 @@ private static void assertAttributes( public void testExtractAttributes() { { SearchRequest searchRequest = new SearchRequest(); - Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, false, false, null); } { @@ -160,13 +163,19 @@ public void testExtractAttributes() { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchRequest.source(searchSourceBuilder); searchSourceBuilder.pointInTimeBuilder(new PointInTimeBuilder(BytesArray.EMPTY)); - Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, false, false, "pit"); } { SearchRequest searchRequest = new SearchRequest(); searchRequest.scroll(new TimeValue(randomIntBetween(1, 10))); - Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, false, false, "scroll"); } { @@ -175,7 +184,10 @@ public void testExtractAttributes() { searchRequest.source(searchSourceBuilder); searchSourceBuilder.sort("@timestamp"); searchSourceBuilder.query(new RangeQueryBuilder("@timestamp")); - Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null); } { @@ -195,7 +207,10 @@ public void testExtractAttributes() { if (randomBoolean()) { boolQueryBuilder.should(new RangeQueryBuilder("event.ingested")); } - Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null); } { @@ -216,7 +231,10 @@ public void testExtractAttributes() { boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp")); searchSourceBuilder.query(boolQueryBuilder); - Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null); } { @@ -229,7 +247,10 @@ public void testExtractAttributes() { boolQueryBuilder.must(new RangeQueryBuilder("event.ingested")); boolQueryBuilder.must(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10))); searchSourceBuilder.query(boolQueryBuilder); - Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, true, null); } { @@ -240,7 +261,10 @@ public void testExtractAttributes() { BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); boolQueryBuilder.should(new RangeQueryBuilder("@timestamp")); searchSourceBuilder.query(boolQueryBuilder); - Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null); } { @@ -251,7 +275,10 @@ public void testExtractAttributes() { BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); boolQueryBuilder.should(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10))); searchSourceBuilder.query(boolQueryBuilder); - Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, false, false, null); } { @@ -260,7 +287,10 @@ public void testExtractAttributes() { searchRequest.source(searchSourceBuilder); searchSourceBuilder.sort("@timestamp"); searchSourceBuilder.query(new ConstantScoreQueryBuilder(new RangeQueryBuilder("@timestamp"))); - Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null); } { @@ -269,7 +299,10 @@ public void testExtractAttributes() { searchRequest.source(searchSourceBuilder); searchSourceBuilder.sort("@timestamp"); searchSourceBuilder.query(new BoostingQueryBuilder(new RangeQueryBuilder("@timestamp"), new MatchAllQueryBuilder())); - Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null); } } @@ -288,7 +321,10 @@ public void testDepthLimit() { newBoolQueryBuilder = innerBoolQueryBuilder; } newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); - Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, true, false, null); } { @@ -304,7 +340,10 @@ public void testDepthLimit() { newBoolQueryBuilder = innerBoolQueryBuilder; } newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); - Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(searchRequest); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, false, false, null); } } diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java index 5a8a4bd10d08f..792b676f6002b 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; @@ -82,6 +83,133 @@ protected Collection> getPlugins() { return pluginList(TestTelemetryPlugin.class); } + public void testOthersDottedIndexName() { + createIndex(".whatever"); + createIndex(".kibana"); + { + SearchResponse searchResponse = client().prepareSearch(".whatever").setQuery(simpleQueryStringQuery("foo")).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse); + } finally { + searchResponse.decRef(); + } + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + Map attributes = measurement.attributes(); + assertEquals(3, attributes.size()); + assertEquals(".others", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + } + { + SearchResponse searchResponse = client().prepareSearch(".kibana*").setQuery(simpleQueryStringQuery("foo")).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse); + } finally { + searchResponse.decRef(); + } + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(2, measurements.size()); + Measurement measurement = measurements.getLast(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + Map attributes = measurement.attributes(); + assertEquals(3, attributes.size()); + assertEquals(".kibana", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + } + { + SearchResponse searchResponse = client().prepareSearch(".*").setQuery(simpleQueryStringQuery("foo")).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse); + } finally { + searchResponse.decRef(); + } + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(3, measurements.size()); + Measurement measurement = measurements.getLast(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + // two dotted indices: categorized as "user" + assertSimpleQueryAttributes(measurement.attributes()); + } + { + SearchResponse searchResponse = client().prepareSearch(".kibana", ".whatever").setQuery(simpleQueryStringQuery("foo")).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse); + } finally { + searchResponse.decRef(); + } + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(4, measurements.size()); + Measurement measurement = measurements.getLast(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + // two dotted indices: categorized as "user" + assertSimpleQueryAttributes(measurement.attributes()); + } + { + SearchResponse searchResponse = client().prepareSearch(".kibana", ".does_not_exist") + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN) + .setQuery(simpleQueryStringQuery("foo")) + .get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse); + } finally { + searchResponse.decRef(); + } + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(5, measurements.size()); + Measurement measurement = measurements.getLast(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + Map attributes = measurement.attributes(); + assertEquals(3, attributes.size()); + // because the second index does not exist, yet the search goes through, the remaining index is categorized correctly + assertEquals(".kibana", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + } + { + SearchResponse searchResponse = client().prepareSearch("_all").setQuery(simpleQueryStringQuery("foo")).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1", "2"); + } finally { + searchResponse.decRef(); + } + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(6, measurements.size()); + Measurement measurement = measurements.getLast(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertSimpleQueryAttributes(measurement.attributes()); + } + } + + public void testIndexNameMustExist() { + SearchResponse searchResponse = client().prepareSearch(".must_exist") + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN) + .setQuery(simpleQueryStringQuery("foo")) + .get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse); + } finally { + searchResponse.decRef(); + } + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + // edge case rather than under .others (as it's a dotted index name), the index is categorized under "user" because no existing + // indices are targeted. + assertSimpleQueryAttributes(measurement.attributes()); + } + public void testSimpleQuery() { SearchResponse searchResponse = client().prepareSearch(indexName).setQuery(simpleQueryStringQuery("foo")).get(); try { @@ -119,10 +247,10 @@ public void testSimpleQueryAgainstAlias() { RestUtils.REST_MASTER_TIMEOUT_DEFAULT, new TimeValue(30, TimeUnit.SECONDS) ); - indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add().indices(indexName).alias("alias")); + indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add().indices(indexName).alias(".alias")); IndicesAliasesResponse indicesAliasesResponse = client().admin().indices().aliases(indicesAliasesRequest).actionGet(); assertFalse(indicesAliasesResponse.hasErrors()); - SearchResponse searchResponse = client().prepareSearch("alias").setQuery(simpleQueryStringQuery("foo")).get(); + SearchResponse searchResponse = client().prepareSearch(".alias").setQuery(simpleQueryStringQuery("foo")).get(); try { assertNoFailures(searchResponse); assertSearchHits(searchResponse, "1", "2");