From 7682f3789767aaeb751eac9a71ae957f7d68dde5 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 16 Sep 2025 10:50:07 +0200 Subject: [PATCH 1/5] Add relevant attributes to shard search latency APM metrics We already record latency of the query and fetch phase as APM metrics. We'd like to be able to slice such latencies based on some recurring categories of the request: - does it have agg or hit only? - is it sorted by field or by score? - does it have a time range filter? - does it target user data or internal indices? This commit introduces introspection for a shard search request and stores the extracted attributes together with the shard phase latency metrics. This builds on top of #134232 to use the same infra and store the same attributes for shard level latency metrics. --- .../SearchRequestAttributesExtractor.java | 32 ++- .../stats/ShardSearchPhaseAPMMetrics.java | 21 +- .../ShardSearchPhaseAPMMetricsTests.java | 205 +++++++++++++----- 3 files changed, 181 insertions(+), 77 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java index b0f5aeeb0e0f1..284a6184ea57f 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java @@ -12,6 +12,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.BoostingQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; @@ -20,6 +22,7 @@ import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.ScoreSortBuilder; import org.elasticsearch.search.sort.SortBuilder; @@ -42,17 +45,37 @@ private SearchRequestAttributesExtractor() {} /** * Introspects the provided search request and extracts metadata from it about some of its characteristics. - * */ public static Map extractAttributes(SearchRequest searchRequest, String[] localIndices) { + return extractAttributes(searchRequest.source(), searchRequest.scroll(), localIndices); + } + + /** + * Introspects the provided shard search request and extracts metadata from it about some of its characteristics. + */ + public static Map extractAttributes(ShardSearchRequest shardSearchRequest) { + Map attributes = extractAttributes( + shardSearchRequest.source(), + shardSearchRequest.scroll(), + shardSearchRequest.shardId().getIndexName() + ); + boolean isSystem = ((EsExecutors.EsThread) Thread.currentThread()).isSystem(); + attributes.put(SYSTEM_THREAD_ATTRIBUTE_NAME, isSystem); + return attributes; + } + + private static Map extractAttributes( + SearchSourceBuilder searchSourceBuilder, + TimeValue scroll, + String... localIndices + ) { String target = extractIndices(localIndices); String pitOrScroll = null; - if (searchRequest.scroll() != null) { + if (scroll != null) { pitOrScroll = SCROLL; } - SearchSourceBuilder searchSourceBuilder = searchRequest.source(); if (searchSourceBuilder == null) { return buildAttributesMap(target, ScoreSortBuilder.NAME, HITS_ONLY, false, false, false, pitOrScroll); } @@ -144,7 +167,7 @@ private static final class QueryMetadataBuilder { private static final String TARGET_USER = "user"; private static final String ERROR = "error"; - static String extractIndices(String[] indices) { + static String extractIndices(String... indices) { try { // Note that indices are expected to be resolved, meaning wildcards are not handled on purpose // If indices resolve to data streams, the name of the data stream is returned as opposed to its backing indices @@ -213,6 +236,7 @@ static String extractPrimarySort(SortBuilder primarySortBuilder) { private static final String PIT = "pit"; private static final String SCROLL = "scroll"; + public static final String SYSTEM_THREAD_ATTRIBUTE_NAME = "system_thread"; public static final Map SEARCH_SCROLL_ATTRIBUTES = Map.of(QUERY_TYPE_ATTRIBUTE, SCROLL); static String extractQueryType(SearchSourceBuilder searchSourceBuilder) { diff --git a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java index 6b523a154379e..38ec698af8d9c 100644 --- a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java +++ b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java @@ -9,13 +9,13 @@ package org.elasticsearch.index.search.stats; -import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.action.search.SearchRequestAttributesExtractor; import org.elasticsearch.index.shard.SearchOperationListener; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.telemetry.metric.LongHistogram; import org.elasticsearch.telemetry.metric.MeterRegistry; -import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -24,14 +24,9 @@ public final class ShardSearchPhaseAPMMetrics implements SearchOperationListener public static final String QUERY_SEARCH_PHASE_METRIC = "es.search.shards.phases.query.duration.histogram"; public static final String FETCH_SEARCH_PHASE_METRIC = "es.search.shards.phases.fetch.duration.histogram"; - public static final String SYSTEM_THREAD_ATTRIBUTE_NAME = "system_thread"; - private final LongHistogram queryPhaseMetric; private final LongHistogram fetchPhaseMetric; - // Avoid allocating objects in the search path and multithreading clashes - private static final ThreadLocal> THREAD_LOCAL_ATTRS = ThreadLocal.withInitial(() -> new HashMap<>(1)); - public ShardSearchPhaseAPMMetrics(MeterRegistry meterRegistry) { this.queryPhaseMetric = meterRegistry.registerLongHistogram( QUERY_SEARCH_PHASE_METRIC, @@ -47,18 +42,16 @@ public ShardSearchPhaseAPMMetrics(MeterRegistry meterRegistry) { @Override public void onQueryPhase(SearchContext searchContext, long tookInNanos) { - recordPhaseLatency(queryPhaseMetric, tookInNanos); + recordPhaseLatency(queryPhaseMetric, tookInNanos, searchContext.request()); } @Override public void onFetchPhase(SearchContext searchContext, long tookInNanos) { - recordPhaseLatency(fetchPhaseMetric, tookInNanos); + recordPhaseLatency(fetchPhaseMetric, tookInNanos, searchContext.request()); } - private static void recordPhaseLatency(LongHistogram histogramMetric, long tookInNanos) { - Map attrs = ShardSearchPhaseAPMMetrics.THREAD_LOCAL_ATTRS.get(); - boolean isSystem = ((EsExecutors.EsThread) Thread.currentThread()).isSystem(); - attrs.put(SYSTEM_THREAD_ATTRIBUTE_NAME, isSystem); - histogramMetric.record(TimeUnit.NANOSECONDS.toMillis(tookInNanos), attrs); + private static void recordPhaseLatency(LongHistogram histogramMetric, long tookInNanos, ShardSearchRequest request) { + Map attributes = SearchRequestAttributesExtractor.extractAttributes(request); + histogramMetric.record(TimeUnit.NANOSECONDS.toMillis(tookInNanos), attributes); } } diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java index 80bb7ebc8ddb8..ba1b5914b8ed4 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.search.TelemetryMetrics; +import org.elasticsearch.action.search.SearchRequestAttributesExtractor; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; @@ -21,18 +22,18 @@ import org.elasticsearch.telemetry.Measurement; import org.elasticsearch.telemetry.TestTelemetryPlugin; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; import java.util.Collection; import java.util.List; -import java.util.stream.Stream; +import java.util.Map; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery; import static org.elasticsearch.index.search.stats.ShardSearchPhaseAPMMetrics.FETCH_SEARCH_PHASE_METRIC; import static org.elasticsearch.index.search.stats.ShardSearchPhaseAPMMetrics.QUERY_SEARCH_PHASE_METRIC; -import static org.elasticsearch.index.search.stats.ShardSearchPhaseAPMMetrics.SYSTEM_THREAD_ATTRIBUTE_NAME; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertScrollResponsesAndHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHitsWithoutFailures; @@ -47,7 +48,7 @@ protected boolean resetNodeAfterTest() { } @Before - private void setUpIndex() throws Exception { + public void setUpIndex() { createIndex( indexName, Settings.builder() @@ -65,7 +66,7 @@ private void setUpIndex() throws Exception { } @After - private void afterTest() { + public void afterTest() { resetMeter(); } @@ -74,72 +75,162 @@ protected Collection> getPlugins() { return pluginList(TestTelemetryPlugin.class, TestSystemIndexPlugin.class); } - public void testMetricsDfsQueryThenFetch() throws InterruptedException { - checkMetricsDfsQueryThenFetch(indexName, false); - } - - public void testMetricsDfsQueryThenFetchSystem() throws InterruptedException { - checkMetricsDfsQueryThenFetch(TestSystemIndexPlugin.INDEX_NAME, true); - } - - private void checkMetricsDfsQueryThenFetch(String indexName, boolean isSystemIndex) throws InterruptedException { + public void testMetricsDfsQueryThenFetch() { assertSearchHitsWithoutFailures( client().prepareSearch(indexName).setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setQuery(simpleQueryStringQuery("doc1")), "1" ); - checkNumberOfMeasurementsForPhase(QUERY_SEARCH_PHASE_METRIC, isSystemIndex); - assertNotEquals(0, getNumberOfMeasurementsForPhase(FETCH_SEARCH_PHASE_METRIC)); - checkMetricsAttributes(isSystemIndex); - } - - public void testSearchTransportMetricsQueryThenFetch() throws InterruptedException { - checkSearchTransportMetricsQueryThenFetch(indexName, false); + final List queryMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(QUERY_SEARCH_PHASE_METRIC); + assertEquals(num_primaries, queryMeasurements.size()); + final List fetchMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(FETCH_SEARCH_PHASE_METRIC); + assertEquals(1, fetchMeasurements.size()); + assertAttributes(fetchMeasurements, false, false); } - public void testSearchTransportMetricsQueryThenFetchSystem() throws InterruptedException { - checkSearchTransportMetricsQueryThenFetch(TestSystemIndexPlugin.INDEX_NAME, true); + public void testMetricsDfsQueryThenFetchSystem() { + assertSearchHitsWithoutFailures( + client().prepareSearch(TestSystemIndexPlugin.INDEX_NAME) + .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) + .setQuery(simpleQueryStringQuery("doc1")), + "1" + ); + final List queryMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(QUERY_SEARCH_PHASE_METRIC); + assertEquals(1, queryMeasurements.size()); + final List fetchMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(FETCH_SEARCH_PHASE_METRIC); + assertEquals(1, fetchMeasurements.size()); + assertAttributes(fetchMeasurements, true, false); } - private void checkSearchTransportMetricsQueryThenFetch(String indexName, boolean isSystemIndex) throws InterruptedException { + public void testSearchTransportMetricsQueryThenFetch() { assertSearchHitsWithoutFailures( client().prepareSearch(indexName).setSearchType(SearchType.QUERY_THEN_FETCH).setQuery(simpleQueryStringQuery("doc1")), "1" ); - checkNumberOfMeasurementsForPhase(QUERY_SEARCH_PHASE_METRIC, isSystemIndex); - assertNotEquals(0, getNumberOfMeasurementsForPhase(FETCH_SEARCH_PHASE_METRIC)); - checkMetricsAttributes(isSystemIndex); + final List queryMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(QUERY_SEARCH_PHASE_METRIC); + assertEquals(num_primaries, queryMeasurements.size()); + assertAttributes(queryMeasurements, false, false); + final List fetchMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(FETCH_SEARCH_PHASE_METRIC); + assertEquals(1, fetchMeasurements.size()); + assertAttributes(fetchMeasurements, false, false); } - public void testSearchTransportMetricsScroll() throws InterruptedException { - checkSearchTransportMetricsScroll(indexName, false); + public void testSearchTransportMetricsQueryThenFetchSystem() { + assertSearchHitsWithoutFailures( + client().prepareSearch(TestSystemIndexPlugin.INDEX_NAME) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setQuery(simpleQueryStringQuery("doc1")), + "1" + ); + final List queryMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(QUERY_SEARCH_PHASE_METRIC); + assertEquals(1, queryMeasurements.size()); + assertAttributes(queryMeasurements, true, false); + final List fetchMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(FETCH_SEARCH_PHASE_METRIC); + assertEquals(1, fetchMeasurements.size()); + assertAttributes(fetchMeasurements, true, false); } - public void testSearchTransportMetricsScrollSystem() throws InterruptedException { - checkSearchTransportMetricsScroll(TestSystemIndexPlugin.INDEX_NAME, true); + public void testSearchMultipleIndices() { + assertSearchHitsWithoutFailures( + client().prepareSearch(indexName, TestSystemIndexPlugin.INDEX_NAME) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setQuery(simpleQueryStringQuery("doc1")), + "1", + "1" + ); + { + final List queryMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(QUERY_SEARCH_PHASE_METRIC); + assertEquals(num_primaries + 1, queryMeasurements.size()); + int userTarget = 0; + int systemTarget = 0; + for (Measurement measurement : queryMeasurements) { + Map attributes = measurement.attributes(); + assertEquals(4, attributes.size()); + + String target = attributes.get("target").toString(); + if (target.equals("user")) { + userTarget++; + } else { + systemTarget++; + assertEquals(".others", target); + assertEquals(true, measurement.attributes().get(SearchRequestAttributesExtractor.SYSTEM_THREAD_ATTRIBUTE_NAME)); + } + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + } + assertEquals(num_primaries, userTarget); + assertEquals(1, systemTarget); + } + { + final List fetchMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(FETCH_SEARCH_PHASE_METRIC); + assertEquals(2, fetchMeasurements.size()); + int userTarget = 0; + int systemTarget = 0; + for (Measurement measurement : fetchMeasurements) { + Map attributes = measurement.attributes(); + assertEquals(4, attributes.size()); + + String target = attributes.get("target").toString(); + if (target.equals("user")) { + userTarget++; + } else { + systemTarget++; + assertEquals(".others", target); + assertEquals(true, measurement.attributes().get(SearchRequestAttributesExtractor.SYSTEM_THREAD_ATTRIBUTE_NAME)); + } + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + } + assertEquals(1, userTarget); + assertEquals(1, systemTarget); + } } - private void checkSearchTransportMetricsScroll(String indexName, boolean isSystemIndex) throws InterruptedException { + public void testSearchTransportMetricsScroll() { assertScrollResponsesAndHitCount( client(), TimeValue.timeValueSeconds(60), - client().prepareSearch(indexName) - .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) - .setSize(1) - .setQuery(simpleQueryStringQuery("doc1 doc2")), + client().prepareSearch(indexName).setSize(1).setQuery(simpleQueryStringQuery("doc1 doc2")), 2, (respNum, response) -> { + final List queryMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(QUERY_SEARCH_PHASE_METRIC); + assertEquals(num_primaries, queryMeasurements.size()); + assertAttributes(queryMeasurements, false, true); // No hits, no fetching done - assertEquals(isSystemIndex ? 1 : num_primaries, getNumberOfMeasurementsForPhase(QUERY_SEARCH_PHASE_METRIC)); if (response.getHits().getHits().length > 0) { - assertNotEquals(0, getNumberOfMeasurementsForPhase(FETCH_SEARCH_PHASE_METRIC)); + final List fetchMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement( + FETCH_SEARCH_PHASE_METRIC + ); + assertThat(fetchMeasurements.size(), Matchers.greaterThan(0)); + int numFetchShards = Math.min(2, num_primaries); + assertThat(fetchMeasurements.size(), Matchers.lessThanOrEqualTo(numFetchShards)); + assertAttributes(fetchMeasurements, false, true); } else { - assertEquals(isSystemIndex ? 1 : 0, getNumberOfMeasurementsForPhase(FETCH_SEARCH_PHASE_METRIC)); + final List fetchMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement( + FETCH_SEARCH_PHASE_METRIC + ); + assertEquals(0, fetchMeasurements.size()); } - checkMetricsAttributes(isSystemIndex); resetMeter(); } ); + } + public void testSearchTransportMetricsScrollSystem() { + assertScrollResponsesAndHitCount( + client(), + TimeValue.timeValueSeconds(60), + client().prepareSearch(TestSystemIndexPlugin.INDEX_NAME).setSize(1).setQuery(simpleQueryStringQuery("doc1 doc2")), + 2, + (respNum, response) -> { + final List queryMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(QUERY_SEARCH_PHASE_METRIC); + assertEquals(1, queryMeasurements.size()); + assertAttributes(queryMeasurements, true, true); + final List fetchMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(FETCH_SEARCH_PHASE_METRIC); + assertEquals(1, fetchMeasurements.size()); + assertAttributes(fetchMeasurements, true, true); + resetMeter(); + } + ); } private void resetMeter() { @@ -150,26 +241,22 @@ private TestTelemetryPlugin getTestTelemetryPlugin() { return getInstanceFromNode(PluginsService.class).filterPlugins(TestTelemetryPlugin.class).toList().get(0); } - private void checkNumberOfMeasurementsForPhase(String phase, boolean isSystemIndex) { - int numMeasurements = getNumberOfMeasurementsForPhase(phase); - assertEquals(isSystemIndex ? 1 : num_primaries, numMeasurements); - } - - private int getNumberOfMeasurementsForPhase(String phase) { - final List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(phase); - return measurements.size(); - } - - private void checkMetricsAttributes(boolean isSystem) { - final List queryMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(QUERY_SEARCH_PHASE_METRIC); - final List fetchMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(QUERY_SEARCH_PHASE_METRIC); - assertTrue( - Stream.concat(queryMeasurements.stream(), fetchMeasurements.stream()).allMatch(m -> checkMeasurementAttributes(m, isSystem)) - ); - } - - private boolean checkMeasurementAttributes(Measurement m, boolean isSystem) { - return ((boolean) m.attributes().get(SYSTEM_THREAD_ATTRIBUTE_NAME)) == isSystem; + private static void assertAttributes(List measurements, boolean isSystem, boolean isScroll) { + for (Measurement measurement : measurements) { + Map attributes = measurement.attributes(); + assertEquals(isScroll ? 5 : 4, attributes.size()); + if (isSystem) { + assertEquals(".others", attributes.get("target")); + } else { + assertEquals("user", attributes.get("target")); + } + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + if (isScroll) { + assertEquals("scroll", attributes.get("pit_scroll")); + } + assertEquals(isSystem, measurement.attributes().get(SearchRequestAttributesExtractor.SYSTEM_THREAD_ATTRIBUTE_NAME)); + } } public static class TestSystemIndexPlugin extends Plugin implements SystemIndexPlugin { From da04c59b6bd46711e4057a51c391492602f95bb3 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 16 Sep 2025 13:10:40 +0200 Subject: [PATCH 2/5] Update docs/changelog/134798.yaml --- docs/changelog/134798.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/134798.yaml diff --git a/docs/changelog/134798.yaml b/docs/changelog/134798.yaml new file mode 100644 index 0000000000000..55b49df0d0b16 --- /dev/null +++ b/docs/changelog/134798.yaml @@ -0,0 +1,5 @@ +pr: 134798 +summary: Add relevant attributes to shard search latency APM metrics +area: Search +type: enhancement +issues: [] From b1413f6d2bbe8cd750dc842272b9311ba2d0b351 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Thu, 18 Sep 2025 11:20:11 +0200 Subject: [PATCH 3/5] iter --- .../SearchRequestAttributesExtractor.java | 14 ++- .../ShardSearchPhaseAPMMetricsTests.java | 85 ++++++++++++++++--- 2 files changed, 84 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java index 284a6184ea57f..fcc52e8892c18 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java @@ -290,8 +290,18 @@ private static void introspectQueryBuilder(QueryBuilder queryBuilder, QueryMetad break; case RangeQueryBuilder range: switch (range.fieldName()) { - case TIMESTAMP -> queryMetadataBuilder.rangeOnTimestamp = true; - case EVENT_INGESTED -> queryMetadataBuilder.rangeOnEventIngested = true; + // don't track unbounded ranges, they translate to either match_none if the field does not exist + // or match_all if the field is mapped + case TIMESTAMP -> { + if (range.to() != null || range.from() != null) { + queryMetadataBuilder.rangeOnTimestamp = true; + } + } + case EVENT_INGESTED -> { + if (range.to() != null || range.from() != null) { + queryMetadataBuilder.rangeOnEventIngested = true; + } + } } break; case KnnVectorQueryBuilder knn: diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java index ba1b5914b8ed4..865ea3d8bf6bd 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java @@ -14,6 +14,8 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.indices.ExecutorNames; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.plugins.Plugin; @@ -34,7 +36,11 @@ import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery; import static org.elasticsearch.index.search.stats.ShardSearchPhaseAPMMetrics.FETCH_SEARCH_PHASE_METRIC; import static org.elasticsearch.index.search.stats.ShardSearchPhaseAPMMetrics.QUERY_SEARCH_PHASE_METRIC; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertScrollResponsesAndHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHitsWithoutFailures; public class ShardSearchPhaseAPMMetricsTests extends ESSingleNodeTestCase { @@ -58,11 +64,17 @@ public void setUpIndex() { ); ensureGreen(indexName); - prepareIndex(indexName).setId("1").setSource("body", "doc1").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex(indexName).setId("2").setSource("body", "doc2").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(indexName).setId("1").setSource("body", "doc1", "@timestamp", "2024-11-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(indexName).setId("2").setSource("body", "doc2", "@timestamp", "2024-12-01").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex(TestSystemIndexPlugin.INDEX_NAME).setId("1").setSource("body", "doc1").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex(TestSystemIndexPlugin.INDEX_NAME).setId("2").setSource("body", "doc2").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(TestSystemIndexPlugin.INDEX_NAME).setId("1") + .setSource("body", "doc1", "@timestamp", "2024-11-01") + .setRefreshPolicy(IMMEDIATE) + .get(); + prepareIndex(TestSystemIndexPlugin.INDEX_NAME).setId("2") + .setSource("body", "doc2", "@timestamp", "2024-12-01") + .setRefreshPolicy(IMMEDIATE) + .get(); } @After @@ -233,14 +245,6 @@ public void testSearchTransportMetricsScrollSystem() { ); } - private void resetMeter() { - getTestTelemetryPlugin().resetMeter(); - } - - private TestTelemetryPlugin getTestTelemetryPlugin() { - return getInstanceFromNode(PluginsService.class).filterPlugins(TestTelemetryPlugin.class).toList().get(0); - } - private static void assertAttributes(List measurements, boolean isSystem, boolean isScroll) { for (Measurement measurement : measurements) { Map attributes = measurement.attributes(); @@ -255,10 +259,65 @@ private static void assertAttributes(List measurements, boolean isS if (isScroll) { assertEquals("scroll", attributes.get("pit_scroll")); } - assertEquals(isSystem, measurement.attributes().get(SearchRequestAttributesExtractor.SYSTEM_THREAD_ATTRIBUTE_NAME)); + assertEquals(isSystem, attributes.get(SearchRequestAttributesExtractor.SYSTEM_THREAD_ATTRIBUTE_NAME)); } } + public void testTimeRangeFilterOneResult() { + RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder("@timestamp").from("2024-12-01"); + // target the system index because it has one shard, that simplifies testing. Otherwise, only when the two docs end up indexed + // on the same shard do you get the time range as attribute. + assertSearchHitsWithoutFailures(client().prepareSearch(TestSystemIndexPlugin.INDEX_NAME).setQuery(rangeQueryBuilder), "2"); + final List queryMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(QUERY_SEARCH_PHASE_METRIC); + assertEquals(1, queryMeasurements.size()); + assertTimeRangeAttributes(queryMeasurements, ".others", true); + final List fetchMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(FETCH_SEARCH_PHASE_METRIC); + assertEquals(1, fetchMeasurements.size()); + assertTimeRangeAttributes(fetchMeasurements, ".others", true); + } + + private static void assertTimeRangeAttributes(List measurements, String target, boolean isSystem) { + for (Measurement measurement : measurements) { + Map attributes = measurement.attributes(); + assertEquals(5, attributes.size()); + assertEquals(target, attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals(true, attributes.get("range_timestamp")); + assertEquals(isSystem, attributes.get(SearchRequestAttributesExtractor.SYSTEM_THREAD_ATTRIBUTE_NAME)); + } + } + + public void testTimeRangeFilterAllResults() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from("2024-10-01")); + // enable can match: empty shards get filtered out by the can match round + assertResponse(client().prepareSearch(indexName).setPreFilterShardSize(1).setQuery(boolQueryBuilder), searchResponse -> { + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 2); + assertSearchHits(searchResponse, "1", "2"); + assertThat(searchResponse.getSkippedShards(), Matchers.greaterThanOrEqualTo(num_primaries - 2)); + }); + final List queryMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(QUERY_SEARCH_PHASE_METRIC); + // the two docs are at most spread across two shards, other shards are empty and get filtered out + assertThat(queryMeasurements.size(), Matchers.lessThanOrEqualTo(2)); + // no range info stored because we had no bounds after rewrite, basically a match_all + assertAttributes(queryMeasurements, false, false); + final List fetchMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(FETCH_SEARCH_PHASE_METRIC); + // in this case, each shard queried has results to be fetched + assertEquals(queryMeasurements.size(), fetchMeasurements.size()); + // no range info stored because we had no bounds after rewrite, basically a match_all + assertAttributes(fetchMeasurements, false, false); + } + + private void resetMeter() { + getTestTelemetryPlugin().resetMeter(); + } + + private TestTelemetryPlugin getTestTelemetryPlugin() { + return getInstanceFromNode(PluginsService.class).filterPlugins(TestTelemetryPlugin.class).toList().get(0); + } + public static class TestSystemIndexPlugin extends Plugin implements SystemIndexPlugin { static final String INDEX_NAME = ".test-system-index"; From c575028767bba5bfb51c22cc240d6d33121dd3fc Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Thu, 18 Sep 2025 11:54:28 +0200 Subject: [PATCH 4/5] adjust tests --- ...SearchRequestAttributesExtractorTests.java | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java index ceb7e8cc9e8de..ba9ac5b69807b 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java @@ -188,6 +188,18 @@ public void testExtractAttributes() { searchRequest, searchRequest.indices() ); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, false, false, null); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + searchSourceBuilder.query(new RangeQueryBuilder("@timestamp").from("2021-11-11")); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null); } { @@ -202,10 +214,10 @@ public void testExtractAttributes() { boolQueryBuilder.must(boolQueryBuilderNew); boolQueryBuilder = boolQueryBuilderNew; } - boolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); + boolQueryBuilder.must(new RangeQueryBuilder("@timestamp").from("2021-11-11")); searchSourceBuilder.query(boolQueryBuilder); if (randomBoolean()) { - boolQueryBuilder.should(new RangeQueryBuilder("event.ingested")); + boolQueryBuilder.should(new RangeQueryBuilder("event.ingested").from("2021-11-11")); } Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( searchRequest, @@ -229,7 +241,7 @@ public void testExtractAttributes() { boolQueryBuilder.should(new RangeQueryBuilder("event.ingested")); } - boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp")); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from("2021-11-11")); searchSourceBuilder.query(boolQueryBuilder); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( searchRequest, @@ -243,8 +255,8 @@ public void testExtractAttributes() { searchRequest.source(searchSourceBuilder); searchSourceBuilder.sort("@timestamp"); BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - boolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); - boolQueryBuilder.must(new RangeQueryBuilder("event.ingested")); + boolQueryBuilder.must(new RangeQueryBuilder("@timestamp").from("2021-11-11")); + boolQueryBuilder.must(new RangeQueryBuilder("event.ingested").from("2021-11-11")); boolQueryBuilder.must(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10))); searchSourceBuilder.query(boolQueryBuilder); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( @@ -259,7 +271,7 @@ public void testExtractAttributes() { searchRequest.source(searchSourceBuilder); searchSourceBuilder.sort("@timestamp"); BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - boolQueryBuilder.should(new RangeQueryBuilder("@timestamp")); + boolQueryBuilder.should(new RangeQueryBuilder("@timestamp").from("2021-11-11")); searchSourceBuilder.query(boolQueryBuilder); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( searchRequest, @@ -273,7 +285,7 @@ public void testExtractAttributes() { searchRequest.source(searchSourceBuilder); searchSourceBuilder.sort("@timestamp"); BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - boolQueryBuilder.should(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10))); + boolQueryBuilder.should(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10)).from("2021-11-11")); searchSourceBuilder.query(boolQueryBuilder); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( searchRequest, @@ -286,7 +298,7 @@ public void testExtractAttributes() { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchRequest.source(searchSourceBuilder); searchSourceBuilder.sort("@timestamp"); - searchSourceBuilder.query(new ConstantScoreQueryBuilder(new RangeQueryBuilder("@timestamp"))); + searchSourceBuilder.query(new ConstantScoreQueryBuilder(new RangeQueryBuilder("@timestamp").from("2021-11-11"))); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( searchRequest, searchRequest.indices() @@ -298,7 +310,7 @@ public void testExtractAttributes() { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchRequest.source(searchSourceBuilder); searchSourceBuilder.sort("@timestamp"); - searchSourceBuilder.query(new BoostingQueryBuilder(new RangeQueryBuilder("@timestamp"), new MatchAllQueryBuilder())); + searchSourceBuilder.query(new BoostingQueryBuilder(new RangeQueryBuilder("@timestamp").from("2021-11-11"), new MatchAllQueryBuilder())); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( searchRequest, searchRequest.indices() @@ -320,7 +332,7 @@ public void testDepthLimit() { newBoolQueryBuilder.must(innerBoolQueryBuilder); newBoolQueryBuilder = innerBoolQueryBuilder; } - newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); + newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp").from("2021-11-11")); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( searchRequest, searchRequest.indices() @@ -339,7 +351,7 @@ public void testDepthLimit() { newBoolQueryBuilder.must(innerBoolQueryBuilder); newBoolQueryBuilder = innerBoolQueryBuilder; } - newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); + newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp").from("2021-11-11")); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( searchRequest, searchRequest.indices() From a96f84f4ed345a8afc11cd319251a4899b1a5920 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 18 Sep 2025 10:02:21 +0000 Subject: [PATCH 5/5] [CI] Auto commit changes from spotless --- .../action/search/SearchRequestAttributesExtractorTests.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java index ba9ac5b69807b..ad695eb49f99e 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java @@ -310,7 +310,9 @@ public void testExtractAttributes() { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchRequest.source(searchSourceBuilder); searchSourceBuilder.sort("@timestamp"); - searchSourceBuilder.query(new BoostingQueryBuilder(new RangeQueryBuilder("@timestamp").from("2021-11-11"), new MatchAllQueryBuilder())); + searchSourceBuilder.query( + new BoostingQueryBuilder(new RangeQueryBuilder("@timestamp").from("2021-11-11"), new MatchAllQueryBuilder()) + ); Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( searchRequest, searchRequest.indices()