Skip to content

Commit 5e7f305

Browse files
javannajavanna
authored
Add relevant attributes to shard search latency APM metrics (#134798)
We already record latency of the query and fetch phase as APM metrics. We'd like to be able to slice such latencies based on some recurring categories of the request: - does it have agg or hit only? - is it sorted by field or by score? - does it have a time range filter? - does it target user data or internal indices? This commit introduces introspection for a shard search request and stores the extracted attributes together with the shard phase latency metrics. This builds on top of #134232 to use the same infra and store the same attributes for shard level latency metrics.
1 parent 4bb963e commit 5e7f305

File tree

5 files changed

+285
-93
lines changed

5 files changed

+285
-93
lines changed

‎docs/changelog/134798.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 134798
2+
summary: Add relevant attributes to shard search latency APM metrics
3+
area: Search
4+
type: enhancement
5+
issues: []

‎server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java‎

Lines changed: 40 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
import org.apache.logging.log4j.LogManager;
1313
import org.apache.logging.log4j.Logger;
1414
import org.elasticsearch.common.regex.Regex;
15+
import org.elasticsearch.common.util.concurrent.EsExecutors;
16+
import org.elasticsearch.core.TimeValue;
1517
import org.elasticsearch.index.query.BoolQueryBuilder;
1618
import org.elasticsearch.index.query.BoostingQueryBuilder;
1719
import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
@@ -20,6 +22,7 @@
2022
import org.elasticsearch.index.query.RangeQueryBuilder;
2123
import org.elasticsearch.search.SearchService;
2224
import org.elasticsearch.search.builder.SearchSourceBuilder;
25+
import org.elasticsearch.search.internal.ShardSearchRequest;
2326
import org.elasticsearch.search.sort.FieldSortBuilder;
2427
import org.elasticsearch.search.sort.ScoreSortBuilder;
2528
import org.elasticsearch.search.sort.SortBuilder;
@@ -42,17 +45,37 @@ private SearchRequestAttributesExtractor() {}
4245

4346
/**
4447
* Introspects the provided search request and extracts metadata from it about some of its characteristics.
45-
*
4648
*/
4749
public static Map<String, Object> extractAttributes(SearchRequest searchRequest, String[] localIndices) {
50+
return extractAttributes(searchRequest.source(), searchRequest.scroll(), localIndices);
51+
}
52+
53+
/**
54+
* Introspects the provided shard search request and extracts metadata from it about some of its characteristics.
55+
*/
56+
public static Map<String, Object> extractAttributes(ShardSearchRequest shardSearchRequest) {
57+
Map<String, Object> attributes = extractAttributes(
58+
shardSearchRequest.source(),
59+
shardSearchRequest.scroll(),
60+
shardSearchRequest.shardId().getIndexName()
61+
);
62+
boolean isSystem = ((EsExecutors.EsThread) Thread.currentThread()).isSystem();
63+
attributes.put(SYSTEM_THREAD_ATTRIBUTE_NAME, isSystem);
64+
return attributes;
65+
}
66+
67+
private static Map<String, Object> extractAttributes(
68+
SearchSourceBuilder searchSourceBuilder,
69+
TimeValue scroll,
70+
String... localIndices
71+
) {
4872
String target = extractIndices(localIndices);
4973

5074
String pitOrScroll = null;
51-
if (searchRequest.scroll() != null) {
75+
if (scroll != null) {
5276
pitOrScroll = SCROLL;
5377
}
5478

55-
SearchSourceBuilder searchSourceBuilder = searchRequest.source();
5679
if (searchSourceBuilder == null) {
5780
return buildAttributesMap(target, ScoreSortBuilder.NAME, HITS_ONLY, false, false, false, pitOrScroll);
5881
}
@@ -144,7 +167,7 @@ private static final class QueryMetadataBuilder {
144167
private static final String TARGET_USER = "user";
145168
private static final String ERROR = "error";
146169

147-
static String extractIndices(String[] indices) {
170+
static String extractIndices(String... indices) {
148171
try {
149172
// Note that indices are expected to be resolved, meaning wildcards are not handled on purpose
150173
// If indices resolve to data streams, the name of the data stream is returned as opposed to its backing indices
@@ -213,6 +236,7 @@ static String extractPrimarySort(SortBuilder<?> primarySortBuilder) {
213236
private static final String PIT = "pit";
214237
private static final String SCROLL = "scroll";
215238

239+
public static final String SYSTEM_THREAD_ATTRIBUTE_NAME = "system_thread";
216240
public static final Map<String, Object> SEARCH_SCROLL_ATTRIBUTES = Map.of(QUERY_TYPE_ATTRIBUTE, SCROLL);
217241

218242
static String extractQueryType(SearchSourceBuilder searchSourceBuilder) {
@@ -266,8 +290,18 @@ private static void introspectQueryBuilder(QueryBuilder queryBuilder, QueryMetad
266290
break;
267291
case RangeQueryBuilder range:
268292
switch (range.fieldName()) {
269-
case TIMESTAMP -> queryMetadataBuilder.rangeOnTimestamp = true;
270-
case EVENT_INGESTED -> queryMetadataBuilder.rangeOnEventIngested = true;
293+
// don't track unbounded ranges, they translate to either match_none if the field does not exist
294+
// or match_all if the field is mapped
295+
case TIMESTAMP -> {
296+
if (range.to() != null || range.from() != null) {
297+
queryMetadataBuilder.rangeOnTimestamp = true;
298+
}
299+
}
300+
case EVENT_INGESTED -> {
301+
if (range.to() != null || range.from() != null) {
302+
queryMetadataBuilder.rangeOnEventIngested = true;
303+
}
304+
}
271305
}
272306
break;
273307
case KnnVectorQueryBuilder knn:

‎server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java‎

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,13 @@
99

1010
package org.elasticsearch.index.search.stats;
1111

12-
import org.elasticsearch.common.util.concurrent.EsExecutors;
12+
import org.elasticsearch.action.search.SearchRequestAttributesExtractor;
1313
import org.elasticsearch.index.shard.SearchOperationListener;
1414
import org.elasticsearch.search.internal.SearchContext;
15+
import org.elasticsearch.search.internal.ShardSearchRequest;
1516
import org.elasticsearch.telemetry.metric.LongHistogram;
1617
import org.elasticsearch.telemetry.metric.MeterRegistry;
1718

18-
import java.util.HashMap;
1919
import java.util.Map;
2020
import java.util.concurrent.TimeUnit;
2121

@@ -24,14 +24,9 @@ public final class ShardSearchPhaseAPMMetrics implements SearchOperationListener
2424
public static final String QUERY_SEARCH_PHASE_METRIC = "es.search.shards.phases.query.duration.histogram";
2525
public static final String FETCH_SEARCH_PHASE_METRIC = "es.search.shards.phases.fetch.duration.histogram";
2626

27-
public static final String SYSTEM_THREAD_ATTRIBUTE_NAME = "system_thread";
28-
2927
private final LongHistogram queryPhaseMetric;
3028
private final LongHistogram fetchPhaseMetric;
3129

32-
// Avoid allocating objects in the search path and multithreading clashes
33-
private static final ThreadLocal<Map<String, Object>> THREAD_LOCAL_ATTRS = ThreadLocal.withInitial(() -> new HashMap<>(1));
34-
3530
public ShardSearchPhaseAPMMetrics(MeterRegistry meterRegistry) {
3631
this.queryPhaseMetric = meterRegistry.registerLongHistogram(
3732
QUERY_SEARCH_PHASE_METRIC,
@@ -47,18 +42,16 @@ public ShardSearchPhaseAPMMetrics(MeterRegistry meterRegistry) {
4742

4843
@Override
4944
public void onQueryPhase(SearchContext searchContext, long tookInNanos) {
50-
recordPhaseLatency(queryPhaseMetric, tookInNanos);
45+
recordPhaseLatency(queryPhaseMetric, tookInNanos, searchContext.request());
5146
}
5247

5348
@Override
5449
public void onFetchPhase(SearchContext searchContext, long tookInNanos) {
55-
recordPhaseLatency(fetchPhaseMetric, tookInNanos);
50+
recordPhaseLatency(fetchPhaseMetric, tookInNanos, searchContext.request());
5651
}
5752

58-
private static void recordPhaseLatency(LongHistogram histogramMetric, long tookInNanos) {
59-
Map<String, Object> attrs = ShardSearchPhaseAPMMetrics.THREAD_LOCAL_ATTRS.get();
60-
boolean isSystem = ((EsExecutors.EsThread) Thread.currentThread()).isSystem();
61-
attrs.put(SYSTEM_THREAD_ATTRIBUTE_NAME, isSystem);
62-
histogramMetric.record(TimeUnit.NANOSECONDS.toMillis(tookInNanos), attrs);
53+
private static void recordPhaseLatency(LongHistogram histogramMetric, long tookInNanos, ShardSearchRequest request) {
54+
Map<String, Object> attributes = SearchRequestAttributesExtractor.extractAttributes(request);
55+
histogramMetric.record(TimeUnit.NANOSECONDS.toMillis(tookInNanos), attributes);
6356
}
6457
}

‎server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,18 @@ public void testExtractAttributes() {
188188
searchRequest,
189189
searchRequest.indices()
190190
);
191+
assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, false, false, null);
192+
}
193+
{
194+
SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10));
195+
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
196+
searchRequest.source(searchSourceBuilder);
197+
searchSourceBuilder.sort("@timestamp");
198+
searchSourceBuilder.query(new RangeQueryBuilder("@timestamp").from("2021-11-11"));
199+
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
200+
searchRequest,
201+
searchRequest.indices()
202+
);
191203
assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null);
192204
}
193205
{
@@ -202,10 +214,10 @@ public void testExtractAttributes() {
202214
boolQueryBuilder.must(boolQueryBuilderNew);
203215
boolQueryBuilder = boolQueryBuilderNew;
204216
}
205-
boolQueryBuilder.must(new RangeQueryBuilder("@timestamp"));
217+
boolQueryBuilder.must(new RangeQueryBuilder("@timestamp").from("2021-11-11"));
206218
searchSourceBuilder.query(boolQueryBuilder);
207219
if (randomBoolean()) {
208-
boolQueryBuilder.should(new RangeQueryBuilder("event.ingested"));
220+
boolQueryBuilder.should(new RangeQueryBuilder("event.ingested").from("2021-11-11"));
209221
}
210222
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
211223
searchRequest,
@@ -229,7 +241,7 @@ public void testExtractAttributes() {
229241
boolQueryBuilder.should(new RangeQueryBuilder("event.ingested"));
230242
}
231243

232-
boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp"));
244+
boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from("2021-11-11"));
233245
searchSourceBuilder.query(boolQueryBuilder);
234246
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
235247
searchRequest,
@@ -243,8 +255,8 @@ public void testExtractAttributes() {
243255
searchRequest.source(searchSourceBuilder);
244256
searchSourceBuilder.sort("@timestamp");
245257
BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
246-
boolQueryBuilder.must(new RangeQueryBuilder("@timestamp"));
247-
boolQueryBuilder.must(new RangeQueryBuilder("event.ingested"));
258+
boolQueryBuilder.must(new RangeQueryBuilder("@timestamp").from("2021-11-11"));
259+
boolQueryBuilder.must(new RangeQueryBuilder("event.ingested").from("2021-11-11"));
248260
boolQueryBuilder.must(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10)));
249261
searchSourceBuilder.query(boolQueryBuilder);
250262
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
@@ -259,7 +271,7 @@ public void testExtractAttributes() {
259271
searchRequest.source(searchSourceBuilder);
260272
searchSourceBuilder.sort("@timestamp");
261273
BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
262-
boolQueryBuilder.should(new RangeQueryBuilder("@timestamp"));
274+
boolQueryBuilder.should(new RangeQueryBuilder("@timestamp").from("2021-11-11"));
263275
searchSourceBuilder.query(boolQueryBuilder);
264276
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
265277
searchRequest,
@@ -273,7 +285,7 @@ public void testExtractAttributes() {
273285
searchRequest.source(searchSourceBuilder);
274286
searchSourceBuilder.sort("@timestamp");
275287
BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
276-
boolQueryBuilder.should(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10)));
288+
boolQueryBuilder.should(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10)).from("2021-11-11"));
277289
searchSourceBuilder.query(boolQueryBuilder);
278290
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
279291
searchRequest,
@@ -286,7 +298,7 @@ public void testExtractAttributes() {
286298
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
287299
searchRequest.source(searchSourceBuilder);
288300
searchSourceBuilder.sort("@timestamp");
289-
searchSourceBuilder.query(new ConstantScoreQueryBuilder(new RangeQueryBuilder("@timestamp")));
301+
searchSourceBuilder.query(new ConstantScoreQueryBuilder(new RangeQueryBuilder("@timestamp").from("2021-11-11")));
290302
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
291303
searchRequest,
292304
searchRequest.indices()
@@ -298,7 +310,9 @@ public void testExtractAttributes() {
298310
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
299311
searchRequest.source(searchSourceBuilder);
300312
searchSourceBuilder.sort("@timestamp");
301-
searchSourceBuilder.query(new BoostingQueryBuilder(new RangeQueryBuilder("@timestamp"), new MatchAllQueryBuilder()));
313+
searchSourceBuilder.query(
314+
new BoostingQueryBuilder(new RangeQueryBuilder("@timestamp").from("2021-11-11"), new MatchAllQueryBuilder())
315+
);
302316
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
303317
searchRequest,
304318
searchRequest.indices()
@@ -320,7 +334,7 @@ public void testDepthLimit() {
320334
newBoolQueryBuilder.must(innerBoolQueryBuilder);
321335
newBoolQueryBuilder = innerBoolQueryBuilder;
322336
}
323-
newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp"));
337+
newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp").from("2021-11-11"));
324338
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
325339
searchRequest,
326340
searchRequest.indices()
@@ -339,7 +353,7 @@ public void testDepthLimit() {
339353
newBoolQueryBuilder.must(innerBoolQueryBuilder);
340354
newBoolQueryBuilder = innerBoolQueryBuilder;
341355
}
342-
newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp"));
356+
newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp").from("2021-11-11"));
343357
Map<String, Object> stringObjectMap = SearchRequestAttributesExtractor.extractAttributes(
344358
searchRequest,
345359
searchRequest.indices()

0 commit comments

Comments
 (0)