Measure search load per index #122262
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| pr: 122262 | ||
| summary: Measure search load per index | ||
| area: Search | ||
| type: feature | ||
| issues: [] |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the "Elastic License | ||
| * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side | ||
| * Public License v 1"; you may not use this file except in compliance with, at | ||
| * your election, the "Elastic License 2.0", the "GNU Affero General Public | ||
| * License v3.0 only", or the "Server Side Public License, v 1". | ||
| */ | ||
|
|
||
| package org.elasticsearch.cluster.coordination; | ||
|
|
||
| import org.elasticsearch.cluster.ClusterChangedEvent; | ||
| import org.elasticsearch.cluster.ClusterStateListener; | ||
| import org.elasticsearch.index.Index; | ||
| import org.elasticsearch.index.search.stats.ShardSearchPerIndexTimeTrackingMetrics; | ||
|
|
||
| /** | ||
| * Service responsible for cleaning up task execution time tracking for deleted indices. | ||
| * Implements the ClusterStateListener interface to listen for cluster state changes. | ||
| */ | ||
| public class SearchIndexTimeTrackingCleanupService implements ClusterStateListener { | ||
|
|
||
| private ShardSearchPerIndexTimeTrackingMetrics listener; | ||
|
|
||
| /** | ||
| * Constructor. | ||
| * | ||
| * @param listener the listener for shard search time tracking metrics | ||
| */ | ||
| public SearchIndexTimeTrackingCleanupService(ShardSearchPerIndexTimeTrackingMetrics listener) { | ||
| this.listener = listener; | ||
| } | ||
|
|
||
| /** | ||
| * Called when the cluster state changes. Stops tracking execution time for deleted indices. | ||
| * | ||
| * @param event the cluster changed event | ||
| */ | ||
| @Override | ||
| public void clusterChanged(ClusterChangedEvent event) { | ||
| for (Index index : event.indicesDeleted()) { | ||
| listener.stopTrackingIndex(index.getName()); | ||
| } | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,146 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the "Elastic License | ||
| * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side | ||
| * Public License v 1"; you may not use this file except in compliance with, at | ||
| * your election, the "Elastic License 2.0", the "GNU Affero General Public | ||
| * License v3.0 only", or the "Server Side Public License, v 1". | ||
| */ | ||
|
|
||
| package org.elasticsearch.index.search.stats; | ||
|
|
||
| import org.apache.logging.log4j.LogManager; | ||
| import org.apache.logging.log4j.Logger; | ||
| import org.elasticsearch.common.ExponentiallyWeightedMovingAverage; | ||
| import org.elasticsearch.core.Tuple; | ||
| import org.elasticsearch.index.shard.IndexShard; | ||
| import org.elasticsearch.index.shard.SearchOperationListener; | ||
| import org.elasticsearch.search.internal.SearchContext; | ||
|
|
||
| import java.util.concurrent.ConcurrentHashMap; | ||
| import java.util.concurrent.atomic.LongAdder; | ||
|
|
||
| /** | ||
| * This class implements the {@link SearchOperationListener} interface to track the execution time of search operations | ||
| * on a per-index basis. It uses a {@link ConcurrentHashMap} to store the execution times and an | ||
| * {@link ExponentiallyWeightedMovingAverage} to calculate the exponentially weighted moving average (EWMA) of the | ||
| * execution times. | ||
| */ | ||
| public final class ShardSearchPerIndexTimeTrackingMetrics implements SearchOperationListener { | ||
|
|
||
| private static final Logger logger = LogManager.getLogger(ShardSearchPerIndexTimeTrackingMetrics.class); | ||
|
|
||
| private final ConcurrentHashMap<String, Tuple<LongAdder, ExponentiallyWeightedMovingAverage>> indexExecutionTime; | ||
|
|
||
| private final double ewmaAlpha; | ||
|
|
||
| /** | ||
| * Constructs a new ShardSearchPerIndexTimeTrackingMetrics instance with the specified EWMA alpha value. | ||
| * | ||
| * @param ewmaAlpha the alpha value for the EWMA calculation | ||
| */ | ||
| public ShardSearchPerIndexTimeTrackingMetrics(double ewmaAlpha) { | ||
| this.indexExecutionTime = new ConcurrentHashMap<>(); | ||
| this.ewmaAlpha = ewmaAlpha; | ||
| } | ||
|
|
||
| /** | ||
| * Tracks the execution time of the query phase of a search operation. | ||
| * | ||
| * @param searchContext the search context | ||
| * @param tookInNanos the time taken in nanoseconds | ||
| */ | ||
| @Override | ||
| public void onQueryPhase(SearchContext searchContext, long tookInNanos) { | ||
| trackExecutionTime(searchContext, tookInNanos); | ||
| } | ||
|
|
||
| /** | ||
| * Tracks the execution time of a failed query phase of a search operation. | ||
| * | ||
| * @param searchContext the search context | ||
| * @param tookInNanos the time taken in nanoseconds | ||
| */ | ||
| @Override | ||
| public void onFailedQueryPhase(SearchContext searchContext, long tookInNanos) { | ||
| trackExecutionTime(searchContext, tookInNanos); | ||
| } | ||
|
|
||
| /** | ||
| * Tracks the execution time of the fetch phase of a search operation. | ||
| * | ||
| * @param searchContext the search context | ||
| * @param tookInNanos the time taken in nanoseconds | ||
| */ | ||
| @Override | ||
| public void onFetchPhase(SearchContext searchContext, long tookInNanos) { | ||
| trackExecutionTime(searchContext, tookInNanos); | ||
| } | ||
|
|
||
| /** | ||
| * Tracks the execution time of a failed fetch phase of a search operation. | ||
| * | ||
| * @param searchContext the search context | ||
| * @param tookInNanos the time taken in nanoseconds | ||
| */ | ||
| @Override | ||
| public void onFailedFetchPhase(SearchContext searchContext, long tookInNanos) { | ||
| trackExecutionTime(searchContext, tookInNanos); | ||
| } | ||
|
|
||
| /** | ||
| * Tracks the execution time of a search operation. | ||
| * | ||
| * @param searchContext the search context | ||
| * @param tookInNanos the time taken in nanoseconds | ||
| */ | ||
| private void trackExecutionTime(SearchContext searchContext, long tookInNanos) { | ||
| IndexShard indexShard = searchContext.indexShard(); | ||
| if (indexShard != null && indexShard.isSystem() == false) { | ||
| String indexName = indexShard.shardId().getIndexName(); | ||
| if (indexName != null) { | ||
| Tuple<LongAdder, ExponentiallyWeightedMovingAverage> t = indexExecutionTime.computeIfAbsent( | ||
| indexName, | ||
| k -> new Tuple<>(new LongAdder(), new ExponentiallyWeightedMovingAverage(ewmaAlpha, 0)) | ||
| ); | ||
| t.v1().add(tookInNanos); | ||
| t.v2().addValue(tookInNanos); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Gets the total execution time for tasks associated with a specific index. | ||
| * | ||
| * @param indexName the name of the index | ||
| * @return the total execution time for the index | ||
| */ | ||
| public long getSearchLoadPerIndex(String indexName) { | ||
| Tuple<LongAdder, ExponentiallyWeightedMovingAverage> t = indexExecutionTime.get(indexName); | ||
| return (t != null) ? t.v1().sum() : 0; | ||
| } | ||
|
|
||
| /** | ||
| * Gets the exponentially weighted moving average (EWMA) of the execution time for tasks associated with a specific index name. | ||
| * | ||
| * @param indexName the name of the index | ||
| * @return the EWMA of the execution time for the index | ||
| */ | ||
| public double getLoadEMWAPerIndex(String indexName) { | ||
|
There was a problem hiding this comment. The There was a problem hiding this comment. Maybe some more context here: |
||
| Tuple<LongAdder, ExponentiallyWeightedMovingAverage> t = indexExecutionTime.get(indexName); | ||
| return (t != null) ? t.v2().getAverage() : 0; | ||
| } | ||
|
|
||
| /** | ||
| * Stops tracking the execution time for tasks associated with a specific index. | ||
| * | ||
| * @param indexName the name of the index | ||
| */ | ||
| public void stopTrackingIndex(String indexName) { | ||
| if (indexExecutionTime.containsKey(indexName)) { | ||
| indexExecutionTime.remove(indexName); | ||
| } else { | ||
| logger.debug("Trying to stop tracking index [{}] that was never tracked", indexName); | ||
| } | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I'm sorry for saying it in such a straightforward manner, but do we really want to add more logic based on this class?
Especially on a per-shard basis, using the math in
ExponentiallyWeightedMovingAverageseems questionable.We calculate
newValue = alpha * lastValue + (1 - alpha) * currentValue. In a large number of use-cases you may see the fetch and query times be an order of magnitude apart. So now, assuming the query always matches, we will essentially flap between two values constantly for a shard?Why not just use the existing metrics we have in
org.elasticsearch.index.search.stats.ShardSearchStats? EWMA makes no sense here, if anything isn't total query time and it's derivative what we care about?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Good point, we left it in because we where not sure about it and I asked Dimi to leave a comment on the PR about this.
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
All right, let's check how it can be adapted into the
ShardSearchStats