From 168c6c53793ff0e0a36685ba2c47abd1c9bac316 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 8 Jul 2025 16:09:31 -0500 Subject: [PATCH 01/34] First attempt at improving statsByShard performance --- .../admin/indices/stats/CommonStats.java | 23 ++++- .../indices/IndicesQueryCache.java | 63 ++++++++++++++ .../elasticsearch/indices/IndicesService.java | 87 +++++++++++++++---- 3 files changed, 153 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java index 1351674fffbfa..024d6af9cda43 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java @@ -155,6 +155,18 @@ public CommonStats(CommonStatsFlags flags) { * Filters the given flags for {@link CommonStatsFlags#SHARD_LEVEL} flags and calculates the corresponding statistics. */ public static CommonStats getShardLevelStats(IndicesQueryCache indicesQueryCache, IndexShard indexShard, CommonStatsFlags flags) { + return getShardLevelStats(indicesQueryCache, indexShard, flags, null); + } + + /** + * Overload that takes a precomputed shared RAM map for O(N) stats. + */ + public static CommonStats getShardLevelStats( + IndicesQueryCache indicesQueryCache, + IndexShard indexShard, + CommonStatsFlags flags, + java.util.Map precomputedSharedRam + ) { // Filter shard level flags CommonStatsFlags filteredFlags = flags.clone(); for (CommonStatsFlags.Flag flag : filteredFlags.getFlags()) { @@ -174,7 +186,16 @@ public static CommonStats getShardLevelStats(IndicesQueryCache indicesQueryCache case Refresh -> stats.refresh = indexShard.refreshStats(); case Flush -> stats.flush = indexShard.flushStats(); case Warmer -> stats.warmer = indexShard.warmerStats(); - case QueryCache -> stats.queryCache = indicesQueryCache.getStats(indexShard.shardId()); + case QueryCache -> { + if (precomputedSharedRam != null && precomputedSharedRam.containsKey(indexShard.shardId())) { + stats.queryCache = indicesQueryCache.getStats( + indexShard.shardId(), + precomputedSharedRam.get(indexShard.shardId()) + ); + } else { + stats.queryCache = indicesQueryCache.getStats(indexShard.shardId()); + } + } case FieldData -> stats.fieldData = indexShard.fieldDataStats(flags.fieldDataFields()); case Completion -> stats.completion = indexShard.completionStats(flags.completionDataFields()); case Segments -> stats.segments = indexShard.segmentStats( diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index 9bca59e9e4d62..2a9b9982d3acc 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -32,6 +32,7 @@ import java.io.Closeable; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.IdentityHashMap; import java.util.Map; import java.util.Set; @@ -67,6 +68,17 @@ public class IndicesQueryCache implements QueryCache, Closeable { private final Map shardStats = new ConcurrentHashMap<>(); private volatile long sharedRamBytesUsed; + // Package-private for IndicesService efficient stats collection + long getCacheSizeForShard(ShardId shardId) { + Stats stats = shardStats.get(shardId); + return stats != null ? stats.cacheSize : 0L; + } + + // Package-private for IndicesService efficient stats collection + long getSharedRamBytesUsed() { + return sharedRamBytesUsed; + } + // This is a hack for the fact that the close listener for the // ShardCoreKeyMap will be called before onDocIdSetEviction // See onDocIdSetEviction for more info @@ -139,6 +151,57 @@ public QueryCacheStats getStats(ShardId shard) { return queryCacheStats; } + /** + * Overload to allow passing in a precomputed shared RAM split for this shard. + */ + public QueryCacheStats getStats(ShardId shard, long precomputedSharedRamBytesUsed) { + final QueryCacheStats queryCacheStats = toQueryCacheStatsSafe(shardStats.get(shard)); + queryCacheStats.addRamBytesUsed(precomputedSharedRamBytesUsed); + return queryCacheStats; + } + + /** + * Precompute the shared RAM split for all shards, returning a map of ShardId to the additional shared RAM bytes used. + * This avoids O(N^2) when collecting stats for all shards. + */ + public Map computeAllShardSharedRamBytesUsed() { + Map result = new HashMap<>(); + if (sharedRamBytesUsed == 0L) { + for (ShardId shardId : shardStats.keySet()) { + result.put(shardId, 0L); + } + return result; + } + long totalSize = 0L; + int shardCount = 0; + boolean anyNonZero = false; + for (Stats stats : shardStats.values()) { + shardCount += 1; + if (stats.cacheSize > 0L) { + anyNonZero = true; + totalSize += stats.cacheSize; + } + } + if (shardCount == 0) { + return result; + } + if (anyNonZero == false) { + // All shards have zero cache footprint, apportion equally + long perShard = Math.round((double) sharedRamBytesUsed / shardCount); + for (ShardId shardId : shardStats.keySet()) { + result.put(shardId, perShard); + } + } else { + // Apportion proportionally to cache footprint + for (Map.Entry entry : shardStats.entrySet()) { + long cacheSize = entry.getValue().cacheSize; + long ram = (totalSize == 0) ? 0L : Math.round((double) sharedRamBytesUsed * cacheSize / totalSize); + result.put(entry.getKey(), ram); + } + } + return result; + } + @Override public Weight doCache(Weight weight, QueryCachingPolicy policy) { while (weight instanceof CachingWeightWrapper) { diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 8fdc53e6b795f..f7041c9f6df7d 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -167,6 +167,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; import java.util.Iterator; @@ -519,33 +520,81 @@ static Map statsByIndex(final IndicesService indicesService, } static Map> statsByShard(final IndicesService indicesService, final CommonStatsFlags flags) { - final Map> statsByShard = new HashMap<>(); - + IndicesQueryCache queryCache = indicesService.getIndicesQueryCache(); + // First pass: gather all shards, cache sizes, and compute totals + class ShardCacheInfo { + final IndexService indexService; + final IndexShard indexShard; + final org.elasticsearch.index.shard.ShardId shardId; + final long cacheSize; + + ShardCacheInfo(IndexService is, IndexShard shard, org.elasticsearch.index.shard.ShardId id, long size) { + this.indexService = is; + this.indexShard = shard; + this.shardId = id; + this.cacheSize = size; + } + } + List shardInfos = new ArrayList<>(); + long totalSize = 0L; + int shardCount = 0; + boolean anyNonZero = false; for (final IndexService indexService : indicesService) { for (final IndexShard indexShard : indexService) { - try { - final IndexShardStats indexShardStats = indicesService.indexShardStats(indicesService, indexShard, flags); - - if (indexShardStats == null) { - continue; - } - - if (statsByShard.containsKey(indexService.index()) == false) { - statsByShard.put(indexService.index(), arrayAsArrayList(indexShardStats)); - } else { - statsByShard.get(indexService.index()).add(indexShardStats); - } - } catch (IllegalIndexShardStateException | AlreadyClosedException e) { - // we can safely ignore illegal state on ones that are closing for example - logger.trace(() -> format("%s ignoring shard stats", indexShard.shardId()), e); + org.elasticsearch.index.shard.ShardId shardId = indexShard.shardId(); + long cacheSize = queryCache.getCacheSizeForShard(shardId); + shardInfos.add(new ShardCacheInfo(indexService, indexShard, shardId, cacheSize)); + shardCount++; + if (cacheSize > 0L) { + anyNonZero = true; + totalSize += cacheSize; } } } - + long sharedRamBytesUsed = queryCache.getSharedRamBytesUsed(); + final Map> statsByShard = new HashMap<>(); + // Second pass: build stats, compute shared RAM on the fly + for (ShardCacheInfo info : shardInfos) { + long sharedRam = 0L; + if (sharedRamBytesUsed != 0L) { + if (anyNonZero == false) { + sharedRam = Math.round((double) sharedRamBytesUsed / shardCount); + } else if (totalSize != 0) { + sharedRam = Math.round((double) sharedRamBytesUsed * info.cacheSize / totalSize); + } + } + try { + final IndexShardStats indexShardStats = indicesService.indexShardStats( + indicesService, + info.indexShard, + flags, + Collections.singletonMap(info.shardId, sharedRam) + ); + if (indexShardStats == null) { + continue; + } + if (statsByShard.containsKey(info.indexService.index()) == false) { + statsByShard.put(info.indexService.index(), arrayAsArrayList(indexShardStats)); + } else { + statsByShard.get(info.indexService.index()).add(indexShardStats); + } + } catch (IllegalIndexShardStateException | AlreadyClosedException e) { + logger.trace(() -> format("%s ignoring shard stats", info.shardId), e); + } + } return statsByShard; } IndexShardStats indexShardStats(final IndicesService indicesService, final IndexShard indexShard, final CommonStatsFlags flags) { + return indexShardStats(indicesService, indexShard, flags, null); + } + + IndexShardStats indexShardStats( + final IndicesService indicesService, + final IndexShard indexShard, + final CommonStatsFlags flags, + Map precomputedSharedRam + ) { if (indexShard.routingEntry() == null) { return null; } @@ -570,7 +619,7 @@ IndexShardStats indexShardStats(final IndicesService indicesService, final Index new ShardStats( indexShard.routingEntry(), indexShard.shardPath(), - CommonStats.getShardLevelStats(indicesService.getIndicesQueryCache(), indexShard, flags), + CommonStats.getShardLevelStats(indicesService.getIndicesQueryCache(), indexShard, flags, precomputedSharedRam), commitStats, seqNoStats, retentionLeaseStats, From 4a42e34ec0f1d816d497ca37af62f1b1f48600b4 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 8 Jul 2025 16:11:20 -0500 Subject: [PATCH 02/34] Update docs/changelog/130857.yaml --- docs/changelog/130857.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/130857.yaml diff --git a/docs/changelog/130857.yaml b/docs/changelog/130857.yaml new file mode 100644 index 0000000000000..6e42d882070f7 --- /dev/null +++ b/docs/changelog/130857.yaml @@ -0,0 +1,5 @@ +pr: 130857 +summary: First attempt at improving `statsByShard` performance +area: Stats +type: bug +issues: [] From f381bb920e0f53a98c06454ef9a02d4ccc8ed0bf Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 8 Jul 2025 16:50:11 -0500 Subject: [PATCH 03/34] reducing memory usage --- .../elasticsearch/indices/IndicesService.java | 75 ++++++++----------- 1 file changed, 32 insertions(+), 43 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index f7041c9f6df7d..adbb92331cd5a 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -167,7 +167,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; import java.util.Iterator; @@ -521,29 +520,15 @@ static Map statsByIndex(final IndicesService indicesService, static Map> statsByShard(final IndicesService indicesService, final CommonStatsFlags flags) { IndicesQueryCache queryCache = indicesService.getIndicesQueryCache(); + boolean hasQueryCache = queryCache != null; // First pass: gather all shards, cache sizes, and compute totals - class ShardCacheInfo { - final IndexService indexService; - final IndexShard indexShard; - final org.elasticsearch.index.shard.ShardId shardId; - final long cacheSize; - - ShardCacheInfo(IndexService is, IndexShard shard, org.elasticsearch.index.shard.ShardId id, long size) { - this.indexService = is; - this.indexShard = shard; - this.shardId = id; - this.cacheSize = size; - } - } - List shardInfos = new ArrayList<>(); long totalSize = 0L; int shardCount = 0; boolean anyNonZero = false; + // First pass: compute totals only for (final IndexService indexService : indicesService) { for (final IndexShard indexShard : indexService) { - org.elasticsearch.index.shard.ShardId shardId = indexShard.shardId(); - long cacheSize = queryCache.getCacheSizeForShard(shardId); - shardInfos.add(new ShardCacheInfo(indexService, indexShard, shardId, cacheSize)); + long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(indexShard.shardId()) : 0L; shardCount++; if (cacheSize > 0L) { anyNonZero = true; @@ -551,35 +536,39 @@ class ShardCacheInfo { } } } - long sharedRamBytesUsed = queryCache.getSharedRamBytesUsed(); + long sharedRamBytesUsed = hasQueryCache ? queryCache.getSharedRamBytesUsed() : 0L; final Map> statsByShard = new HashMap<>(); // Second pass: build stats, compute shared RAM on the fly - for (ShardCacheInfo info : shardInfos) { - long sharedRam = 0L; - if (sharedRamBytesUsed != 0L) { - if (anyNonZero == false) { - sharedRam = Math.round((double) sharedRamBytesUsed / shardCount); - } else if (totalSize != 0) { - sharedRam = Math.round((double) sharedRamBytesUsed * info.cacheSize / totalSize); - } - } - try { - final IndexShardStats indexShardStats = indicesService.indexShardStats( - indicesService, - info.indexShard, - flags, - Collections.singletonMap(info.shardId, sharedRam) - ); - if (indexShardStats == null) { - continue; + for (final IndexService indexService : indicesService) { + for (final IndexShard indexShard : indexService) { + org.elasticsearch.index.shard.ShardId shardId = indexShard.shardId(); + long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(shardId) : 0L; + long sharedRam = 0L; + if (sharedRamBytesUsed != 0L) { + if (anyNonZero == false) { + sharedRam = Math.round((double) sharedRamBytesUsed / shardCount); + } else if (totalSize != 0) { + sharedRam = Math.round((double) sharedRamBytesUsed * cacheSize / totalSize); + } } - if (statsByShard.containsKey(info.indexService.index()) == false) { - statsByShard.put(info.indexService.index(), arrayAsArrayList(indexShardStats)); - } else { - statsByShard.get(info.indexService.index()).add(indexShardStats); + try { + final IndexShardStats indexShardStats = indicesService.indexShardStats( + indicesService, + indexShard, + flags, + java.util.Collections.singletonMap(shardId, sharedRam) + ); + if (indexShardStats == null) { + continue; + } + if (statsByShard.containsKey(indexService.index()) == false) { + statsByShard.put(indexService.index(), arrayAsArrayList(indexShardStats)); + } else { + statsByShard.get(indexService.index()).add(indexShardStats); + } + } catch (IllegalIndexShardStateException | AlreadyClosedException e) { + logger.trace(() -> format("%s ignoring shard stats", shardId), e); } - } catch (IllegalIndexShardStateException | AlreadyClosedException e) { - logger.trace(() -> format("%s ignoring shard stats", info.shardId), e); } } return statsByShard; From 41bdbeafcdb033786cfee344d8f56aacf64ac6ba Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 8 Jul 2025 16:59:57 -0500 Subject: [PATCH 04/34] removing unused method --- .../indices/IndicesQueryCache.java | 42 ------------------- 1 file changed, 42 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index 2a9b9982d3acc..1e2b06a82403d 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -160,48 +160,6 @@ public QueryCacheStats getStats(ShardId shard, long precomputedSharedRamBytesUse return queryCacheStats; } - /** - * Precompute the shared RAM split for all shards, returning a map of ShardId to the additional shared RAM bytes used. - * This avoids O(N^2) when collecting stats for all shards. - */ - public Map computeAllShardSharedRamBytesUsed() { - Map result = new HashMap<>(); - if (sharedRamBytesUsed == 0L) { - for (ShardId shardId : shardStats.keySet()) { - result.put(shardId, 0L); - } - return result; - } - long totalSize = 0L; - int shardCount = 0; - boolean anyNonZero = false; - for (Stats stats : shardStats.values()) { - shardCount += 1; - if (stats.cacheSize > 0L) { - anyNonZero = true; - totalSize += stats.cacheSize; - } - } - if (shardCount == 0) { - return result; - } - if (anyNonZero == false) { - // All shards have zero cache footprint, apportion equally - long perShard = Math.round((double) sharedRamBytesUsed / shardCount); - for (ShardId shardId : shardStats.keySet()) { - result.put(shardId, perShard); - } - } else { - // Apportion proportionally to cache footprint - for (Map.Entry entry : shardStats.entrySet()) { - long cacheSize = entry.getValue().cacheSize; - long ram = (totalSize == 0) ? 0L : Math.round((double) sharedRamBytesUsed * cacheSize / totalSize); - result.put(entry.getKey(), ram); - } - } - return result; - } - @Override public Weight doCache(Weight weight, QueryCachingPolicy policy) { while (weight instanceof CachingWeightWrapper) { From 212186a4d338a36254262cbbdddfe7c6b284d9c9 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 8 Jul 2025 17:06:39 -0500 Subject: [PATCH 05/34] minor cleanup --- .../action/admin/indices/stats/CommonStats.java | 9 +++------ .../org/elasticsearch/indices/IndicesQueryCache.java | 1 - .../java/org/elasticsearch/indices/IndicesService.java | 9 ++------- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java index 024d6af9cda43..0193ba4a9a022 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java @@ -165,7 +165,7 @@ public static CommonStats getShardLevelStats( IndicesQueryCache indicesQueryCache, IndexShard indexShard, CommonStatsFlags flags, - java.util.Map precomputedSharedRam + Long precomputedSharedRam ) { // Filter shard level flags CommonStatsFlags filteredFlags = flags.clone(); @@ -187,11 +187,8 @@ public static CommonStats getShardLevelStats( case Flush -> stats.flush = indexShard.flushStats(); case Warmer -> stats.warmer = indexShard.warmerStats(); case QueryCache -> { - if (precomputedSharedRam != null && precomputedSharedRam.containsKey(indexShard.shardId())) { - stats.queryCache = indicesQueryCache.getStats( - indexShard.shardId(), - precomputedSharedRam.get(indexShard.shardId()) - ); + if (precomputedSharedRam != null) { + stats.queryCache = indicesQueryCache.getStats(indexShard.shardId(), precomputedSharedRam); } else { stats.queryCache = indicesQueryCache.getStats(indexShard.shardId()); } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index 1e2b06a82403d..f050d2f15a8f3 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -32,7 +32,6 @@ import java.io.Closeable; import java.io.IOException; import java.util.Collections; -import java.util.HashMap; import java.util.IdentityHashMap; import java.util.Map; import java.util.Set; diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index adbb92331cd5a..c0461301fe881 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -552,12 +552,7 @@ static Map> statsByShard(final IndicesService indic } } try { - final IndexShardStats indexShardStats = indicesService.indexShardStats( - indicesService, - indexShard, - flags, - java.util.Collections.singletonMap(shardId, sharedRam) - ); + final IndexShardStats indexShardStats = indicesService.indexShardStats(indicesService, indexShard, flags, sharedRam); if (indexShardStats == null) { continue; } @@ -582,7 +577,7 @@ IndexShardStats indexShardStats( final IndicesService indicesService, final IndexShard indexShard, final CommonStatsFlags flags, - Map precomputedSharedRam + Long precomputedSharedRam ) { if (indexShard.routingEntry() == null) { return null; From 16971dfcd75227dd6651ede9e29d903ea5b6b4ba Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 9 Jul 2025 13:44:48 -0500 Subject: [PATCH 06/34] minor cleanup --- .../java/org/elasticsearch/indices/IndicesService.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index c0461301fe881..580cfa7437696 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -541,7 +541,7 @@ static Map> statsByShard(final IndicesService indic // Second pass: build stats, compute shared RAM on the fly for (final IndexService indexService : indicesService) { for (final IndexShard indexShard : indexService) { - org.elasticsearch.index.shard.ShardId shardId = indexShard.shardId(); + ShardId shardId = indexShard.shardId(); long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(shardId) : 0L; long sharedRam = 0L; if (sharedRamBytesUsed != 0L) { @@ -552,7 +552,7 @@ static Map> statsByShard(final IndicesService indic } } try { - final IndexShardStats indexShardStats = indicesService.indexShardStats(indicesService, indexShard, flags, sharedRam); + final IndexShardStats indexShardStats = indicesService.indexShardStats(indicesService, indexShard, flags); if (indexShardStats == null) { continue; } @@ -570,7 +570,8 @@ static Map> statsByShard(final IndicesService indic } IndexShardStats indexShardStats(final IndicesService indicesService, final IndexShard indexShard, final CommonStatsFlags flags) { - return indexShardStats(indicesService, indexShard, flags, null); + // Default to 0L for precomputedSharedRam for backward compatibility + return indexShardStats(indicesService, indexShard, flags, 0L); } IndexShardStats indexShardStats( From c8fc29e5d1cf732a58f98d8484f1913a2dafdfa1 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Thu, 10 Jul 2025 14:10:05 -0500 Subject: [PATCH 07/34] fixing IndicesServiceTests --- .../java/org/elasticsearch/indices/IndicesServiceTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java index 7f2aa62f7cf83..2717e45d3475a 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java @@ -619,8 +619,8 @@ public void testStatsByShardDoesNotDieFromExpectedExceptions() { } } - when(mockIndicesService.iterator()).thenReturn(Collections.singleton(indexService).iterator()); - when(indexService.iterator()).thenReturn(shards.iterator()); + when(mockIndicesService.iterator()).thenAnswer(invocation -> Collections.singleton(indexService).iterator()); + when(indexService.iterator()).thenAnswer(unused -> shards.iterator()); when(indexService.index()).thenReturn(index); // real one, which has a logger defined From 4455f70773aa4f71cdce9d88e8d12ffacedb8807 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 16 Jul 2025 15:29:33 -0500 Subject: [PATCH 08/34] removing all uses of N^2 code --- .../stats/TransportClusterStatsAction.java | 34 ++++++++++- .../admin/indices/stats/CommonStats.java | 17 +----- .../stats/TransportIndicesStatsAction.java | 40 ++++++++++++- .../indices/IndicesQueryCache.java | 56 +------------------ .../elasticsearch/indices/IndicesService.java | 7 +-- .../cluster/stats/VersionStatsTests.java | 2 +- .../index/shard/IndexShardTests.java | 2 +- .../indices/IndicesQueryCacheTests.java | 38 ++++++------- .../indices/IndicesServiceCloseTests.java | 14 ++--- .../indices/IndicesServiceTests.java | 9 ++- 10 files changed, 112 insertions(+), 107 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 05a916a076b95..4c2e4da7fa8bc 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -48,6 +48,8 @@ import org.elasticsearch.index.seqno.RetentionLeaseStats; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndicesQueryCache; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.node.NodeService; @@ -257,9 +259,39 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq false, false ); + + IndicesQueryCache queryCache = indicesService.getIndicesQueryCache(); + boolean hasQueryCache = queryCache != null; + // First pass: gather all shards, cache sizes, and compute totals + long totalSize = 0L; + int shardCount = 0; + boolean anyNonZero = false; + // First pass: compute totals only + for (final IndexService indexService : indicesService) { + for (final IndexShard indexShard : indexService) { + long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(indexShard.shardId()) : 0L; + shardCount++; + if (cacheSize > 0L) { + anyNonZero = true; + totalSize += cacheSize; + } + } + } + long sharedRamBytesUsed = hasQueryCache ? queryCache.getSharedRamBytesUsed() : 0L; + List shardsStats = new ArrayList<>(); for (IndexService indexService : indicesService) { for (IndexShard indexShard : indexService) { + ShardId shardId = indexShard.shardId(); + long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(shardId) : 0L; + long sharedRam = 0L; + if (sharedRamBytesUsed != 0L) { + if (anyNonZero == false) { + sharedRam = Math.round((double) sharedRamBytesUsed / shardCount); + } else if (totalSize != 0) { + sharedRam = Math.round((double) sharedRamBytesUsed * cacheSize / totalSize); + } + } cancellableTask.ensureNotCancelled(); if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) { // only report on fully started shards @@ -280,7 +312,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq new ShardStats( indexShard.routingEntry(), indexShard.shardPath(), - CommonStats.getShardLevelStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS), + CommonStats.getShardLevelStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS, sharedRam), commitStats, seqNoStats, retentionLeaseStats, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java index 0193ba4a9a022..d74ef67f97bd9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java @@ -154,18 +154,11 @@ public CommonStats(CommonStatsFlags flags) { /** * Filters the given flags for {@link CommonStatsFlags#SHARD_LEVEL} flags and calculates the corresponding statistics. */ - public static CommonStats getShardLevelStats(IndicesQueryCache indicesQueryCache, IndexShard indexShard, CommonStatsFlags flags) { - return getShardLevelStats(indicesQueryCache, indexShard, flags, null); - } - - /** - * Overload that takes a precomputed shared RAM map for O(N) stats. - */ public static CommonStats getShardLevelStats( IndicesQueryCache indicesQueryCache, IndexShard indexShard, CommonStatsFlags flags, - Long precomputedSharedRam + long precomputedSharedRam ) { // Filter shard level flags CommonStatsFlags filteredFlags = flags.clone(); @@ -186,13 +179,7 @@ public static CommonStats getShardLevelStats( case Refresh -> stats.refresh = indexShard.refreshStats(); case Flush -> stats.flush = indexShard.flushStats(); case Warmer -> stats.warmer = indexShard.warmerStats(); - case QueryCache -> { - if (precomputedSharedRam != null) { - stats.queryCache = indicesQueryCache.getStats(indexShard.shardId(), precomputedSharedRam); - } else { - stats.queryCache = indicesQueryCache.getStats(indexShard.shardId()); - } - } + case QueryCache -> stats.queryCache = indicesQueryCache.getStats(indexShard.shardId(), 0L); case FieldData -> stats.fieldData = indexShard.fieldDataStats(flags.fieldDataFields()); case Completion -> stats.completion = indexShard.completionStats(flags.completionDataFields()); case Segments -> stats.segments = indexShard.segmentStats( diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java index 210fb42ca584b..ee07209f3c319 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java @@ -27,6 +27,8 @@ import org.elasticsearch.index.seqno.RetentionLeaseStats; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndicesQueryCache; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.CancellableTask; @@ -112,9 +114,45 @@ protected IndicesStatsRequest readRequestFrom(StreamInput in) throws IOException protected void shardOperation(IndicesStatsRequest request, ShardRouting shardRouting, Task task, ActionListener listener) { ActionListener.completeWith(listener, () -> { assert task instanceof CancellableTask; + IndicesQueryCache queryCache = indicesService.getIndicesQueryCache(); + boolean hasQueryCache = queryCache != null; + + // First pass: gather all shards, cache sizes, and compute totals + long totalSize = 0L; + int shardCount = 0; + boolean anyNonZero = false; + // First pass: compute totals only + for (final IndexService indexService : indicesService) { + for (final IndexShard indexShard : indexService) { + long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(indexShard.shardId()) : 0L; + shardCount++; + if (cacheSize > 0L) { + anyNonZero = true; + totalSize += cacheSize; + } + } + } + long sharedRamBytesUsed = hasQueryCache ? queryCache.getSharedRamBytesUsed() : 0L; + IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex()); IndexShard indexShard = indexService.getShard(shardRouting.shardId().id()); - CommonStats commonStats = CommonStats.getShardLevelStats(indicesService.getIndicesQueryCache(), indexShard, request.flags()); + ShardId shardId = indexShard.shardId(); + long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(shardId) : 0L; + long sharedRam = 0L; + if (sharedRamBytesUsed != 0L) { + if (anyNonZero == false) { + sharedRam = Math.round((double) sharedRamBytesUsed / shardCount); + } else if (totalSize != 0) { + sharedRam = Math.round((double) sharedRamBytesUsed * cacheSize / totalSize); + } + } + + CommonStats commonStats = CommonStats.getShardLevelStats( + indicesService.getIndicesQueryCache(), + indexShard, + request.flags(), + sharedRam + ); CommitStats commitStats; SeqNoStats seqNoStats; RetentionLeaseStats retentionLeaseStats; diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index f050d2f15a8f3..ec39115bf0abc 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -68,13 +68,13 @@ public class IndicesQueryCache implements QueryCache, Closeable { private volatile long sharedRamBytesUsed; // Package-private for IndicesService efficient stats collection - long getCacheSizeForShard(ShardId shardId) { + public long getCacheSizeForShard(ShardId shardId) { Stats stats = shardStats.get(shardId); return stats != null ? stats.cacheSize : 0L; } // Package-private for IndicesService efficient stats collection - long getSharedRamBytesUsed() { + public long getSharedRamBytesUsed() { return sharedRamBytesUsed; } @@ -100,59 +100,7 @@ private static QueryCacheStats toQueryCacheStatsSafe(@Nullable Stats stats) { return stats == null ? new QueryCacheStats() : stats.toQueryCacheStats(); } - private long getShareOfAdditionalRamBytesUsed(long cacheSize) { - if (sharedRamBytesUsed == 0L) { - return 0L; - } - - // We also have some shared ram usage that we try to distribute proportionally to the cache footprint of each shard. - // TODO avoid looping over all local shards here - see https://github.com/elastic/elasticsearch/issues/97222 - long totalSize = 0L; - int shardCount = 0; - if (cacheSize == 0L) { - for (final var stats : shardStats.values()) { - shardCount += 1; - if (stats.cacheSize > 0L) { - // some shard has nonzero cache footprint, so we apportion the shared size by cache footprint, and this shard has none - return 0L; - } - } - } else { - // branchless loop for the common case - for (final var stats : shardStats.values()) { - shardCount += 1; - totalSize += stats.cacheSize; - } - } - - if (shardCount == 0) { - // Sometimes it's not possible to do this when there are no shard entries at all, which can happen as the shared ram usage can - // extend beyond the closing of all shards. - return 0L; - } - - final long additionalRamBytesUsed; - if (totalSize == 0) { - // all shards have zero cache footprint, so we apportion the size of the shared bytes equally across all shards - additionalRamBytesUsed = Math.round((double) sharedRamBytesUsed / shardCount); - } else { - // some shards have nonzero cache footprint, so we apportion the size of the shared bytes proportionally to cache footprint - additionalRamBytesUsed = Math.round((double) sharedRamBytesUsed * cacheSize / totalSize); - } - assert additionalRamBytesUsed >= 0L : additionalRamBytesUsed; - return additionalRamBytesUsed; - } - /** Get usage statistics for the given shard. */ - public QueryCacheStats getStats(ShardId shard) { - final QueryCacheStats queryCacheStats = toQueryCacheStatsSafe(shardStats.get(shard)); - queryCacheStats.addRamBytesUsed(getShareOfAdditionalRamBytesUsed(queryCacheStats.getCacheSize())); - return queryCacheStats; - } - - /** - * Overload to allow passing in a precomputed shared RAM split for this shard. - */ public QueryCacheStats getStats(ShardId shard, long precomputedSharedRamBytesUsed) { final QueryCacheStats queryCacheStats = toQueryCacheStatsSafe(shardStats.get(shard)); queryCacheStats.addRamBytesUsed(precomputedSharedRamBytesUsed); diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 580cfa7437696..a25771ecb2187 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -552,7 +552,7 @@ static Map> statsByShard(final IndicesService indic } } try { - final IndexShardStats indexShardStats = indicesService.indexShardStats(indicesService, indexShard, flags); + final IndexShardStats indexShardStats = indicesService.indexShardStats(indicesService, indexShard, flags, sharedRam); if (indexShardStats == null) { continue; } @@ -569,11 +569,6 @@ static Map> statsByShard(final IndicesService indic return statsByShard; } - IndexShardStats indexShardStats(final IndicesService indicesService, final IndexShard indexShard, final CommonStatsFlags flags) { - // Default to 0L for precomputedSharedRam for backward compatibility - return indexShardStats(indicesService, indexShard, flags, 0L); - } - IndexShardStats indexShardStats( final IndicesService indicesService, final IndexShard indexShard, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java index fbd6e0916eefe..2ab6c89ad265d 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java @@ -115,7 +115,7 @@ public void testCreation() { ShardStats shardStats = new ShardStats( shardRouting, new ShardPath(false, path, path, shardRouting.shardId()), - CommonStats.getShardLevelStats(null, indexShard, new CommonStatsFlags(CommonStatsFlags.Flag.Store)), + CommonStats.getShardLevelStats(null, indexShard, new CommonStatsFlags(CommonStatsFlags.Flag.Store), 0L), null, null, null, diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index cc682901876b6..b2d9ec9b18912 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -1624,7 +1624,7 @@ public void testShardStats() throws IOException { ShardStats stats = new ShardStats( shard.routingEntry(), shard.shardPath(), - CommonStats.getShardLevelStats(new IndicesQueryCache(Settings.EMPTY), shard, new CommonStatsFlags()), + CommonStats.getShardLevelStats(new IndicesQueryCache(Settings.EMPTY), shard, new CommonStatsFlags(), 0L), shard.commitStats(), shard.seqNoStats(), shard.getRetentionLeaseStats(), diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java index 4f73672471942..f8488fedba32f 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java @@ -102,7 +102,7 @@ public void testBasics() throws IOException { IndicesQueryCache cache = new IndicesQueryCache(settings); s.setQueryCache(cache); - QueryCacheStats stats = cache.getStats(shard); + QueryCacheStats stats = cache.getStats(shard, 0L); assertEquals(0L, stats.getCacheSize()); assertEquals(0L, stats.getCacheCount()); assertEquals(0L, stats.getHitCount()); @@ -110,7 +110,7 @@ public void testBasics() throws IOException { assertEquals(1, s.count(new DummyQuery(0))); - stats = cache.getStats(shard); + stats = cache.getStats(shard, 0L); assertEquals(1L, stats.getCacheSize()); assertEquals(1L, stats.getCacheCount()); assertEquals(0L, stats.getHitCount()); @@ -120,7 +120,7 @@ public void testBasics() throws IOException { assertEquals(1, s.count(new DummyQuery(i))); } - stats = cache.getStats(shard); + stats = cache.getStats(shard, 0L); assertEquals(10L, stats.getCacheSize()); assertEquals(20L, stats.getCacheCount()); assertEquals(0L, stats.getHitCount()); @@ -128,7 +128,7 @@ public void testBasics() throws IOException { s.count(new DummyQuery(10)); - stats = cache.getStats(shard); + stats = cache.getStats(shard, 0L); assertEquals(10L, stats.getCacheSize()); assertEquals(20L, stats.getCacheCount()); assertEquals(1L, stats.getHitCount()); @@ -137,7 +137,7 @@ public void testBasics() throws IOException { IOUtils.close(r, dir); // got emptied, but no changes to other metrics - stats = cache.getStats(shard); + stats = cache.getStats(shard, 0L); assertEquals(0L, stats.getCacheSize()); assertEquals(20L, stats.getCacheCount()); assertEquals(1L, stats.getHitCount()); @@ -146,7 +146,7 @@ public void testBasics() throws IOException { cache.onClose(shard); // forgot everything - stats = cache.getStats(shard); + stats = cache.getStats(shard, 0L); assertEquals(0L, stats.getCacheSize()); assertEquals(0L, stats.getCacheCount()); assertEquals(0L, stats.getHitCount()); @@ -186,13 +186,13 @@ public void testTwoShards() throws IOException { assertEquals(1, s1.count(new DummyQuery(0))); - QueryCacheStats stats1 = cache.getStats(shard1); + QueryCacheStats stats1 = cache.getStats(shard1, 0L); assertEquals(1L, stats1.getCacheSize()); assertEquals(1L, stats1.getCacheCount()); assertEquals(0L, stats1.getHitCount()); assertEquals(2L, stats1.getMissCount()); - QueryCacheStats stats2 = cache.getStats(shard2); + QueryCacheStats stats2 = cache.getStats(shard2, 0L); assertEquals(0L, stats2.getCacheSize()); assertEquals(0L, stats2.getCacheCount()); assertEquals(0L, stats2.getHitCount()); @@ -200,13 +200,13 @@ public void testTwoShards() throws IOException { assertEquals(1, s2.count(new DummyQuery(0))); - stats1 = cache.getStats(shard1); + stats1 = cache.getStats(shard1, 0L); assertEquals(1L, stats1.getCacheSize()); assertEquals(1L, stats1.getCacheCount()); assertEquals(0L, stats1.getHitCount()); assertEquals(2L, stats1.getMissCount()); - stats2 = cache.getStats(shard2); + stats2 = cache.getStats(shard2, 0L); assertEquals(1L, stats2.getCacheSize()); assertEquals(1L, stats2.getCacheCount()); assertEquals(0L, stats2.getHitCount()); @@ -216,13 +216,13 @@ public void testTwoShards() throws IOException { assertEquals(1, s2.count(new DummyQuery(i))); } - stats1 = cache.getStats(shard1); + stats1 = cache.getStats(shard1, 0L); assertEquals(0L, stats1.getCacheSize()); // evicted assertEquals(1L, stats1.getCacheCount()); assertEquals(0L, stats1.getHitCount()); assertEquals(2L, stats1.getMissCount()); - stats2 = cache.getStats(shard2); + stats2 = cache.getStats(shard2, 0L); assertEquals(10L, stats2.getCacheSize()); assertEquals(20L, stats2.getCacheCount()); assertEquals(1L, stats2.getHitCount()); @@ -231,13 +231,13 @@ public void testTwoShards() throws IOException { IOUtils.close(r1, dir1); // no changes - stats1 = cache.getStats(shard1); + stats1 = cache.getStats(shard1, 0L); assertEquals(0L, stats1.getCacheSize()); assertEquals(1L, stats1.getCacheCount()); assertEquals(0L, stats1.getHitCount()); assertEquals(2L, stats1.getMissCount()); - stats2 = cache.getStats(shard2); + stats2 = cache.getStats(shard2, 0L); assertEquals(10L, stats2.getCacheSize()); assertEquals(20L, stats2.getCacheCount()); assertEquals(1L, stats2.getHitCount()); @@ -246,13 +246,13 @@ public void testTwoShards() throws IOException { cache.onClose(shard1); // forgot everything about shard1 - stats1 = cache.getStats(shard1); + stats1 = cache.getStats(shard1, 0L); assertEquals(0L, stats1.getCacheSize()); assertEquals(0L, stats1.getCacheCount()); assertEquals(0L, stats1.getHitCount()); assertEquals(0L, stats1.getMissCount()); - stats2 = cache.getStats(shard2); + stats2 = cache.getStats(shard2, 0L); assertEquals(10L, stats2.getCacheSize()); assertEquals(20L, stats2.getCacheCount()); assertEquals(1L, stats2.getHitCount()); @@ -262,14 +262,14 @@ public void testTwoShards() throws IOException { cache.onClose(shard2); // forgot everything about shard2 - stats1 = cache.getStats(shard1); + stats1 = cache.getStats(shard1, 0L); assertEquals(0L, stats1.getCacheSize()); assertEquals(0L, stats1.getCacheCount()); assertEquals(0L, stats1.getHitCount()); assertEquals(0L, stats1.getMissCount()); assertEquals(0L, stats1.getMemorySizeInBytes()); - stats2 = cache.getStats(shard2); + stats2 = cache.getStats(shard2, 0L); assertEquals(0L, stats2.getCacheSize()); assertEquals(0L, stats2.getCacheCount()); assertEquals(0L, stats2.getHitCount()); @@ -318,7 +318,7 @@ public void testStatsOnEviction() throws IOException { assertEquals(1, s2.count(new DummyQuery(i))); } - QueryCacheStats stats1 = cache.getStats(shard1); + QueryCacheStats stats1 = cache.getStats(shard1, 0L); assertEquals(0L, stats1.getCacheSize()); assertEquals(1L, stats1.getCacheCount()); diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java index a48ba63882734..00ecfc3305b7f 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java @@ -179,17 +179,17 @@ public void testCloseAfterRequestHasUsedQueryCache() throws Exception { assertEquals(1, searcher.getIndexReader().maxDoc()); Query query = LongPoint.newRangeQuery("foo", 0, 5); - assertEquals(0L, cache.getStats(shard.shardId()).getCacheSize()); + assertEquals(0L, cache.getStats(shard.shardId(), 0L).getCacheSize()); searcher.search(new ConstantScoreQuery(query), 1); - assertEquals(1L, cache.getStats(shard.shardId()).getCacheSize()); + assertEquals(1L, cache.getStats(shard.shardId(), 0L).getCacheSize()); searcher.close(); assertEquals(2, indicesService.indicesRefCount.refCount()); - assertEquals(1L, cache.getStats(shard.shardId()).getCacheSize()); + assertEquals(1L, cache.getStats(shard.shardId(), 0L).getCacheSize()); node.close(); assertEquals(0, indicesService.indicesRefCount.refCount()); - assertEquals(0L, cache.getStats(shard.shardId()).getCacheSize()); + assertEquals(0L, cache.getStats(shard.shardId(), 0L).getCacheSize()); } public void testCloseWhileOngoingRequestUsesQueryCache() throws Exception { @@ -220,13 +220,13 @@ public void testCloseWhileOngoingRequestUsesQueryCache() throws Exception { assertEquals(1, indicesService.indicesRefCount.refCount()); Query query = LongPoint.newRangeQuery("foo", 0, 5); - assertEquals(0L, cache.getStats(shard.shardId()).getCacheSize()); + assertEquals(0L, cache.getStats(shard.shardId(), 0L).getCacheSize()); searcher.search(new ConstantScoreQuery(query), 1); - assertEquals(1L, cache.getStats(shard.shardId()).getCacheSize()); + assertEquals(1L, cache.getStats(shard.shardId(), 0L).getCacheSize()); searcher.close(); assertEquals(0, indicesService.indicesRefCount.refCount()); - assertEquals(0L, cache.getStats(shard.shardId()).getCacheSize()); + assertEquals(0L, cache.getStats(shard.shardId(), 0L).getCacheSize()); } public void testCloseWhileOngoingRequestUsesRequestCache() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java index 2717e45d3475a..5661e86285523 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java @@ -105,6 +105,7 @@ import static org.hamcrest.Matchers.matchesRegex; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -613,9 +614,13 @@ public void testStatsByShardDoesNotDieFromExpectedExceptions() { shardStats.add(successfulShardStats); - when(mockIndicesService.indexShardStats(mockIndicesService, shard, CommonStatsFlags.ALL)).thenReturn(successfulShardStats); + when(mockIndicesService.indexShardStats(mockIndicesService, shard, CommonStatsFlags.ALL, any())).thenReturn( + successfulShardStats + ); } else { - when(mockIndicesService.indexShardStats(mockIndicesService, shard, CommonStatsFlags.ALL)).thenThrow(expectedException); + when(mockIndicesService.indexShardStats(mockIndicesService, shard, CommonStatsFlags.ALL, any())).thenThrow( + expectedException + ); } } From 34659a8a99371a6180291edd17c9dcef91a12037 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 16 Jul 2025 16:57:08 -0500 Subject: [PATCH 09/34] fixing test --- .../org/elasticsearch/indices/IndicesServiceTests.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java index 5661e86285523..2a034103478b8 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java @@ -105,7 +105,6 @@ import static org.hamcrest.Matchers.matchesRegex; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -614,13 +613,11 @@ public void testStatsByShardDoesNotDieFromExpectedExceptions() { shardStats.add(successfulShardStats); - when(mockIndicesService.indexShardStats(mockIndicesService, shard, CommonStatsFlags.ALL, any())).thenReturn( + when(mockIndicesService.indexShardStats(mockIndicesService, shard, CommonStatsFlags.ALL, 0L)).thenReturn( successfulShardStats ); } else { - when(mockIndicesService.indexShardStats(mockIndicesService, shard, CommonStatsFlags.ALL, any())).thenThrow( - expectedException - ); + when(mockIndicesService.indexShardStats(mockIndicesService, shard, CommonStatsFlags.ALL, 0L)).thenThrow(expectedException); } } From 1d77b9dbe63f286d1f0264ba0d95429365ea511d Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 16 Jul 2025 17:18:08 -0500 Subject: [PATCH 10/34] moving code into reusable methods --- .../stats/TransportClusterStatsAction.java | 30 +------------ .../stats/TransportIndicesStatsAction.java | 30 ++----------- .../indices/IndicesQueryCache.java | 42 +++++++++++++++++++ .../elasticsearch/indices/IndicesService.java | 32 ++------------ 4 files changed, 50 insertions(+), 84 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 4c2e4da7fa8bc..3f7edd79cc604 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -48,7 +48,6 @@ import org.elasticsearch.index.seqno.RetentionLeaseStats; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesQueryCache; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.injection.guice.Inject; @@ -261,37 +260,12 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq ); IndicesQueryCache queryCache = indicesService.getIndicesQueryCache(); - boolean hasQueryCache = queryCache != null; - // First pass: gather all shards, cache sizes, and compute totals - long totalSize = 0L; - int shardCount = 0; - boolean anyNonZero = false; - // First pass: compute totals only - for (final IndexService indexService : indicesService) { - for (final IndexShard indexShard : indexService) { - long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(indexShard.shardId()) : 0L; - shardCount++; - if (cacheSize > 0L) { - anyNonZero = true; - totalSize += cacheSize; - } - } - } - long sharedRamBytesUsed = hasQueryCache ? queryCache.getSharedRamBytesUsed() : 0L; + IndicesQueryCache.CacheTotals cacheTotals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); List shardsStats = new ArrayList<>(); for (IndexService indexService : indicesService) { for (IndexShard indexShard : indexService) { - ShardId shardId = indexShard.shardId(); - long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(shardId) : 0L; - long sharedRam = 0L; - if (sharedRamBytesUsed != 0L) { - if (anyNonZero == false) { - sharedRam = Math.round((double) sharedRamBytesUsed / shardCount); - } else if (totalSize != 0) { - sharedRam = Math.round((double) sharedRamBytesUsed * cacheSize / totalSize); - } - } + long sharedRam = IndicesQueryCache.getSharedRamSize(queryCache, indexShard, cacheTotals); cancellableTask.ensureNotCancelled(); if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) { // only report on fully started shards diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java index ee07209f3c319..5a09a89910fc9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java @@ -115,37 +115,13 @@ protected void shardOperation(IndicesStatsRequest request, ShardRouting shardRou ActionListener.completeWith(listener, () -> { assert task instanceof CancellableTask; IndicesQueryCache queryCache = indicesService.getIndicesQueryCache(); - boolean hasQueryCache = queryCache != null; - - // First pass: gather all shards, cache sizes, and compute totals - long totalSize = 0L; - int shardCount = 0; - boolean anyNonZero = false; - // First pass: compute totals only - for (final IndexService indexService : indicesService) { - for (final IndexShard indexShard : indexService) { - long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(indexShard.shardId()) : 0L; - shardCount++; - if (cacheSize > 0L) { - anyNonZero = true; - totalSize += cacheSize; - } - } - } - long sharedRamBytesUsed = hasQueryCache ? queryCache.getSharedRamBytesUsed() : 0L; + + IndicesQueryCache.CacheTotals cacheTotals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex()); IndexShard indexShard = indexService.getShard(shardRouting.shardId().id()); ShardId shardId = indexShard.shardId(); - long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(shardId) : 0L; - long sharedRam = 0L; - if (sharedRamBytesUsed != 0L) { - if (anyNonZero == false) { - sharedRam = Math.round((double) sharedRamBytesUsed / shardCount); - } else if (totalSize != 0) { - sharedRam = Math.round((double) sharedRamBytesUsed * cacheSize / totalSize); - } - } + long sharedRam = IndicesQueryCache.getSharedRamSize(queryCache, indexShard, cacheTotals); CommonStats commonStats = CommonStats.getShardLevelStats( indicesService.getIndicesQueryCache(), diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index ec39115bf0abc..97b42de64a716 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -26,7 +26,9 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Predicates; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.cache.query.QueryCacheStats; +import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import java.io.Closeable; @@ -118,6 +120,46 @@ public Weight doCache(Weight weight, QueryCachingPolicy policy) { return new CachingWeightWrapper(in); } + public static CacheTotals getCacheTotalsForAllShards(IndicesService indicesService) { + IndicesQueryCache queryCache = indicesService.getIndicesQueryCache(); + boolean hasQueryCache = queryCache != null; + long totalSize = 0L; + int shardCount = 0; + boolean anyNonZero = false; + for (final IndexService indexService : indicesService) { + for (final IndexShard indexShard : indexService) { + long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(indexShard.shardId()) : 0L; + shardCount++; + if (cacheSize > 0L) { + anyNonZero = true; + totalSize += cacheSize; + } + } + } + long sharedRamBytesUsed = hasQueryCache ? queryCache.getSharedRamBytesUsed() : 0L; + return new CacheTotals(totalSize, shardCount, anyNonZero, sharedRamBytesUsed); + } + + public static long getSharedRamSize(IndicesQueryCache queryCache, IndexShard indexShard, CacheTotals cacheTotals) { + long sharedRamBytesUsed = cacheTotals.sharedRamBytesUsed(); + long totalSize = cacheTotals.totalSize(); + boolean anyNonZero = cacheTotals.anyNonZero(); + int shardCount = cacheTotals.shardCount(); + ShardId shardId = indexShard.shardId(); + long cacheSize = queryCache != null ? queryCache.getCacheSizeForShard(shardId) : 0L; + long sharedRam = 0L; + if (sharedRamBytesUsed != 0L) { + if (anyNonZero == false) { + sharedRam = Math.round((double) sharedRamBytesUsed / shardCount); + } else if (totalSize != 0) { + sharedRam = Math.round((double) sharedRamBytesUsed * cacheSize / totalSize); + } + } + return sharedRam; + } + + public record CacheTotals(long totalSize, int shardCount, boolean anyNonZero, long sharedRamBytesUsed) {} + private class CachingWeightWrapper extends Weight { private final Weight in; diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index a25771ecb2187..44ede7ed2c7e1 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -520,37 +520,11 @@ static Map statsByIndex(final IndicesService indicesService, static Map> statsByShard(final IndicesService indicesService, final CommonStatsFlags flags) { IndicesQueryCache queryCache = indicesService.getIndicesQueryCache(); - boolean hasQueryCache = queryCache != null; - // First pass: gather all shards, cache sizes, and compute totals - long totalSize = 0L; - int shardCount = 0; - boolean anyNonZero = false; - // First pass: compute totals only - for (final IndexService indexService : indicesService) { - for (final IndexShard indexShard : indexService) { - long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(indexShard.shardId()) : 0L; - shardCount++; - if (cacheSize > 0L) { - anyNonZero = true; - totalSize += cacheSize; - } - } - } - long sharedRamBytesUsed = hasQueryCache ? queryCache.getSharedRamBytesUsed() : 0L; + IndicesQueryCache.CacheTotals cacheTotals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); final Map> statsByShard = new HashMap<>(); - // Second pass: build stats, compute shared RAM on the fly for (final IndexService indexService : indicesService) { for (final IndexShard indexShard : indexService) { - ShardId shardId = indexShard.shardId(); - long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(shardId) : 0L; - long sharedRam = 0L; - if (sharedRamBytesUsed != 0L) { - if (anyNonZero == false) { - sharedRam = Math.round((double) sharedRamBytesUsed / shardCount); - } else if (totalSize != 0) { - sharedRam = Math.round((double) sharedRamBytesUsed * cacheSize / totalSize); - } - } + long sharedRam = IndicesQueryCache.getSharedRamSize(queryCache, indexShard, cacheTotals); try { final IndexShardStats indexShardStats = indicesService.indexShardStats(indicesService, indexShard, flags, sharedRam); if (indexShardStats == null) { @@ -562,7 +536,7 @@ static Map> statsByShard(final IndicesService indic statsByShard.get(indexService.index()).add(indexShardStats); } } catch (IllegalIndexShardStateException | AlreadyClosedException e) { - logger.trace(() -> format("%s ignoring shard stats", shardId), e); + logger.trace(() -> format("%s ignoring shard stats", indexShard.shardId()), e); } } } From 10fac5bf2158e12c3c7e07ceab098a45596ce22c Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 16 Jul 2025 17:41:27 -0500 Subject: [PATCH 11/34] cleanup --- .../stats/TransportClusterStatsAction.java | 5 +- .../stats/TransportIndicesStatsAction.java | 5 +- .../indices/IndicesQueryCache.java | 77 ++++++++++--------- .../elasticsearch/indices/IndicesService.java | 3 +- 4 files changed, 45 insertions(+), 45 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 3f7edd79cc604..87aaed11b5314 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -258,14 +258,11 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq false, false ); - - IndicesQueryCache queryCache = indicesService.getIndicesQueryCache(); IndicesQueryCache.CacheTotals cacheTotals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); - List shardsStats = new ArrayList<>(); for (IndexService indexService : indicesService) { for (IndexShard indexShard : indexService) { - long sharedRam = IndicesQueryCache.getSharedRamSize(queryCache, indexShard, cacheTotals); + long sharedRam = IndicesQueryCache.getSharedRamSizeForShard(indicesService.getIndicesQueryCache(), indexShard, cacheTotals); cancellableTask.ensureNotCancelled(); if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) { // only report on fully started shards diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java index 5a09a89910fc9..fca005e3e2b42 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java @@ -114,14 +114,11 @@ protected IndicesStatsRequest readRequestFrom(StreamInput in) throws IOException protected void shardOperation(IndicesStatsRequest request, ShardRouting shardRouting, Task task, ActionListener listener) { ActionListener.completeWith(listener, () -> { assert task instanceof CancellableTask; - IndicesQueryCache queryCache = indicesService.getIndicesQueryCache(); - IndicesQueryCache.CacheTotals cacheTotals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); - IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex()); IndexShard indexShard = indexService.getShard(shardRouting.shardId().id()); ShardId shardId = indexShard.shardId(); - long sharedRam = IndicesQueryCache.getSharedRamSize(queryCache, indexShard, cacheTotals); + long sharedRam = IndicesQueryCache.getSharedRamSizeForShard(indicesService.getIndicesQueryCache(), indexShard, cacheTotals); CommonStats commonStats = CommonStats.getShardLevelStats( indicesService.getIndicesQueryCache(), diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index 97b42de64a716..94c6eeca65be9 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -102,63 +102,70 @@ private static QueryCacheStats toQueryCacheStatsSafe(@Nullable Stats stats) { return stats == null ? new QueryCacheStats() : stats.toQueryCacheStats(); } - /** Get usage statistics for the given shard. */ - public QueryCacheStats getStats(ShardId shard, long precomputedSharedRamBytesUsed) { - final QueryCacheStats queryCacheStats = toQueryCacheStatsSafe(shardStats.get(shard)); - queryCacheStats.addRamBytesUsed(precomputedSharedRamBytesUsed); - return queryCacheStats; - } - - @Override - public Weight doCache(Weight weight, QueryCachingPolicy policy) { - while (weight instanceof CachingWeightWrapper) { - weight = ((CachingWeightWrapper) weight).in; - } - final Weight in = cache.doCache(weight, policy); - // We wrap the weight to track the readers it sees and map them with - // the shards they belong to - return new CachingWeightWrapper(in); - } - public static CacheTotals getCacheTotalsForAllShards(IndicesService indicesService) { IndicesQueryCache queryCache = indicesService.getIndicesQueryCache(); boolean hasQueryCache = queryCache != null; long totalSize = 0L; int shardCount = 0; - boolean anyNonZero = false; for (final IndexService indexService : indicesService) { for (final IndexShard indexShard : indexService) { long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(indexShard.shardId()) : 0L; shardCount++; if (cacheSize > 0L) { - anyNonZero = true; totalSize += cacheSize; } } } long sharedRamBytesUsed = hasQueryCache ? queryCache.getSharedRamBytesUsed() : 0L; - return new CacheTotals(totalSize, shardCount, anyNonZero, sharedRamBytesUsed); + return new CacheTotals(totalSize, shardCount, sharedRamBytesUsed); } - public static long getSharedRamSize(IndicesQueryCache queryCache, IndexShard indexShard, CacheTotals cacheTotals) { + public static long getSharedRamSizeForShard(IndicesQueryCache queryCache, IndexShard indexShard, CacheTotals cacheTotals) { long sharedRamBytesUsed = cacheTotals.sharedRamBytesUsed(); - long totalSize = cacheTotals.totalSize(); - boolean anyNonZero = cacheTotals.anyNonZero(); + if (sharedRamBytesUsed == 0L) { + return 0L; + } + int shardCount = cacheTotals.shardCount(); - ShardId shardId = indexShard.shardId(); - long cacheSize = queryCache != null ? queryCache.getCacheSizeForShard(shardId) : 0L; - long sharedRam = 0L; - if (sharedRamBytesUsed != 0L) { - if (anyNonZero == false) { - sharedRam = Math.round((double) sharedRamBytesUsed / shardCount); - } else if (totalSize != 0) { - sharedRam = Math.round((double) sharedRamBytesUsed * cacheSize / totalSize); - } + if (shardCount == 0) { + // Sometimes it's not possible to do this when there are no shard entries at all, which can happen as the shared ram usage can + // extend beyond the closing of all shards. + return 0L; } - return sharedRam; + + long totalSize = cacheTotals.totalSize(); + long cacheSize = queryCache != null ? queryCache.getCacheSizeForShard(indexShard.shardId()) : 0L; + final long additionalRamBytesUsed; + if (totalSize == 0) { + // all shards have zero cache footprint, so we apportion the size of the shared bytes equally across all shards + additionalRamBytesUsed = Math.round((double) sharedRamBytesUsed / shardCount); + } else { + // some shards have nonzero cache footprint, so we apportion the size of the shared bytes proportionally to cache footprint + additionalRamBytesUsed = Math.round((double) sharedRamBytesUsed * cacheSize / totalSize); + } + assert additionalRamBytesUsed >= 0L : additionalRamBytesUsed; + return additionalRamBytesUsed; + } + + public record CacheTotals(long totalSize, int shardCount, long sharedRamBytesUsed) {} + + /** Get usage statistics for the given shard. */ + public QueryCacheStats getStats(ShardId shard, long precomputedSharedRamBytesUsed) { + final QueryCacheStats queryCacheStats = toQueryCacheStatsSafe(shardStats.get(shard)); + queryCacheStats.addRamBytesUsed(precomputedSharedRamBytesUsed); + return queryCacheStats; } - public record CacheTotals(long totalSize, int shardCount, boolean anyNonZero, long sharedRamBytesUsed) {} + @Override + public Weight doCache(Weight weight, QueryCachingPolicy policy) { + while (weight instanceof CachingWeightWrapper) { + weight = ((CachingWeightWrapper) weight).in; + } + final Weight in = cache.doCache(weight, policy); + // We wrap the weight to track the readers it sees and map them with + // the shards they belong to + return new CachingWeightWrapper(in); + } private class CachingWeightWrapper extends Weight { diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 44ede7ed2c7e1..2c049ce66c9e3 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -519,12 +519,11 @@ static Map statsByIndex(final IndicesService indicesService, } static Map> statsByShard(final IndicesService indicesService, final CommonStatsFlags flags) { - IndicesQueryCache queryCache = indicesService.getIndicesQueryCache(); IndicesQueryCache.CacheTotals cacheTotals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); final Map> statsByShard = new HashMap<>(); for (final IndexService indexService : indicesService) { for (final IndexShard indexShard : indexService) { - long sharedRam = IndicesQueryCache.getSharedRamSize(queryCache, indexShard, cacheTotals); + long sharedRam = IndicesQueryCache.getSharedRamSizeForShard(indicesService.getIndicesQueryCache(), indexShard, cacheTotals); try { final IndexShardStats indexShardStats = indicesService.indexShardStats(indicesService, indexShard, flags, sharedRam); if (indexShardStats == null) { From 21ac4f49f5a6e6a07f4c03afa2ed2d2f42eb8d11 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 16 Jul 2025 17:43:14 -0500 Subject: [PATCH 12/34] moving code back --- .../indices/IndicesQueryCache.java | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index 94c6eeca65be9..f87d5199b1fbd 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -102,6 +102,24 @@ private static QueryCacheStats toQueryCacheStatsSafe(@Nullable Stats stats) { return stats == null ? new QueryCacheStats() : stats.toQueryCacheStats(); } + /** Get usage statistics for the given shard. */ + public QueryCacheStats getStats(ShardId shard, long precomputedSharedRamBytesUsed) { + final QueryCacheStats queryCacheStats = toQueryCacheStatsSafe(shardStats.get(shard)); + queryCacheStats.addRamBytesUsed(precomputedSharedRamBytesUsed); + return queryCacheStats; + } + + @Override + public Weight doCache(Weight weight, QueryCachingPolicy policy) { + while (weight instanceof CachingWeightWrapper) { + weight = ((CachingWeightWrapper) weight).in; + } + final Weight in = cache.doCache(weight, policy); + // We wrap the weight to track the readers it sees and map them with + // the shards they belong to + return new CachingWeightWrapper(in); + } + public static CacheTotals getCacheTotalsForAllShards(IndicesService indicesService) { IndicesQueryCache queryCache = indicesService.getIndicesQueryCache(); boolean hasQueryCache = queryCache != null; @@ -149,24 +167,6 @@ public static long getSharedRamSizeForShard(IndicesQueryCache queryCache, IndexS public record CacheTotals(long totalSize, int shardCount, long sharedRamBytesUsed) {} - /** Get usage statistics for the given shard. */ - public QueryCacheStats getStats(ShardId shard, long precomputedSharedRamBytesUsed) { - final QueryCacheStats queryCacheStats = toQueryCacheStatsSafe(shardStats.get(shard)); - queryCacheStats.addRamBytesUsed(precomputedSharedRamBytesUsed); - return queryCacheStats; - } - - @Override - public Weight doCache(Weight weight, QueryCachingPolicy policy) { - while (weight instanceof CachingWeightWrapper) { - weight = ((CachingWeightWrapper) weight).in; - } - final Weight in = cache.doCache(weight, policy); - // We wrap the weight to track the readers it sees and map them with - // the shards they belong to - return new CachingWeightWrapper(in); - } - private class CachingWeightWrapper extends Weight { private final Weight in; From 3873deb7823e4b1e7ebf98036e981b423e994b25 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Thu, 17 Jul 2025 16:35:42 -0500 Subject: [PATCH 13/34] adding unit tests --- .../stats/TransportIndicesStatsAction.java | 2 - .../indices/IndicesQueryCache.java | 64 +++++++++++-------- .../indices/IndicesQueryCacheTests.java | 59 +++++++++++++++++ 3 files changed, 95 insertions(+), 30 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java index fca005e3e2b42..e86d47a82bd83 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java @@ -117,9 +117,7 @@ protected void shardOperation(IndicesStatsRequest request, ShardRouting shardRou IndicesQueryCache.CacheTotals cacheTotals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex()); IndexShard indexShard = indexService.getShard(shardRouting.shardId().id()); - ShardId shardId = indexShard.shardId(); long sharedRam = IndicesQueryCache.getSharedRamSizeForShard(indicesService.getIndicesQueryCache(), indexShard, cacheTotals); - CommonStats commonStats = CommonStats.getShardLevelStats( indicesService.getIndicesQueryCache(), indexShard, diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index f87d5199b1fbd..8032344fb524b 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -69,13 +69,11 @@ public class IndicesQueryCache implements QueryCache, Closeable { private final Map shardStats = new ConcurrentHashMap<>(); private volatile long sharedRamBytesUsed; - // Package-private for IndicesService efficient stats collection public long getCacheSizeForShard(ShardId shardId) { Stats stats = shardStats.get(shardId); return stats != null ? stats.cacheSize : 0L; } - // Package-private for IndicesService efficient stats collection public long getSharedRamBytesUsed() { return sharedRamBytesUsed; } @@ -102,24 +100,11 @@ private static QueryCacheStats toQueryCacheStatsSafe(@Nullable Stats stats) { return stats == null ? new QueryCacheStats() : stats.toQueryCacheStats(); } - /** Get usage statistics for the given shard. */ - public QueryCacheStats getStats(ShardId shard, long precomputedSharedRamBytesUsed) { - final QueryCacheStats queryCacheStats = toQueryCacheStatsSafe(shardStats.get(shard)); - queryCacheStats.addRamBytesUsed(precomputedSharedRamBytesUsed); - return queryCacheStats; - } - - @Override - public Weight doCache(Weight weight, QueryCachingPolicy policy) { - while (weight instanceof CachingWeightWrapper) { - weight = ((CachingWeightWrapper) weight).in; - } - final Weight in = cache.doCache(weight, policy); - // We wrap the weight to track the readers it sees and map them with - // the shards they belong to - return new CachingWeightWrapper(in); - } - + /** + * This computes the total cache size in bytes, and the total shard count in the cache for all shards. + * @param indicesService + * @return A CacheTotals object containing the computed total cache size and shard count + */ public static CacheTotals getCacheTotalsForAllShards(IndicesService indicesService) { IndicesQueryCache queryCache = indicesService.getIndicesQueryCache(); boolean hasQueryCache = queryCache != null; @@ -129,17 +114,22 @@ public static CacheTotals getCacheTotalsForAllShards(IndicesService indicesServi for (final IndexShard indexShard : indexService) { long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(indexShard.shardId()) : 0L; shardCount++; - if (cacheSize > 0L) { - totalSize += cacheSize; - } + assert cacheSize >= 0 : "Unexpected cache size of " + cacheSize + " for shard " + indexShard.shardId(); + totalSize += cacheSize; } } - long sharedRamBytesUsed = hasQueryCache ? queryCache.getSharedRamBytesUsed() : 0L; - return new CacheTotals(totalSize, shardCount, sharedRamBytesUsed); + return new CacheTotals(totalSize, shardCount); } + /** + * This method computes the shared RAM size in bytes for the given indexShard. + * @param queryCache + * @param indexShard The shard to compute the shared RAM size for + * @param cacheTotals Shard totals computed in getCacheTotalsForAllShards() + * @return + */ public static long getSharedRamSizeForShard(IndicesQueryCache queryCache, IndexShard indexShard, CacheTotals cacheTotals) { - long sharedRamBytesUsed = cacheTotals.sharedRamBytesUsed(); + long sharedRamBytesUsed = queryCache != null ? queryCache.getSharedRamBytesUsed() : 0L; if (sharedRamBytesUsed == 0L) { return 0L; } @@ -152,7 +142,7 @@ public static long getSharedRamSizeForShard(IndicesQueryCache queryCache, IndexS } long totalSize = cacheTotals.totalSize(); - long cacheSize = queryCache != null ? queryCache.getCacheSizeForShard(indexShard.shardId()) : 0L; + long cacheSize = queryCache.getCacheSizeForShard(indexShard.shardId()); final long additionalRamBytesUsed; if (totalSize == 0) { // all shards have zero cache footprint, so we apportion the size of the shared bytes equally across all shards @@ -165,7 +155,25 @@ public static long getSharedRamSizeForShard(IndicesQueryCache queryCache, IndexS return additionalRamBytesUsed; } - public record CacheTotals(long totalSize, int shardCount, long sharedRamBytesUsed) {} + public record CacheTotals(long totalSize, int shardCount) {} + + /** Get usage statistics for the given shard. */ + public QueryCacheStats getStats(ShardId shard, long precomputedSharedRamBytesUsed) { + final QueryCacheStats queryCacheStats = toQueryCacheStatsSafe(shardStats.get(shard)); + queryCacheStats.addRamBytesUsed(precomputedSharedRamBytesUsed); + return queryCacheStats; + } + + @Override + public Weight doCache(Weight weight, QueryCachingPolicy policy) { + while (weight instanceof CachingWeightWrapper) { + weight = ((CachingWeightWrapper) weight).in; + } + final Weight in = cache.doCache(weight, policy); + // We wrap the weight to track the readers it sees and map them with + // the shards they belong to + return new CachingWeightWrapper(in); + } private class CachingWeightWrapper extends Weight { diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java index f8488fedba32f..2a43261e9d348 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java @@ -29,12 +29,19 @@ import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.cache.query.QueryCacheStats; import org.elasticsearch.index.cache.query.TrivialQueryCachingPolicy; +import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.List; + +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class IndicesQueryCacheTests extends ESTestCase { @@ -404,4 +411,56 @@ public void testDelegatesScorerSupplier() throws Exception { cache.onClose(shard); cache.close(); } + public void testGetCacheTotalsForAllShards() throws Exception { + ShardId shardId1 = new ShardId("index", "_na_", 0); + ShardId shardId2 = new ShardId("index", "_na_", 1); + + IndexShard shard1 = mock(IndexShard.class); + IndexShard shard2 = mock(IndexShard.class); + when(shard1.shardId()).thenReturn(shardId1); + when(shard2.shardId()).thenReturn(shardId2); + + IndexService indexService = mock(IndexService.class, RETURNS_DEEP_STUBS); + when(indexService.iterator()).thenReturn(List.of(shard1, shard2).iterator()); + + IndicesService indicesService = mock(IndicesService.class, RETURNS_DEEP_STUBS); + when(indicesService.iterator()).thenReturn(List.of(indexService).iterator()); + IndicesQueryCache queryCache = mock(IndicesQueryCache.class); + when(indicesService.getIndicesQueryCache()).thenReturn(queryCache); + when(queryCache.getCacheSizeForShard(shardId1)).thenReturn(100L); + when(queryCache.getCacheSizeForShard(shardId2)).thenReturn(200L); + + IndicesQueryCache.CacheTotals totals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); + assertEquals(300L, totals.totalSize()); + assertEquals(2, totals.shardCount()); + } + + public void testGetSharedRamSizeForShard() { + ShardId shardId1 = new ShardId("index", "_na_", 0); + ShardId shardId2 = new ShardId("index", "_na_", 1); + IndexShard shard1 = mock(IndexShard.class); + IndexShard shard2 = mock(IndexShard.class); + when(shard1.shardId()).thenReturn(shardId1); + when(shard2.shardId()).thenReturn(shardId2); + + IndicesQueryCache.CacheTotals totals = new IndicesQueryCache.CacheTotals(300L, 2); + IndicesQueryCache queryCache = mock(IndicesQueryCache.class); + // Case 1: sharedRamBytesUsed = 0 + when(queryCache.getSharedRamBytesUsed()).thenReturn(0L); + long sharedRam = IndicesQueryCache.getSharedRamSizeForShard(queryCache, shard1, totals); + assertEquals(0L, sharedRam); + // Case 2: sharedRamBytesUsed > 0, totalSize > 0, proportional + when(queryCache.getSharedRamBytesUsed()).thenReturn(600L); + when(queryCache.getCacheSizeForShard(shardId1)).thenReturn(100L); + long sharedRam1 = IndicesQueryCache.getSharedRamSizeForShard(queryCache, shard1, totals); + assertEquals(200L, sharedRam1); + when(queryCache.getCacheSizeForShard(shardId2)).thenReturn(200L); + long sharedRam2 = IndicesQueryCache.getSharedRamSizeForShard(queryCache, shard2, totals); + assertEquals(400L, sharedRam2); + // Case 3: totalSize == 0, shared equally + IndicesQueryCache.CacheTotals zeroTotals = new IndicesQueryCache.CacheTotals(0L, 2); + when(queryCache.getSharedRamBytesUsed()).thenReturn(600L); + long sharedRamEq = IndicesQueryCache.getSharedRamSizeForShard(queryCache, shard1, zeroTotals); + assertEquals(300L, sharedRamEq); + } } From 4871c7c5125578e89408326d472750f200400049 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 17 Jul 2025 21:44:37 +0000 Subject: [PATCH 14/34] [CI] Auto commit changes from spotless --- .../action/admin/indices/stats/TransportIndicesStatsAction.java | 1 - .../java/org/elasticsearch/indices/IndicesQueryCacheTests.java | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java index e86d47a82bd83..821d187225c57 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java @@ -27,7 +27,6 @@ import org.elasticsearch.index.seqno.RetentionLeaseStats; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesQueryCache; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.injection.guice.Inject; diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java index 2a43261e9d348..267c7a3f8ab95 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java @@ -411,6 +411,7 @@ public void testDelegatesScorerSupplier() throws Exception { cache.onClose(shard); cache.close(); } + public void testGetCacheTotalsForAllShards() throws Exception { ShardId shardId1 = new ShardId("index", "_na_", 0); ShardId shardId2 = new ShardId("index", "_na_", 1); From 941454488deb28f2e9ccac9228741a8b8b546c31 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 26 Sep 2025 13:37:24 -0500 Subject: [PATCH 15/34] making sure it works with changes from #135298 --- .../indices/IndicesQueryCache.java | 19 +- .../indices/IndicesQueryCacheTests.java | 195 +++++++++++++++++- 2 files changed, 204 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index 8032344fb524b..b1d90706df481 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -140,16 +140,23 @@ public static long getSharedRamSizeForShard(IndicesQueryCache queryCache, IndexS // extend beyond the closing of all shards. return 0L; } - - long totalSize = cacheTotals.totalSize(); - long cacheSize = queryCache.getCacheSizeForShard(indexShard.shardId()); + /* + * We have some shared ram usage that we try to distribute proportionally to the number of segment-requestss in the cache for each + * shard. + */ + long totalItemsInCache = cacheTotals.totalSize(); + long itemsInCacheForShard = queryCache.getCacheSizeForShard(indexShard.shardId()); final long additionalRamBytesUsed; - if (totalSize == 0) { + if (totalItemsInCache == 0) { // all shards have zero cache footprint, so we apportion the size of the shared bytes equally across all shards additionalRamBytesUsed = Math.round((double) sharedRamBytesUsed / shardCount); } else { - // some shards have nonzero cache footprint, so we apportion the size of the shared bytes proportionally to cache footprint - additionalRamBytesUsed = Math.round((double) sharedRamBytesUsed * cacheSize / totalSize); + /* + * some shards have nonzero cache footprint, so we apportion the size of the shared bytes proportionally to the number of + * segment-requests in the cache for this shard (the number and size of documents associated with those requests is irrelevant + * for this calculation). + */ + additionalRamBytesUsed = Math.round((double) sharedRamBytesUsed * itemsInCacheForShard / totalItemsInCache); } assert additionalRamBytesUsed >= 0L : additionalRamBytesUsed; return additionalRamBytesUsed; diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java index 267c7a3f8ab95..578b5f3e6502f 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.ScorerSupplier; import org.apache.lucene.search.Weight; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Accountable; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.IOUtils; @@ -36,31 +37,47 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; +import java.io.Closeable; import java.io.IOException; +import java.util.ArrayList; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThan; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class IndicesQueryCacheTests extends ESTestCase { - private static class DummyQuery extends Query { + private static class DummyQuery extends Query implements Accountable { - private final int id; + private final String id; + private final long sizeInCache; DummyQuery(int id) { + this(Integer.toString(id), 10); + } + + DummyQuery(String id) { + this(id, 10); + } + + DummyQuery(String id, long sizeInCache) { this.id = id; + this.sizeInCache = sizeInCache; } @Override public boolean equals(Object obj) { - return sameClassAs(obj) && id == ((DummyQuery) obj).id; + return sameClassAs(obj) && id.equals(((DummyQuery) obj).id); } @Override public int hashCode() { - return 31 * classHash() + id; + return 31 * classHash() + id.hashCode(); } @Override @@ -89,6 +106,10 @@ public boolean isCacheable(LeafReaderContext ctx) { }; } + @Override + public long ramBytesUsed() { + return sizeInCache; + } } public void testBasics() throws IOException { @@ -464,4 +485,170 @@ public void testGetSharedRamSizeForShard() { long sharedRamEq = IndicesQueryCache.getSharedRamSizeForShard(queryCache, shard1, zeroTotals); assertEquals(300L, sharedRamEq); } + + public void testGetStatsMemory() throws Exception { + /* + * This test creates 2 shards, one with two segments and one with one. It makes unique queries against all 3 segments (so that each + * query will be cached, up to the max cache size), and then asserts various things about the cache memory. Most importantly, it + * asserts that the memory the cache attributes to each shard is proportional to the number of segment-queries for the shard in the + * cache (and not to the number of documents in the query). + */ + String indexName = randomIdentifier(); + String uuid = randomUUID(); + ShardId shard1 = new ShardId(indexName, uuid, 0); + ShardId shard2 = new ShardId(indexName, uuid, 1); + List closeableList = new ArrayList<>(); + // We're going to create 2 segments for shard1, and 1 segment for shard2: + int shard1Segment1Docs = randomIntBetween(11, 1000); + int shard1Segment2Docs = randomIntBetween(1, 10); + int shard2Segment1Docs = randomIntBetween(1, 10); + IndexSearcher shard1Segment1Searcher = initializeSegment(shard1, shard1Segment1Docs, closeableList); + IndexSearcher shard1Segment2Searcher = initializeSegment(shard1, shard1Segment2Docs, closeableList); + IndexSearcher shard2Searcher = initializeSegment(shard2, shard2Segment1Docs, closeableList); + + final int maxCacheSize = 200; + Settings settings = Settings.builder() + .put(IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING.getKey(), maxCacheSize) + .put(IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.getKey(), true) + .build(); + IndicesQueryCache cache = new IndicesQueryCache(settings); + shard1Segment1Searcher.setQueryCache(cache); + shard1Segment2Searcher.setQueryCache(cache); + shard2Searcher.setQueryCache(cache); + + assertEquals(0L, cache.getStats(shard1, 0).getMemorySizeInBytes()); + + final long largeQuerySize = randomIntBetween(100, 1000); + final long smallQuerySize = randomIntBetween(10, 50); + + final int shard1Queries = randomIntBetween(20, 50); + final int shard2Queries = randomIntBetween(5, 10); + + for (int i = 0; i < shard1Queries; ++i) { + shard1Segment1Searcher.count(new DummyQuery("ingest1-" + i, largeQuerySize)); + } + IndicesQueryCache.CacheTotals cacheTotals = new IndicesQueryCache.CacheTotals(shard1Queries, 1); + IndexShard indexShard1 = mock(IndexShard.class); + when(indexShard1.shardId()).thenReturn(shard1); + IndexShard indexShard2 = mock(IndexShard.class); + when(indexShard2.shardId()).thenReturn(shard2); + long sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(cache, indexShard1, cacheTotals); + long sharedRamSizeShard2 = IndicesQueryCache.getSharedRamSizeForShard(cache, indexShard2, cacheTotals); + long shard1Segment1CacheMemory = calculateActualCacheMemoryForSegment(shard1Queries, largeQuerySize, shard1Segment1Docs); + assertThat(cache.getStats(shard1, sharedRamSizeShard1).getMemorySizeInBytes(), equalTo(shard1Segment1CacheMemory)); + assertThat(cache.getStats(shard2, sharedRamSizeShard2).getMemorySizeInBytes(), equalTo(0L)); + for (int i = 0; i < shard2Queries; ++i) { + shard2Searcher.count(new DummyQuery("ingest2-" + i, smallQuerySize)); + } + /* + * Now that we have cached some smaller things for shard2, the cache memory for shard1 has gone down. This is expected because we + * report cache memory proportional to the number of segments for each shard, ignoring the number of documents or the actual + * document sizes. Since the shard2 requests were smaller, the average cache memory size per segment has now gone down. + */ + cacheTotals = new IndicesQueryCache.CacheTotals(shard1Queries + shard2Queries, 2); + sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(cache, indexShard1, cacheTotals); + sharedRamSizeShard2 = IndicesQueryCache.getSharedRamSizeForShard(cache, indexShard2, cacheTotals); + assertThat(cache.getStats(shard1, sharedRamSizeShard1).getMemorySizeInBytes(), lessThan(shard1Segment1CacheMemory)); + long shard1CacheBytes = cache.getStats(shard1, sharedRamSizeShard1).getMemorySizeInBytes(); + long shard2CacheBytes = cache.getStats(shard2, sharedRamSizeShard2).getMemorySizeInBytes(); + long shard2Segment1CacheMemory = calculateActualCacheMemoryForSegment(shard2Queries, smallQuerySize, shard2Segment1Docs); + + long totalMemory = shard1Segment1CacheMemory + shard2Segment1CacheMemory; + // Each shard has some fixed overhead that we need to account for: + long shard1Overhead = calculateOverheadForSegment(shard1Queries, shard1Segment1Docs); + long shard2Overhead = calculateOverheadForSegment(shard2Queries, shard2Segment1Docs); + long totalMemoryMinusOverhead = totalMemory - (shard1Overhead + shard2Overhead); + /* + * Note that the expected amount of memory we're calculating is based on the proportion of the number of queries to each shard + * (since each shard currently only has queries to one segment) + */ + double shard1Segment1CacheMemoryShare = ((double) shard1Queries / (shard1Queries + shard2Queries)) * (totalMemoryMinusOverhead) + + shard1Overhead; + double shard2Segment1CacheMemoryShare = ((double) shard2Queries / (shard1Queries + shard2Queries)) * (totalMemoryMinusOverhead) + + shard2Overhead; + assertThat((double) shard1CacheBytes, closeTo(shard1Segment1CacheMemoryShare, 1)); // accounting for rounding + assertThat((double) shard2CacheBytes, closeTo(shard2Segment1CacheMemoryShare, 1)); // accounting for rounding + + // Now we cache just more "big" searches on shard1, but on a different segment: + for (int i = 0; i < shard1Queries; ++i) { + shard1Segment2Searcher.count(new DummyQuery("ingest3-" + i, largeQuerySize)); + } + long shard1Segment2CacheMemory = calculateActualCacheMemoryForSegment(shard1Queries, largeQuerySize, shard1Segment2Docs); + totalMemory = shard1Segment1CacheMemory + shard2Segment1CacheMemory + shard1Segment2CacheMemory; + // Each shard has some fixed overhead that we need to account for: + shard1Overhead = shard1Overhead + calculateOverheadForSegment(shard1Queries, shard1Segment2Docs); + totalMemoryMinusOverhead = totalMemory - (shard1Overhead + shard2Overhead); + /* + * Note that the expected amount of memory we're calculating is based on the proportion of the number of queries to each segment. + * The number of documents and the size of documents is irrelevant (aside from computing the fixed overhead). + */ + shard1Segment1CacheMemoryShare = ((double) (2 * shard1Queries) / ((2 * shard1Queries) + shard2Queries)) * (totalMemoryMinusOverhead) + + shard1Overhead; + cacheTotals = new IndicesQueryCache.CacheTotals(2 * shard1Queries + shard2Queries, 2); + sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(cache, indexShard1, cacheTotals); + shard1CacheBytes = cache.getStats(shard1, sharedRamSizeShard1).getMemorySizeInBytes(); + assertThat((double) shard1CacheBytes, closeTo(shard1Segment1CacheMemoryShare, 1)); // accounting for rounding + + // Now make sure the cache only has items for shard2: + for (int i = 0; i < (maxCacheSize * 2); ++i) { + shard2Searcher.count(new DummyQuery("ingest4-" + i, smallQuerySize)); + } + cacheTotals = new IndicesQueryCache.CacheTotals(maxCacheSize, 1); + sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(cache, indexShard1, cacheTotals); + sharedRamSizeShard2 = IndicesQueryCache.getSharedRamSizeForShard(cache, indexShard2, cacheTotals); + assertThat(cache.getStats(shard1, sharedRamSizeShard1).getMemorySizeInBytes(), equalTo(0L)); + assertThat( + cache.getStats(shard2, sharedRamSizeShard2).getMemorySizeInBytes(), + equalTo(calculateActualCacheMemoryForSegment(maxCacheSize, smallQuerySize, shard2Segment1Docs)) + ); + + IOUtils.close(closeableList); + cache.onClose(shard1); + cache.onClose(shard2); + cache.close(); + } + + /* + * This calculates the memory that actually used by a segment in the IndicesQueryCache. It assumes queryCount queries are made to the + * segment, and query is querySize bytes in size. It assumes that the shard contains numDocs documents. + */ + private long calculateActualCacheMemoryForSegment(long queryCount, long querySize, long numDocs) { + return (queryCount * (querySize + 24)) + calculateOverheadForSegment(queryCount, numDocs); + } + + /* + * This computes the part of the recorded IndicesQueryCache memory that is assigned to a segment and *not* divided up proportionally + * when the cache reports the memory usage of each shard. + */ + private long calculateOverheadForSegment(long queryCount, long numDocs) { + return queryCount * (112 + (8 * ((numDocs - 1) / 64))); + } + + /* + * This returns an IndexSearcher for a single new segment in the given shard. + */ + private IndexSearcher initializeSegment(ShardId shard, int numDocs, List closeableList) throws Exception { + AtomicReference indexSearcherReference = new AtomicReference<>(); + /* + * Usually creating an IndexWriter like this results in a single segment getting created, but sometimes it results in more. For the + * sake of keeping the calculations in this test simple we want just a single segment. So we do this in an assertBusy. + */ + assertBusy(() -> { + Directory dir = newDirectory(); + IndexWriter indexWriter = new IndexWriter(dir, newIndexWriterConfig()); + for (int i = 0; i < numDocs; i++) { + indexWriter.addDocument(new Document()); + } + DirectoryReader directoryReader = DirectoryReader.open(indexWriter); + indexWriter.close(); + directoryReader = ElasticsearchDirectoryReader.wrap(directoryReader, shard); + IndexSearcher indexSearcher = new IndexSearcher(directoryReader); + indexSearcherReference.set(indexSearcher); + indexSearcher.setQueryCachingPolicy(TrivialQueryCachingPolicy.ALWAYS); + closeableList.add(directoryReader); + closeableList.add(dir); + assertThat(indexSearcher.getLeafContexts().size(), equalTo(1)); + }); + return indexSearcherReference.get(); + } } From d3d25d6e0ffcf0abf4c0becda00f912de38e2b8e Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 26 Sep 2025 14:20:52 -0500 Subject: [PATCH 16/34] fixed a (bad) typo --- .../elasticsearch/action/admin/indices/stats/CommonStats.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java index d74ef67f97bd9..3ac7d8515adc0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java @@ -179,7 +179,7 @@ public static CommonStats getShardLevelStats( case Refresh -> stats.refresh = indexShard.refreshStats(); case Flush -> stats.flush = indexShard.flushStats(); case Warmer -> stats.warmer = indexShard.warmerStats(); - case QueryCache -> stats.queryCache = indicesQueryCache.getStats(indexShard.shardId(), 0L); + case QueryCache -> stats.queryCache = indicesQueryCache.getStats(indexShard.shardId(), precomputedSharedRam); case FieldData -> stats.fieldData = indexShard.fieldDataStats(flags.fieldDataFields()); case Completion -> stats.completion = indexShard.completionStats(flags.completionDataFields()); case Segments -> stats.segments = indexShard.segmentStats( From 94c4ab7da8e30c7558ec4c90fc3cc965bd532327 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 26 Sep 2025 15:46:09 -0500 Subject: [PATCH 17/34] minor cleanup --- .../org/elasticsearch/indices/IndicesQueryCache.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index 9b1465a073d86..e69c3482861d5 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -103,22 +103,22 @@ private static QueryCacheStats toQueryCacheStatsSafe(@Nullable Stats stats) { /** * This computes the total cache size in bytes, and the total shard count in the cache for all shards. * @param indicesService - * @return A CacheTotals object containing the computed total cache size and shard count + * @return A CacheTotals object containing the computed total number of items in the cache and the number of shards seen in the cache */ public static CacheTotals getCacheTotalsForAllShards(IndicesService indicesService) { IndicesQueryCache queryCache = indicesService.getIndicesQueryCache(); boolean hasQueryCache = queryCache != null; - long totalSize = 0L; + long totalItemsInCache = 0L; int shardCount = 0; for (final IndexService indexService : indicesService) { for (final IndexShard indexShard : indexService) { long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(indexShard.shardId()) : 0L; shardCount++; assert cacheSize >= 0 : "Unexpected cache size of " + cacheSize + " for shard " + indexShard.shardId(); - totalSize += cacheSize; + totalItemsInCache += cacheSize; } } - return new CacheTotals(totalSize, shardCount); + return new CacheTotals(totalItemsInCache, shardCount); } /** @@ -144,7 +144,7 @@ public static long getSharedRamSizeForShard(IndicesQueryCache queryCache, IndexS * We have some shared ram usage that we try to distribute proportionally to the number of segment-requestss in the cache for each * shard. */ - long totalItemsInCache = cacheTotals.totalSize(); + long totalItemsInCache = cacheTotals.totalItemsInCache(); long itemsInCacheForShard = queryCache.getCacheSizeForShard(indexShard.shardId()); final long additionalRamBytesUsed; if (totalItemsInCache == 0) { @@ -166,7 +166,7 @@ public static long getSharedRamSizeForShard(IndicesQueryCache queryCache, IndexS return additionalRamBytesUsed; } - public record CacheTotals(long totalSize, int shardCount) {} + public record CacheTotals(long totalItemsInCache, int shardCount) {} /** Get usage statistics for the given shard. */ public QueryCacheStats getStats(ShardId shard, long precomputedSharedRamBytesUsed) { From bf71afc7d307afd649137039b01abdf9b6350c3b Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 26 Sep 2025 15:46:42 -0500 Subject: [PATCH 18/34] fixing description --- docs/changelog/130857.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/130857.yaml b/docs/changelog/130857.yaml index 6e42d882070f7..021b2bc970f73 100644 --- a/docs/changelog/130857.yaml +++ b/docs/changelog/130857.yaml @@ -1,5 +1,5 @@ pr: 130857 -summary: First attempt at improving `statsByShard` performance +summary: Improving statsByShard performance when the number of shards is very large area: Stats type: bug issues: [] From 12557966161796f47793e0a0ed0ccf0f03bbca7b Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 26 Sep 2025 15:48:02 -0500 Subject: [PATCH 19/34] Update docs/changelog/130857.yaml --- docs/changelog/130857.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog/130857.yaml b/docs/changelog/130857.yaml index 021b2bc970f73..5e5a5a309dfeb 100644 --- a/docs/changelog/130857.yaml +++ b/docs/changelog/130857.yaml @@ -2,4 +2,5 @@ pr: 130857 summary: Improving statsByShard performance when the number of shards is very large area: Stats type: bug -issues: [] +issues: + - 97222 From 1d2cbd73b688b76b2db032fadeec1f13dbc361ce Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 26 Sep 2025 15:53:19 -0500 Subject: [PATCH 20/34] improving comments --- .../java/org/elasticsearch/indices/IndicesQueryCache.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index e69c3482861d5..7e10fcbfe27a2 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -126,7 +126,7 @@ public static CacheTotals getCacheTotalsForAllShards(IndicesService indicesServi * @param queryCache * @param indexShard The shard to compute the shared RAM size for * @param cacheTotals Shard totals computed in getCacheTotalsForAllShards() - * @return + * @return the shared RAM size in bytes allocated to the given shard, or 0 if unavailable */ public static long getSharedRamSizeForShard(IndicesQueryCache queryCache, IndexShard indexShard, CacheTotals cacheTotals) { long sharedRamBytesUsed = queryCache != null ? queryCache.getSharedRamBytesUsed() : 0L; @@ -141,7 +141,7 @@ public static long getSharedRamSizeForShard(IndicesQueryCache queryCache, IndexS return 0L; } /* - * We have some shared ram usage that we try to distribute proportionally to the number of segment-requestss in the cache for each + * We have some shared ram usage that we try to distribute proportionally to the number of segment-requests in the cache for each * shard. */ long totalItemsInCache = cacheTotals.totalItemsInCache(); From 441574ff9c35fc0c7f042a4ed22806ab9d0e0d35 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 26 Sep 2025 16:07:00 -0500 Subject: [PATCH 21/34] fixing a bad record field rename --- .../java/org/elasticsearch/indices/IndicesQueryCacheTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java index 578b5f3e6502f..452ff731a40d9 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java @@ -453,7 +453,7 @@ public void testGetCacheTotalsForAllShards() throws Exception { when(queryCache.getCacheSizeForShard(shardId2)).thenReturn(200L); IndicesQueryCache.CacheTotals totals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); - assertEquals(300L, totals.totalSize()); + assertEquals(300L, totals.totalItemsInCache()); assertEquals(2, totals.shardCount()); } From d380eb5cde2e4efc4ffc43fd1fa02cd2c6c601bd Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Fri, 26 Sep 2025 17:59:18 -0400 Subject: [PATCH 22/34] Rely on the ShardId rather than the IndexShard --- .../stats/TransportClusterStatsAction.java | 6 ++++- .../stats/TransportIndicesStatsAction.java | 6 ++++- .../indices/IndicesQueryCache.java | 11 +++++----- .../elasticsearch/indices/IndicesService.java | 6 ++++- .../indices/IndicesQueryCacheTests.java | 22 +++++++++---------- 5 files changed, 32 insertions(+), 19 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index a90f689e448ea..98ab7fb5edd53 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -263,7 +263,11 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq List shardsStats = new ArrayList<>(); for (IndexService indexService : indicesService) { for (IndexShard indexShard : indexService) { - long sharedRam = IndicesQueryCache.getSharedRamSizeForShard(indicesService.getIndicesQueryCache(), indexShard, cacheTotals); + long sharedRam = IndicesQueryCache.getSharedRamSizeForShard( + indicesService.getIndicesQueryCache(), + indexShard.shardId(), + cacheTotals + ); cancellableTask.ensureNotCancelled(); if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) { // only report on fully started shards diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java index 821d187225c57..270997186ecb5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java @@ -116,7 +116,11 @@ protected void shardOperation(IndicesStatsRequest request, ShardRouting shardRou IndicesQueryCache.CacheTotals cacheTotals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex()); IndexShard indexShard = indexService.getShard(shardRouting.shardId().id()); - long sharedRam = IndicesQueryCache.getSharedRamSizeForShard(indicesService.getIndicesQueryCache(), indexShard, cacheTotals); + long sharedRam = IndicesQueryCache.getSharedRamSizeForShard( + indicesService.getIndicesQueryCache(), + indexShard.shardId(), + cacheTotals + ); CommonStats commonStats = CommonStats.getShardLevelStats( indicesService.getIndicesQueryCache(), indexShard, diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index 7e10fcbfe27a2..a1e206e06400b 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -112,9 +112,10 @@ public static CacheTotals getCacheTotalsForAllShards(IndicesService indicesServi int shardCount = 0; for (final IndexService indexService : indicesService) { for (final IndexShard indexShard : indexService) { - long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(indexShard.shardId()) : 0L; + final var shardId = indexShard.shardId(); + long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(shardId) : 0L; shardCount++; - assert cacheSize >= 0 : "Unexpected cache size of " + cacheSize + " for shard " + indexShard.shardId(); + assert cacheSize >= 0 : "Unexpected cache size of " + cacheSize + " for shard " + shardId; totalItemsInCache += cacheSize; } } @@ -124,11 +125,11 @@ public static CacheTotals getCacheTotalsForAllShards(IndicesService indicesServi /** * This method computes the shared RAM size in bytes for the given indexShard. * @param queryCache - * @param indexShard The shard to compute the shared RAM size for + * @param shardId The shard to compute the shared RAM size for * @param cacheTotals Shard totals computed in getCacheTotalsForAllShards() * @return the shared RAM size in bytes allocated to the given shard, or 0 if unavailable */ - public static long getSharedRamSizeForShard(IndicesQueryCache queryCache, IndexShard indexShard, CacheTotals cacheTotals) { + public static long getSharedRamSizeForShard(IndicesQueryCache queryCache, ShardId shardId, CacheTotals cacheTotals) { long sharedRamBytesUsed = queryCache != null ? queryCache.getSharedRamBytesUsed() : 0L; if (sharedRamBytesUsed == 0L) { return 0L; @@ -145,7 +146,7 @@ public static long getSharedRamSizeForShard(IndicesQueryCache queryCache, IndexS * shard. */ long totalItemsInCache = cacheTotals.totalItemsInCache(); - long itemsInCacheForShard = queryCache.getCacheSizeForShard(indexShard.shardId()); + long itemsInCacheForShard = queryCache.getCacheSizeForShard(shardId); final long additionalRamBytesUsed; if (totalItemsInCache == 0) { // all shards have zero cache footprint, so we apportion the size of the shared bytes equally across all shards diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index d2faf0e69b35e..cd1b6cd801a69 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -524,7 +524,11 @@ static Map> statsByShard(final IndicesService indic final Map> statsByShard = new HashMap<>(); for (final IndexService indexService : indicesService) { for (final IndexShard indexShard : indexService) { - long sharedRam = IndicesQueryCache.getSharedRamSizeForShard(indicesService.getIndicesQueryCache(), indexShard, cacheTotals); + long sharedRam = IndicesQueryCache.getSharedRamSizeForShard( + indicesService.getIndicesQueryCache(), + indexShard.shardId(), + cacheTotals + ); try { final IndexShardStats indexShardStats = indicesService.indexShardStats(indicesService, indexShard, flags, sharedRam); if (indexShardStats == null) { diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java index 452ff731a40d9..6a04a491641e5 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java @@ -469,20 +469,20 @@ public void testGetSharedRamSizeForShard() { IndicesQueryCache queryCache = mock(IndicesQueryCache.class); // Case 1: sharedRamBytesUsed = 0 when(queryCache.getSharedRamBytesUsed()).thenReturn(0L); - long sharedRam = IndicesQueryCache.getSharedRamSizeForShard(queryCache, shard1, totals); + long sharedRam = IndicesQueryCache.getSharedRamSizeForShard(queryCache, shardId1, totals); assertEquals(0L, sharedRam); // Case 2: sharedRamBytesUsed > 0, totalSize > 0, proportional when(queryCache.getSharedRamBytesUsed()).thenReturn(600L); when(queryCache.getCacheSizeForShard(shardId1)).thenReturn(100L); - long sharedRam1 = IndicesQueryCache.getSharedRamSizeForShard(queryCache, shard1, totals); + long sharedRam1 = IndicesQueryCache.getSharedRamSizeForShard(queryCache, shardId1, totals); assertEquals(200L, sharedRam1); when(queryCache.getCacheSizeForShard(shardId2)).thenReturn(200L); - long sharedRam2 = IndicesQueryCache.getSharedRamSizeForShard(queryCache, shard2, totals); + long sharedRam2 = IndicesQueryCache.getSharedRamSizeForShard(queryCache, shardId2, totals); assertEquals(400L, sharedRam2); // Case 3: totalSize == 0, shared equally IndicesQueryCache.CacheTotals zeroTotals = new IndicesQueryCache.CacheTotals(0L, 2); when(queryCache.getSharedRamBytesUsed()).thenReturn(600L); - long sharedRamEq = IndicesQueryCache.getSharedRamSizeForShard(queryCache, shard1, zeroTotals); + long sharedRamEq = IndicesQueryCache.getSharedRamSizeForShard(queryCache, shardId1, zeroTotals); assertEquals(300L, sharedRamEq); } @@ -532,8 +532,8 @@ public void testGetStatsMemory() throws Exception { when(indexShard1.shardId()).thenReturn(shard1); IndexShard indexShard2 = mock(IndexShard.class); when(indexShard2.shardId()).thenReturn(shard2); - long sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(cache, indexShard1, cacheTotals); - long sharedRamSizeShard2 = IndicesQueryCache.getSharedRamSizeForShard(cache, indexShard2, cacheTotals); + long sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(cache, shard1, cacheTotals); + long sharedRamSizeShard2 = IndicesQueryCache.getSharedRamSizeForShard(cache, shard2, cacheTotals); long shard1Segment1CacheMemory = calculateActualCacheMemoryForSegment(shard1Queries, largeQuerySize, shard1Segment1Docs); assertThat(cache.getStats(shard1, sharedRamSizeShard1).getMemorySizeInBytes(), equalTo(shard1Segment1CacheMemory)); assertThat(cache.getStats(shard2, sharedRamSizeShard2).getMemorySizeInBytes(), equalTo(0L)); @@ -546,8 +546,8 @@ public void testGetStatsMemory() throws Exception { * document sizes. Since the shard2 requests were smaller, the average cache memory size per segment has now gone down. */ cacheTotals = new IndicesQueryCache.CacheTotals(shard1Queries + shard2Queries, 2); - sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(cache, indexShard1, cacheTotals); - sharedRamSizeShard2 = IndicesQueryCache.getSharedRamSizeForShard(cache, indexShard2, cacheTotals); + sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(cache, shard1, cacheTotals); + sharedRamSizeShard2 = IndicesQueryCache.getSharedRamSizeForShard(cache, shard2, cacheTotals); assertThat(cache.getStats(shard1, sharedRamSizeShard1).getMemorySizeInBytes(), lessThan(shard1Segment1CacheMemory)); long shard1CacheBytes = cache.getStats(shard1, sharedRamSizeShard1).getMemorySizeInBytes(); long shard2CacheBytes = cache.getStats(shard2, sharedRamSizeShard2).getMemorySizeInBytes(); @@ -585,7 +585,7 @@ public void testGetStatsMemory() throws Exception { shard1Segment1CacheMemoryShare = ((double) (2 * shard1Queries) / ((2 * shard1Queries) + shard2Queries)) * (totalMemoryMinusOverhead) + shard1Overhead; cacheTotals = new IndicesQueryCache.CacheTotals(2 * shard1Queries + shard2Queries, 2); - sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(cache, indexShard1, cacheTotals); + sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(cache, shard1, cacheTotals); shard1CacheBytes = cache.getStats(shard1, sharedRamSizeShard1).getMemorySizeInBytes(); assertThat((double) shard1CacheBytes, closeTo(shard1Segment1CacheMemoryShare, 1)); // accounting for rounding @@ -594,8 +594,8 @@ public void testGetStatsMemory() throws Exception { shard2Searcher.count(new DummyQuery("ingest4-" + i, smallQuerySize)); } cacheTotals = new IndicesQueryCache.CacheTotals(maxCacheSize, 1); - sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(cache, indexShard1, cacheTotals); - sharedRamSizeShard2 = IndicesQueryCache.getSharedRamSizeForShard(cache, indexShard2, cacheTotals); + sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(cache, shard1, cacheTotals); + sharedRamSizeShard2 = IndicesQueryCache.getSharedRamSizeForShard(cache, shard2, cacheTotals); assertThat(cache.getStats(shard1, sharedRamSizeShard1).getMemorySizeInBytes(), equalTo(0L)); assertThat( cache.getStats(shard2, sharedRamSizeShard2).getMemorySizeInBytes(), From 641b940380463e3db716826353bc672267536d69 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Fri, 26 Sep 2025 18:00:48 -0400 Subject: [PATCH 23/34] This can be a final long --- .../src/main/java/org/elasticsearch/indices/IndicesService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index cd1b6cd801a69..4bf7782844196 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -551,7 +551,7 @@ IndexShardStats indexShardStats( final IndicesService indicesService, final IndexShard indexShard, final CommonStatsFlags flags, - Long precomputedSharedRam + final long precomputedSharedRam ) { if (indexShard.routingEntry() == null) { return null; From aae6e7faa96287d864ab8c65848040b7cc69e07a Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Fri, 26 Sep 2025 18:09:04 -0400 Subject: [PATCH 24/34] Fix a typo --- .../main/java/org/elasticsearch/indices/IndicesQueryCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index a1e206e06400b..dd259d0d8cb4d 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -269,7 +269,7 @@ QueryCacheStats toQueryCacheStats() { public String toString() { return "{shardId=" + shardId - + ", ramBytedUsed=" + + ", ramBytesUsed=" + ramBytesUsed + ", hitCount=" + hitCount From 556db6a91eb28db9cac11c3d5eda7fc7eb2b0c1c Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Fri, 26 Sep 2025 18:09:15 -0400 Subject: [PATCH 25/34] Fix a typo --- .../main/java/org/elasticsearch/indices/IndicesQueryCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index dd259d0d8cb4d..c8ccedfe7367f 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -383,7 +383,7 @@ protected void onDocIdSetEviction(Object readerCoreKey, int numEntries, long sum if (numEntries > 0) { // We can't use ShardCoreKeyMap here because its core closed // listener is called before the listener of the cache which - // triggers this eviction. So instead we use use stats2 that + // triggers this eviction. So instead we use stats2 that // we only evict when nothing is cached anymore on the segment // instead of relying on close listeners final StatsAndCount statsAndCount = stats2.get(readerCoreKey); From ce81caab6f8e1fe60e2a510135e1a63094ebd216 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Fri, 26 Sep 2025 18:09:29 -0400 Subject: [PATCH 26/34] Remove an unused constructor --- .../org/elasticsearch/indices/IndicesQueryCacheTests.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java index 6a04a491641e5..bd56cce994e49 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java @@ -61,10 +61,6 @@ private static class DummyQuery extends Query implements Accountable { this(Integer.toString(id), 10); } - DummyQuery(String id) { - this(id, 10); - } - DummyQuery(String id, long sizeInCache) { this.id = id; this.sizeInCache = sizeInCache; From be92c4632a82390af2c8b17c2fe36e8f84ea6d83 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Fri, 26 Sep 2025 18:09:48 -0400 Subject: [PATCH 27/34] Tidy up some Intellij gripes --- .../cluster/stats/TransportClusterStatsAction.java | 4 ++-- .../elasticsearch/indices/IndicesQueryCacheTests.java | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 98ab7fb5edd53..4ef4cdbdcd6b0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -319,7 +319,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq clusterStatus, nodeInfo, nodeStats, - shardsStats.toArray(new ShardStats[shardsStats.size()]), + shardsStats.toArray(new ShardStats[0]), searchUsageStats, repositoryUsageStats, ccsTelemetry, @@ -481,7 +481,7 @@ protected void sendItemRequest(String clusterAlias, ActionListener v.acceptResponse(response)); + remoteClustersStats.computeIfPresent(clusterAlias, (ignored, v) -> v.acceptResponse(response)); } } diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java index bd56cce994e49..3c09520cc5a56 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java @@ -87,10 +87,10 @@ public String toString(String field) { } @Override - public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) { return new ConstantScoreWeight(this, boost) { @Override - public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + public ScorerSupplier scorerSupplier(LeafReaderContext context) { Scorer scorer = new ConstantScoreScorer(score(), scoreMode, DocIdSetIterator.all(context.reader().maxDoc())); return new DefaultScorerSupplier(scorer); } @@ -421,7 +421,7 @@ public void testDelegatesScorerSupplier() throws Exception { assertNotSame(weight, cached); assertFalse(weight.scorerCalled); assertFalse(weight.scorerSupplierCalled); - cached.scorerSupplier(s.getIndexReader().leaves().get(0)); + cached.scorerSupplier(s.getIndexReader().leaves().getFirst()); assertFalse(weight.scorerCalled); assertTrue(weight.scorerSupplierCalled); IOUtils.close(r, dir); @@ -429,7 +429,7 @@ public void testDelegatesScorerSupplier() throws Exception { cache.close(); } - public void testGetCacheTotalsForAllShards() throws Exception { + public void testGetCacheTotalsForAllShards() { ShardId shardId1 = new ShardId("index", "_na_", 0); ShardId shardId2 = new ShardId("index", "_na_", 1); @@ -580,7 +580,7 @@ public void testGetStatsMemory() throws Exception { */ shard1Segment1CacheMemoryShare = ((double) (2 * shard1Queries) / ((2 * shard1Queries) + shard2Queries)) * (totalMemoryMinusOverhead) + shard1Overhead; - cacheTotals = new IndicesQueryCache.CacheTotals(2 * shard1Queries + shard2Queries, 2); + cacheTotals = new IndicesQueryCache.CacheTotals(2L * shard1Queries + shard2Queries, 2); sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(cache, shard1, cacheTotals); shard1CacheBytes = cache.getStats(shard1, sharedRamSizeShard1).getMemorySizeInBytes(); assertThat((double) shard1CacheBytes, closeTo(shard1Segment1CacheMemoryShare, 1)); // accounting for rounding From a2542ed045fab33f3185e00f23b407cba532c9fe Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Fri, 26 Sep 2025 18:13:57 -0400 Subject: [PATCH 28/34] Use computeIfAbsent for this --- .../java/org/elasticsearch/indices/IndicesQueryCache.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index c8ccedfe7367f..c9aedb3d881d8 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -366,11 +366,7 @@ protected void onDocIdSetCache(Object readerCoreKey, long ramBytesUsed) { shardStats.cacheCount += 1; shardStats.ramBytesUsed += ramBytesUsed; - StatsAndCount statsAndCount = stats2.get(readerCoreKey); - if (statsAndCount == null) { - statsAndCount = new StatsAndCount(shardStats); - stats2.put(readerCoreKey, statsAndCount); - } + StatsAndCount statsAndCount = stats2.computeIfAbsent(readerCoreKey, ignored -> new StatsAndCount(shardStats)); statsAndCount.count += 1; } From 7bf19d37ec06222988303515f985fb7d2a19578b Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 30 Sep 2025 16:50:29 -0500 Subject: [PATCH 29/34] pre-computing a map of shared ram for all shards --- .../stats/TransportClusterStatsAction.java | 9 +- .../stats/TransportIndicesStatsAction.java | 7 +- .../indices/IndicesQueryCache.java | 28 +++++- .../elasticsearch/indices/IndicesService.java | 8 +- .../indices/IndicesQueryCacheTests.java | 86 ++++++++----------- 5 files changed, 69 insertions(+), 69 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 4ef4cdbdcd6b0..02dcf127cfdd9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.seqno.RetentionLeaseStats; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesQueryCache; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.injection.guice.Inject; @@ -259,15 +260,11 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq false, false ); - IndicesQueryCache.CacheTotals cacheTotals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); + Map shardIdToSharedRam = IndicesQueryCache.getSharedRamForAllShards(indicesService); List shardsStats = new ArrayList<>(); for (IndexService indexService : indicesService) { for (IndexShard indexShard : indexService) { - long sharedRam = IndicesQueryCache.getSharedRamSizeForShard( - indicesService.getIndicesQueryCache(), - indexShard.shardId(), - cacheTotals - ); + long sharedRam = shardIdToSharedRam.get(indexShard.shardId()); cancellableTask.ensureNotCancelled(); if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) { // only report on fully started shards diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java index 270997186ecb5..ba59e3e5f1c0b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java @@ -113,14 +113,9 @@ protected IndicesStatsRequest readRequestFrom(StreamInput in) throws IOException protected void shardOperation(IndicesStatsRequest request, ShardRouting shardRouting, Task task, ActionListener listener) { ActionListener.completeWith(listener, () -> { assert task instanceof CancellableTask; - IndicesQueryCache.CacheTotals cacheTotals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex()); IndexShard indexShard = indexService.getShard(shardRouting.shardId().id()); - long sharedRam = IndicesQueryCache.getSharedRamSizeForShard( - indicesService.getIndicesQueryCache(), - indexShard.shardId(), - cacheTotals - ); + long sharedRam = IndicesQueryCache.getSharedRamSizeForShard(indicesService, indexShard.shardId()); CommonStats commonStats = CommonStats.getShardLevelStats( indicesService.getIndicesQueryCache(), indexShard, diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index c9aedb3d881d8..d011b34c31618 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -34,6 +34,7 @@ import java.io.Closeable; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.IdentityHashMap; import java.util.Map; import java.util.Set; @@ -69,6 +70,22 @@ public class IndicesQueryCache implements QueryCache, Closeable { private final Map shardStats = new ConcurrentHashMap<>(); private volatile long sharedRamBytesUsed; + public static Map getSharedRamForAllShards(IndicesService indicesService) { + Map shardIdToSharedRam = new HashMap<>(); + IndicesQueryCache.CacheTotals cacheTotals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); + for (IndexService indexService : indicesService) { + for (IndexShard indexShard : indexService) { + long sharedRam = IndicesQueryCache.getSharedRamSizeForShard( + indicesService.getIndicesQueryCache(), + indexShard.shardId(), + cacheTotals + ); + shardIdToSharedRam.put(indexShard.shardId(), sharedRam); + } + } + return shardIdToSharedRam; + } + public long getCacheSizeForShard(ShardId shardId) { Stats stats = shardStats.get(shardId); return stats != null ? stats.cacheSize : 0L; @@ -105,7 +122,7 @@ private static QueryCacheStats toQueryCacheStatsSafe(@Nullable Stats stats) { * @param indicesService * @return A CacheTotals object containing the computed total number of items in the cache and the number of shards seen in the cache */ - public static CacheTotals getCacheTotalsForAllShards(IndicesService indicesService) { + private static CacheTotals getCacheTotalsForAllShards(IndicesService indicesService) { IndicesQueryCache queryCache = indicesService.getIndicesQueryCache(); boolean hasQueryCache = queryCache != null; long totalItemsInCache = 0L; @@ -122,6 +139,11 @@ public static CacheTotals getCacheTotalsForAllShards(IndicesService indicesServi return new CacheTotals(totalItemsInCache, shardCount); } + public static long getSharedRamSizeForShard(IndicesService indicesService, ShardId shardId) { + IndicesQueryCache.CacheTotals cacheTotals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); + return getSharedRamSizeForShard(indicesService.getIndicesQueryCache(), shardId, cacheTotals); + } + /** * This method computes the shared RAM size in bytes for the given indexShard. * @param queryCache @@ -129,7 +151,7 @@ public static CacheTotals getCacheTotalsForAllShards(IndicesService indicesServi * @param cacheTotals Shard totals computed in getCacheTotalsForAllShards() * @return the shared RAM size in bytes allocated to the given shard, or 0 if unavailable */ - public static long getSharedRamSizeForShard(IndicesQueryCache queryCache, ShardId shardId, CacheTotals cacheTotals) { + private static long getSharedRamSizeForShard(IndicesQueryCache queryCache, ShardId shardId, CacheTotals cacheTotals) { long sharedRamBytesUsed = queryCache != null ? queryCache.getSharedRamBytesUsed() : 0L; if (sharedRamBytesUsed == 0L) { return 0L; @@ -167,7 +189,7 @@ public static long getSharedRamSizeForShard(IndicesQueryCache queryCache, ShardI return additionalRamBytesUsed; } - public record CacheTotals(long totalItemsInCache, int shardCount) {} + private record CacheTotals(long totalItemsInCache, int shardCount) {} /** Get usage statistics for the given shard. */ public QueryCacheStats getStats(ShardId shard, long precomputedSharedRamBytesUsed) { diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 4bf7782844196..790732296b00a 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -520,15 +520,11 @@ static Map statsByIndex(final IndicesService indicesService, } static Map> statsByShard(final IndicesService indicesService, final CommonStatsFlags flags) { - IndicesQueryCache.CacheTotals cacheTotals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); + Map shardIdToSharedRam = IndicesQueryCache.getSharedRamForAllShards(indicesService); final Map> statsByShard = new HashMap<>(); for (final IndexService indexService : indicesService) { for (final IndexShard indexShard : indexService) { - long sharedRam = IndicesQueryCache.getSharedRamSizeForShard( - indicesService.getIndicesQueryCache(), - indexShard.shardId(), - cacheTotals - ); + long sharedRam = shardIdToSharedRam.get(indexShard.shardId()); try { final IndexShardStats indexShardStats = indicesService.indexShardStats(indicesService, indexShard, flags, sharedRam); if (indexShardStats == null) { diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java index 3c09520cc5a56..bad8723e90073 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.cache.query.QueryCacheStats; import org.elasticsearch.index.cache.query.TrivialQueryCachingPolicy; @@ -46,7 +47,6 @@ import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThan; -import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -429,30 +429,6 @@ public void testDelegatesScorerSupplier() throws Exception { cache.close(); } - public void testGetCacheTotalsForAllShards() { - ShardId shardId1 = new ShardId("index", "_na_", 0); - ShardId shardId2 = new ShardId("index", "_na_", 1); - - IndexShard shard1 = mock(IndexShard.class); - IndexShard shard2 = mock(IndexShard.class); - when(shard1.shardId()).thenReturn(shardId1); - when(shard2.shardId()).thenReturn(shardId2); - - IndexService indexService = mock(IndexService.class, RETURNS_DEEP_STUBS); - when(indexService.iterator()).thenReturn(List.of(shard1, shard2).iterator()); - - IndicesService indicesService = mock(IndicesService.class, RETURNS_DEEP_STUBS); - when(indicesService.iterator()).thenReturn(List.of(indexService).iterator()); - IndicesQueryCache queryCache = mock(IndicesQueryCache.class); - when(indicesService.getIndicesQueryCache()).thenReturn(queryCache); - when(queryCache.getCacheSizeForShard(shardId1)).thenReturn(100L); - when(queryCache.getCacheSizeForShard(shardId2)).thenReturn(200L); - - IndicesQueryCache.CacheTotals totals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); - assertEquals(300L, totals.totalItemsInCache()); - assertEquals(2, totals.shardCount()); - } - public void testGetSharedRamSizeForShard() { ShardId shardId1 = new ShardId("index", "_na_", 0); ShardId shardId2 = new ShardId("index", "_na_", 1); @@ -461,27 +437,41 @@ public void testGetSharedRamSizeForShard() { when(shard1.shardId()).thenReturn(shardId1); when(shard2.shardId()).thenReturn(shardId2); - IndicesQueryCache.CacheTotals totals = new IndicesQueryCache.CacheTotals(300L, 2); - IndicesQueryCache queryCache = mock(IndicesQueryCache.class); // Case 1: sharedRamBytesUsed = 0 - when(queryCache.getSharedRamBytesUsed()).thenReturn(0L); - long sharedRam = IndicesQueryCache.getSharedRamSizeForShard(queryCache, shardId1, totals); + IndicesService indicesService = getTestIndicesService(List.of(Tuple.tuple(shard1, 100L), Tuple.tuple(shard2, 200L)), 0); + long sharedRam = IndicesQueryCache.getSharedRamSizeForShard(indicesService, shardId1); assertEquals(0L, sharedRam); // Case 2: sharedRamBytesUsed > 0, totalSize > 0, proportional - when(queryCache.getSharedRamBytesUsed()).thenReturn(600L); - when(queryCache.getCacheSizeForShard(shardId1)).thenReturn(100L); - long sharedRam1 = IndicesQueryCache.getSharedRamSizeForShard(queryCache, shardId1, totals); + indicesService = getTestIndicesService(List.of(Tuple.tuple(shard1, 100L), Tuple.tuple(shard2, 200L)), 600); + long sharedRam1 = IndicesQueryCache.getSharedRamSizeForShard(indicesService, shardId1); assertEquals(200L, sharedRam1); - when(queryCache.getCacheSizeForShard(shardId2)).thenReturn(200L); - long sharedRam2 = IndicesQueryCache.getSharedRamSizeForShard(queryCache, shardId2, totals); + long sharedRam2 = IndicesQueryCache.getSharedRamSizeForShard(indicesService, shardId2); assertEquals(400L, sharedRam2); // Case 3: totalSize == 0, shared equally - IndicesQueryCache.CacheTotals zeroTotals = new IndicesQueryCache.CacheTotals(0L, 2); - when(queryCache.getSharedRamBytesUsed()).thenReturn(600L); - long sharedRamEq = IndicesQueryCache.getSharedRamSizeForShard(queryCache, shardId1, zeroTotals); + indicesService = getTestIndicesService(List.of(Tuple.tuple(shard1, 0L), Tuple.tuple(shard2, 0L)), 600); + long sharedRamEq = IndicesQueryCache.getSharedRamSizeForShard(indicesService, shardId1); assertEquals(300L, sharedRamEq); } + private IndicesService getTestIndicesService(List> shardsAndSizes, long sharedRamBytesUsed) { + IndicesQueryCache queryCache = mock(IndicesQueryCache.class); + for (Tuple shardAndSize : shardsAndSizes) { + when(queryCache.getCacheSizeForShard(shardAndSize.v1().shardId())).thenReturn(shardAndSize.v2()); + } + when(queryCache.getSharedRamBytesUsed()).thenReturn(sharedRamBytesUsed); + return getTestIndicesService(shardsAndSizes.stream().map(Tuple::v1).toList(), queryCache); + } + + private IndicesService getTestIndicesService(List shards, IndicesQueryCache queryCache) { + IndexService indexService = mock(IndexService.class); + when(indexService.iterator()).thenAnswer(ignored -> shards.iterator()); + IndicesService indicesService = mock(IndicesService.class); + when(indicesService.iterator()).thenAnswer(ignored -> List.of(indexService).iterator()); + when(indicesService.getIndicesQueryCache()).thenReturn(queryCache); + when(indicesService.getIndicesQueryCache()).thenReturn(queryCache); + return indicesService; + } + public void testGetStatsMemory() throws Exception { /* * This test creates 2 shards, one with two segments and one with one. It makes unique queries against all 3 segments (so that each @@ -523,13 +513,13 @@ public void testGetStatsMemory() throws Exception { for (int i = 0; i < shard1Queries; ++i) { shard1Segment1Searcher.count(new DummyQuery("ingest1-" + i, largeQuerySize)); } - IndicesQueryCache.CacheTotals cacheTotals = new IndicesQueryCache.CacheTotals(shard1Queries, 1); IndexShard indexShard1 = mock(IndexShard.class); when(indexShard1.shardId()).thenReturn(shard1); IndexShard indexShard2 = mock(IndexShard.class); when(indexShard2.shardId()).thenReturn(shard2); - long sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(cache, shard1, cacheTotals); - long sharedRamSizeShard2 = IndicesQueryCache.getSharedRamSizeForShard(cache, shard2, cacheTotals); + IndicesService indicesService = getTestIndicesService(List.of(indexShard1), cache); + long sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(indicesService, shard1); + long sharedRamSizeShard2 = IndicesQueryCache.getSharedRamSizeForShard(indicesService, shard2); long shard1Segment1CacheMemory = calculateActualCacheMemoryForSegment(shard1Queries, largeQuerySize, shard1Segment1Docs); assertThat(cache.getStats(shard1, sharedRamSizeShard1).getMemorySizeInBytes(), equalTo(shard1Segment1CacheMemory)); assertThat(cache.getStats(shard2, sharedRamSizeShard2).getMemorySizeInBytes(), equalTo(0L)); @@ -541,9 +531,9 @@ public void testGetStatsMemory() throws Exception { * report cache memory proportional to the number of segments for each shard, ignoring the number of documents or the actual * document sizes. Since the shard2 requests were smaller, the average cache memory size per segment has now gone down. */ - cacheTotals = new IndicesQueryCache.CacheTotals(shard1Queries + shard2Queries, 2); - sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(cache, shard1, cacheTotals); - sharedRamSizeShard2 = IndicesQueryCache.getSharedRamSizeForShard(cache, shard2, cacheTotals); + indicesService = getTestIndicesService(List.of(indexShard1, indexShard2), cache); + sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(indicesService, shard1); + sharedRamSizeShard2 = IndicesQueryCache.getSharedRamSizeForShard(indicesService, shard2); assertThat(cache.getStats(shard1, sharedRamSizeShard1).getMemorySizeInBytes(), lessThan(shard1Segment1CacheMemory)); long shard1CacheBytes = cache.getStats(shard1, sharedRamSizeShard1).getMemorySizeInBytes(); long shard2CacheBytes = cache.getStats(shard2, sharedRamSizeShard2).getMemorySizeInBytes(); @@ -580,8 +570,8 @@ public void testGetStatsMemory() throws Exception { */ shard1Segment1CacheMemoryShare = ((double) (2 * shard1Queries) / ((2 * shard1Queries) + shard2Queries)) * (totalMemoryMinusOverhead) + shard1Overhead; - cacheTotals = new IndicesQueryCache.CacheTotals(2L * shard1Queries + shard2Queries, 2); - sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(cache, shard1, cacheTotals); + indicesService = getTestIndicesService(List.of(indexShard1, indexShard2), cache); + sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(indicesService, shard1); shard1CacheBytes = cache.getStats(shard1, sharedRamSizeShard1).getMemorySizeInBytes(); assertThat((double) shard1CacheBytes, closeTo(shard1Segment1CacheMemoryShare, 1)); // accounting for rounding @@ -589,9 +579,9 @@ public void testGetStatsMemory() throws Exception { for (int i = 0; i < (maxCacheSize * 2); ++i) { shard2Searcher.count(new DummyQuery("ingest4-" + i, smallQuerySize)); } - cacheTotals = new IndicesQueryCache.CacheTotals(maxCacheSize, 1); - sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(cache, shard1, cacheTotals); - sharedRamSizeShard2 = IndicesQueryCache.getSharedRamSizeForShard(cache, shard2, cacheTotals); + indicesService = getTestIndicesService(List.of(indexShard2), cache); + sharedRamSizeShard1 = IndicesQueryCache.getSharedRamSizeForShard(indicesService, shard1); + sharedRamSizeShard2 = IndicesQueryCache.getSharedRamSizeForShard(indicesService, shard2); assertThat(cache.getStats(shard1, sharedRamSizeShard1).getMemorySizeInBytes(), equalTo(0L)); assertThat( cache.getStats(shard2, sharedRamSizeShard2).getMemorySizeInBytes(), From ec57e0e9313c6994eb36ba03dd3c061f2a09e9fc Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 1 Oct 2025 17:30:56 -0400 Subject: [PATCH 30/34] Only map the non-zero values --- .../admin/cluster/stats/TransportClusterStatsAction.java | 3 ++- .../java/org/elasticsearch/indices/IndicesQueryCache.java | 7 +++++-- .../java/org/elasticsearch/indices/IndicesService.java | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 02dcf127cfdd9..ce6ff5bdbf784 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -264,7 +264,8 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq List shardsStats = new ArrayList<>(); for (IndexService indexService : indicesService) { for (IndexShard indexShard : indexService) { - long sharedRam = shardIdToSharedRam.get(indexShard.shardId()); + // get the shared ram for this shard id (or zero if there's nothing in the map) + long sharedRam = shardIdToSharedRam.getOrDefault(indexShard.shardId(), 0L); cancellableTask.ensureNotCancelled(); if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) { // only report on fully started shards diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index d011b34c31618..a29215e662e9d 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -80,10 +80,13 @@ public static Map getSharedRamForAllShards(IndicesService indices indexShard.shardId(), cacheTotals ); - shardIdToSharedRam.put(indexShard.shardId(), sharedRam); + // as a size optimization, only store non-zero values in the map + if (sharedRam > 0L) { + shardIdToSharedRam.put(indexShard.shardId(), sharedRam); + } } } - return shardIdToSharedRam; + return Collections.unmodifiableMap(shardIdToSharedRam); } public long getCacheSizeForShard(ShardId shardId) { diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 790732296b00a..2ec63bbd8656e 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -524,7 +524,8 @@ static Map> statsByShard(final IndicesService indic final Map> statsByShard = new HashMap<>(); for (final IndexService indexService : indicesService) { for (final IndexShard indexShard : indexService) { - long sharedRam = shardIdToSharedRam.get(indexShard.shardId()); + // get the shared ram for this shard id (or zero if there's nothing in the map) + long sharedRam = shardIdToSharedRam.getOrDefault(indexShard.shardId(), 0L); try { final IndexShardStats indexShardStats = indicesService.indexShardStats(indicesService, indexShard, flags, sharedRam); if (indexShardStats == null) { From 4a34321a13b405e472acd9fb578fc333453eb8cd Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Mon, 13 Oct 2025 14:51:52 -0400 Subject: [PATCH 31/34] Externalize the null check --- .../elasticsearch/indices/IndicesQueryCache.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index a29215e662e9d..2156efbfd6e79 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -75,11 +75,10 @@ public static Map getSharedRamForAllShards(IndicesService indices IndicesQueryCache.CacheTotals cacheTotals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); for (IndexService indexService : indicesService) { for (IndexShard indexShard : indexService) { - long sharedRam = IndicesQueryCache.getSharedRamSizeForShard( - indicesService.getIndicesQueryCache(), - indexShard.shardId(), - cacheTotals - ); + final var queryCache = indicesService.getIndicesQueryCache(); + long sharedRam = (queryCache == null) + ? 0L + : IndicesQueryCache.getSharedRamSizeForShard(queryCache, indexShard.shardId(), cacheTotals); // as a size optimization, only store non-zero values in the map if (sharedRam > 0L) { shardIdToSharedRam.put(indexShard.shardId(), sharedRam); @@ -144,7 +143,8 @@ private static CacheTotals getCacheTotalsForAllShards(IndicesService indicesServ public static long getSharedRamSizeForShard(IndicesService indicesService, ShardId shardId) { IndicesQueryCache.CacheTotals cacheTotals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); - return getSharedRamSizeForShard(indicesService.getIndicesQueryCache(), shardId, cacheTotals); + final var queryCache = indicesService.getIndicesQueryCache(); + return (queryCache == null) ? 0L : IndicesQueryCache.getSharedRamSizeForShard(queryCache, shardId, cacheTotals); } /** @@ -155,7 +155,7 @@ public static long getSharedRamSizeForShard(IndicesService indicesService, Shard * @return the shared RAM size in bytes allocated to the given shard, or 0 if unavailable */ private static long getSharedRamSizeForShard(IndicesQueryCache queryCache, ShardId shardId, CacheTotals cacheTotals) { - long sharedRamBytesUsed = queryCache != null ? queryCache.getSharedRamBytesUsed() : 0L; + long sharedRamBytesUsed = queryCache.getSharedRamBytesUsed(); if (sharedRamBytesUsed == 0L) { return 0L; } From a3ac3aa71ee446bda30fbdb5bef2a6021ec09c8a Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Mon, 13 Oct 2025 14:57:03 -0400 Subject: [PATCH 32/34] Make this method instance rather than static --- .../elasticsearch/indices/IndicesQueryCache.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index 2156efbfd6e79..f22523128ebe5 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -76,9 +76,7 @@ public static Map getSharedRamForAllShards(IndicesService indices for (IndexService indexService : indicesService) { for (IndexShard indexShard : indexService) { final var queryCache = indicesService.getIndicesQueryCache(); - long sharedRam = (queryCache == null) - ? 0L - : IndicesQueryCache.getSharedRamSizeForShard(queryCache, indexShard.shardId(), cacheTotals); + long sharedRam = (queryCache == null) ? 0L : queryCache.getSharedRamSizeForShard(indexShard.shardId(), cacheTotals); // as a size optimization, only store non-zero values in the map if (sharedRam > 0L) { shardIdToSharedRam.put(indexShard.shardId(), sharedRam); @@ -144,18 +142,17 @@ private static CacheTotals getCacheTotalsForAllShards(IndicesService indicesServ public static long getSharedRamSizeForShard(IndicesService indicesService, ShardId shardId) { IndicesQueryCache.CacheTotals cacheTotals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); final var queryCache = indicesService.getIndicesQueryCache(); - return (queryCache == null) ? 0L : IndicesQueryCache.getSharedRamSizeForShard(queryCache, shardId, cacheTotals); + return (queryCache == null) ? 0L : queryCache.getSharedRamSizeForShard(shardId, cacheTotals); } /** * This method computes the shared RAM size in bytes for the given indexShard. - * @param queryCache * @param shardId The shard to compute the shared RAM size for * @param cacheTotals Shard totals computed in getCacheTotalsForAllShards() * @return the shared RAM size in bytes allocated to the given shard, or 0 if unavailable */ - private static long getSharedRamSizeForShard(IndicesQueryCache queryCache, ShardId shardId, CacheTotals cacheTotals) { - long sharedRamBytesUsed = queryCache.getSharedRamBytesUsed(); + private long getSharedRamSizeForShard(ShardId shardId, CacheTotals cacheTotals) { + long sharedRamBytesUsed = getSharedRamBytesUsed(); if (sharedRamBytesUsed == 0L) { return 0L; } @@ -171,7 +168,7 @@ private static long getSharedRamSizeForShard(IndicesQueryCache queryCache, Shard * shard. */ long totalItemsInCache = cacheTotals.totalItemsInCache(); - long itemsInCacheForShard = queryCache.getCacheSizeForShard(shardId); + long itemsInCacheForShard = getCacheSizeForShard(shardId); final long additionalRamBytesUsed; if (totalItemsInCache == 0) { // all shards have zero cache footprint, so we apportion the size of the shared bytes equally across all shards From ccd7d2b6c15939813ae823f9b81f3c5100e50e35 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Mon, 13 Oct 2025 14:59:01 -0400 Subject: [PATCH 33/34] Rename this method --- .../action/admin/cluster/stats/TransportClusterStatsAction.java | 2 +- .../main/java/org/elasticsearch/indices/IndicesQueryCache.java | 2 +- .../src/main/java/org/elasticsearch/indices/IndicesService.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index ce6ff5bdbf784..b47fb0094dd31 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -260,7 +260,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq false, false ); - Map shardIdToSharedRam = IndicesQueryCache.getSharedRamForAllShards(indicesService); + Map shardIdToSharedRam = IndicesQueryCache.getSharedRamSizeForAllShards(indicesService); List shardsStats = new ArrayList<>(); for (IndexService indexService : indicesService) { for (IndexShard indexShard : indexService) { diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index f22523128ebe5..2664bd2b26f9b 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -70,7 +70,7 @@ public class IndicesQueryCache implements QueryCache, Closeable { private final Map shardStats = new ConcurrentHashMap<>(); private volatile long sharedRamBytesUsed; - public static Map getSharedRamForAllShards(IndicesService indicesService) { + public static Map getSharedRamSizeForAllShards(IndicesService indicesService) { Map shardIdToSharedRam = new HashMap<>(); IndicesQueryCache.CacheTotals cacheTotals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService); for (IndexService indexService : indicesService) { diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 2ec63bbd8656e..a74b9c6978fd8 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -520,7 +520,7 @@ static Map statsByIndex(final IndicesService indicesService, } static Map> statsByShard(final IndicesService indicesService, final CommonStatsFlags flags) { - Map shardIdToSharedRam = IndicesQueryCache.getSharedRamForAllShards(indicesService); + Map shardIdToSharedRam = IndicesQueryCache.getSharedRamSizeForAllShards(indicesService); final Map> statsByShard = new HashMap<>(); for (final IndexService indexService : indicesService) { for (final IndexShard indexShard : indexService) { From 81bfdbde6e888b58ef93bfa36c776c2632877f76 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Mon, 13 Oct 2025 15:05:35 -0400 Subject: [PATCH 34/34] Add a javadoc, as promised --- .../java/org/elasticsearch/indices/IndicesQueryCache.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index 2664bd2b26f9b..11af6b2e2d5ba 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -70,6 +70,13 @@ public class IndicesQueryCache implements QueryCache, Closeable { private final Map shardStats = new ConcurrentHashMap<>(); private volatile long sharedRamBytesUsed; + /** + * Calculates a map of {@link ShardId} to {@link Long} which contains the calculated share of the {@link IndicesQueryCache} shared ram + * size for a given shard (that is, the sum of all the longs is the size of the indices query cache). Since many shards will not + * participate in the cache, shards whose calculated share is zero will not be contained in the map at all. As a consequence, the + * correct pattern for using the returned map will be via {@link Map#getOrDefault(Object, Object)} with a {@code defaultValue} of + * {@code 0L}. + */ public static Map getSharedRamSizeForAllShards(IndicesService indicesService) { Map shardIdToSharedRam = new HashMap<>(); IndicesQueryCache.CacheTotals cacheTotals = IndicesQueryCache.getCacheTotalsForAllShards(indicesService);