Skip to content

Commit 4455f70

Browse files
masseykemasseyke
committed
removing all uses of N^2 code
1 parent c8fc29e commit 4455f70

File tree

10 files changed

+112
-107
lines changed

10 files changed

+112
-107
lines changed

‎server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@
4848
import org.elasticsearch.index.seqno.RetentionLeaseStats;
4949
import org.elasticsearch.index.seqno.SeqNoStats;
5050
import org.elasticsearch.index.shard.IndexShard;
51+
import org.elasticsearch.index.shard.ShardId;
52+
import org.elasticsearch.indices.IndicesQueryCache;
5153
import org.elasticsearch.indices.IndicesService;
5254
import org.elasticsearch.injection.guice.Inject;
5355
import org.elasticsearch.node.NodeService;
@@ -257,9 +259,39 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
257259
false,
258260
false
259261
);
262+
263+
IndicesQueryCache queryCache = indicesService.getIndicesQueryCache();
264+
boolean hasQueryCache = queryCache != null;
265+
// First pass: gather all shards, cache sizes, and compute totals
266+
long totalSize = 0L;
267+
int shardCount = 0;
268+
boolean anyNonZero = false;
269+
// First pass: compute totals only
270+
for (final IndexService indexService : indicesService) {
271+
for (final IndexShard indexShard : indexService) {
272+
long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(indexShard.shardId()) : 0L;
273+
shardCount++;
274+
if (cacheSize > 0L) {
275+
anyNonZero = true;
276+
totalSize += cacheSize;
277+
}
278+
}
279+
}
280+
long sharedRamBytesUsed = hasQueryCache ? queryCache.getSharedRamBytesUsed() : 0L;
281+
260282
List<ShardStats> shardsStats = new ArrayList<>();
261283
for (IndexService indexService : indicesService) {
262284
for (IndexShard indexShard : indexService) {
285+
ShardId shardId = indexShard.shardId();
286+
long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(shardId) : 0L;
287+
long sharedRam = 0L;
288+
if (sharedRamBytesUsed != 0L) {
289+
if (anyNonZero == false) {
290+
sharedRam = Math.round((double) sharedRamBytesUsed / shardCount);
291+
} else if (totalSize != 0) {
292+
sharedRam = Math.round((double) sharedRamBytesUsed * cacheSize / totalSize);
293+
}
294+
}
263295
cancellableTask.ensureNotCancelled();
264296
if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) {
265297
// only report on fully started shards
@@ -280,7 +312,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
280312
new ShardStats(
281313
indexShard.routingEntry(),
282314
indexShard.shardPath(),
283-
CommonStats.getShardLevelStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS),
315+
CommonStats.getShardLevelStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS, sharedRam),
284316
commitStats,
285317
seqNoStats,
286318
retentionLeaseStats,

‎server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -154,18 +154,11 @@ public CommonStats(CommonStatsFlags flags) {
154154
/**
155155
* Filters the given flags for {@link CommonStatsFlags#SHARD_LEVEL} flags and calculates the corresponding statistics.
156156
*/
157-
public static CommonStats getShardLevelStats(IndicesQueryCache indicesQueryCache, IndexShard indexShard, CommonStatsFlags flags) {
158-
return getShardLevelStats(indicesQueryCache, indexShard, flags, null);
159-
}
160-
161-
/**
162-
* Overload that takes a precomputed shared RAM map for O(N) stats.
163-
*/
164157
public static CommonStats getShardLevelStats(
165158
IndicesQueryCache indicesQueryCache,
166159
IndexShard indexShard,
167160
CommonStatsFlags flags,
168-
Long precomputedSharedRam
161+
long precomputedSharedRam
169162
) {
170163
// Filter shard level flags
171164
CommonStatsFlags filteredFlags = flags.clone();
@@ -186,13 +179,7 @@ public static CommonStats getShardLevelStats(
186179
case Refresh -> stats.refresh = indexShard.refreshStats();
187180
case Flush -> stats.flush = indexShard.flushStats();
188181
case Warmer -> stats.warmer = indexShard.warmerStats();
189-
case QueryCache -> {
190-
if (precomputedSharedRam != null) {
191-
stats.queryCache = indicesQueryCache.getStats(indexShard.shardId(), precomputedSharedRam);
192-
} else {
193-
stats.queryCache = indicesQueryCache.getStats(indexShard.shardId());
194-
}
195-
}
182+
case QueryCache -> stats.queryCache = indicesQueryCache.getStats(indexShard.shardId(), 0L);
196183
case FieldData -> stats.fieldData = indexShard.fieldDataStats(flags.fieldDataFields());
197184
case Completion -> stats.completion = indexShard.completionStats(flags.completionDataFields());
198185
case Segments -> stats.segments = indexShard.segmentStats(

‎server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727
import org.elasticsearch.index.seqno.RetentionLeaseStats;
2828
import org.elasticsearch.index.seqno.SeqNoStats;
2929
import org.elasticsearch.index.shard.IndexShard;
30+
import org.elasticsearch.index.shard.ShardId;
31+
import org.elasticsearch.indices.IndicesQueryCache;
3032
import org.elasticsearch.indices.IndicesService;
3133
import org.elasticsearch.injection.guice.Inject;
3234
import org.elasticsearch.tasks.CancellableTask;
@@ -112,9 +114,45 @@ protected IndicesStatsRequest readRequestFrom(StreamInput in) throws IOException
112114
protected void shardOperation(IndicesStatsRequest request, ShardRouting shardRouting, Task task, ActionListener<ShardStats> listener) {
113115
ActionListener.completeWith(listener, () -> {
114116
assert task instanceof CancellableTask;
117+
IndicesQueryCache queryCache = indicesService.getIndicesQueryCache();
118+
boolean hasQueryCache = queryCache != null;
119+
120+
// First pass: gather all shards, cache sizes, and compute totals
121+
long totalSize = 0L;
122+
int shardCount = 0;
123+
boolean anyNonZero = false;
124+
// First pass: compute totals only
125+
for (final IndexService indexService : indicesService) {
126+
for (final IndexShard indexShard : indexService) {
127+
long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(indexShard.shardId()) : 0L;
128+
shardCount++;
129+
if (cacheSize > 0L) {
130+
anyNonZero = true;
131+
totalSize += cacheSize;
132+
}
133+
}
134+
}
135+
long sharedRamBytesUsed = hasQueryCache ? queryCache.getSharedRamBytesUsed() : 0L;
136+
115137
IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex());
116138
IndexShard indexShard = indexService.getShard(shardRouting.shardId().id());
117-
CommonStats commonStats = CommonStats.getShardLevelStats(indicesService.getIndicesQueryCache(), indexShard, request.flags());
139+
ShardId shardId = indexShard.shardId();
140+
long cacheSize = hasQueryCache ? queryCache.getCacheSizeForShard(shardId) : 0L;
141+
long sharedRam = 0L;
142+
if (sharedRamBytesUsed != 0L) {
143+
if (anyNonZero == false) {
144+
sharedRam = Math.round((double) sharedRamBytesUsed / shardCount);
145+
} else if (totalSize != 0) {
146+
sharedRam = Math.round((double) sharedRamBytesUsed * cacheSize / totalSize);
147+
}
148+
}
149+
150+
CommonStats commonStats = CommonStats.getShardLevelStats(
151+
indicesService.getIndicesQueryCache(),
152+
indexShard,
153+
request.flags(),
154+
sharedRam
155+
);
118156
CommitStats commitStats;
119157
SeqNoStats seqNoStats;
120158
RetentionLeaseStats retentionLeaseStats;

‎server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java‎

Lines changed: 2 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -68,13 +68,13 @@ public class IndicesQueryCache implements QueryCache, Closeable {
6868
private volatile long sharedRamBytesUsed;
6969

7070
// Package-private for IndicesService efficient stats collection
71-
long getCacheSizeForShard(ShardId shardId) {
71+
public long getCacheSizeForShard(ShardId shardId) {
7272
Stats stats = shardStats.get(shardId);
7373
return stats != null ? stats.cacheSize : 0L;
7474
}
7575

7676
// Package-private for IndicesService efficient stats collection
77-
long getSharedRamBytesUsed() {
77+
public long getSharedRamBytesUsed() {
7878
return sharedRamBytesUsed;
7979
}
8080

@@ -100,59 +100,7 @@ private static QueryCacheStats toQueryCacheStatsSafe(@Nullable Stats stats) {
100100
return stats == null ? new QueryCacheStats() : stats.toQueryCacheStats();
101101
}
102102

103-
private long getShareOfAdditionalRamBytesUsed(long cacheSize) {
104-
if (sharedRamBytesUsed == 0L) {
105-
return 0L;
106-
}
107-
108-
// We also have some shared ram usage that we try to distribute proportionally to the cache footprint of each shard.
109-
// TODO avoid looping over all local shards here - see https://github.com/elastic/elasticsearch/issues/97222
110-
long totalSize = 0L;
111-
int shardCount = 0;
112-
if (cacheSize == 0L) {
113-
for (final var stats : shardStats.values()) {
114-
shardCount += 1;
115-
if (stats.cacheSize > 0L) {
116-
// some shard has nonzero cache footprint, so we apportion the shared size by cache footprint, and this shard has none
117-
return 0L;
118-
}
119-
}
120-
} else {
121-
// branchless loop for the common case
122-
for (final var stats : shardStats.values()) {
123-
shardCount += 1;
124-
totalSize += stats.cacheSize;
125-
}
126-
}
127-
128-
if (shardCount == 0) {
129-
// Sometimes it's not possible to do this when there are no shard entries at all, which can happen as the shared ram usage can
130-
// extend beyond the closing of all shards.
131-
return 0L;
132-
}
133-
134-
final long additionalRamBytesUsed;
135-
if (totalSize == 0) {
136-
// all shards have zero cache footprint, so we apportion the size of the shared bytes equally across all shards
137-
additionalRamBytesUsed = Math.round((double) sharedRamBytesUsed / shardCount);
138-
} else {
139-
// some shards have nonzero cache footprint, so we apportion the size of the shared bytes proportionally to cache footprint
140-
additionalRamBytesUsed = Math.round((double) sharedRamBytesUsed * cacheSize / totalSize);
141-
}
142-
assert additionalRamBytesUsed >= 0L : additionalRamBytesUsed;
143-
return additionalRamBytesUsed;
144-
}
145-
146103
/** Get usage statistics for the given shard. */
147-
public QueryCacheStats getStats(ShardId shard) {
148-
final QueryCacheStats queryCacheStats = toQueryCacheStatsSafe(shardStats.get(shard));
149-
queryCacheStats.addRamBytesUsed(getShareOfAdditionalRamBytesUsed(queryCacheStats.getCacheSize()));
150-
return queryCacheStats;
151-
}
152-
153-
/**
154-
* Overload to allow passing in a precomputed shared RAM split for this shard.
155-
*/
156104
public QueryCacheStats getStats(ShardId shard, long precomputedSharedRamBytesUsed) {
157105
final QueryCacheStats queryCacheStats = toQueryCacheStatsSafe(shardStats.get(shard));
158106
queryCacheStats.addRamBytesUsed(precomputedSharedRamBytesUsed);

‎server/src/main/java/org/elasticsearch/indices/IndicesService.java‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -552,7 +552,7 @@ static Map<Index, List<IndexShardStats>> statsByShard(final IndicesService indic
552552
}
553553
}
554554
try {
555-
final IndexShardStats indexShardStats = indicesService.indexShardStats(indicesService, indexShard, flags);
555+
final IndexShardStats indexShardStats = indicesService.indexShardStats(indicesService, indexShard, flags, sharedRam);
556556
if (indexShardStats == null) {
557557
continue;
558558
}
@@ -569,11 +569,6 @@ static Map<Index, List<IndexShardStats>> statsByShard(final IndicesService indic
569569
return statsByShard;
570570
}
571571

572-
IndexShardStats indexShardStats(final IndicesService indicesService, final IndexShard indexShard, final CommonStatsFlags flags) {
573-
// Default to 0L for precomputedSharedRam for backward compatibility
574-
return indexShardStats(indicesService, indexShard, flags, 0L);
575-
}
576-
577572
IndexShardStats indexShardStats(
578573
final IndicesService indicesService,
579574
final IndexShard indexShard,

‎server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ public void testCreation() {
115115
ShardStats shardStats = new ShardStats(
116116
shardRouting,
117117
new ShardPath(false, path, path, shardRouting.shardId()),
118-
CommonStats.getShardLevelStats(null, indexShard, new CommonStatsFlags(CommonStatsFlags.Flag.Store)),
118+
CommonStats.getShardLevelStats(null, indexShard, new CommonStatsFlags(CommonStatsFlags.Flag.Store), 0L),
119119
null,
120120
null,
121121
null,

‎server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1624,7 +1624,7 @@ public void testShardStats() throws IOException {
16241624
ShardStats stats = new ShardStats(
16251625
shard.routingEntry(),
16261626
shard.shardPath(),
1627-
CommonStats.getShardLevelStats(new IndicesQueryCache(Settings.EMPTY), shard, new CommonStatsFlags()),
1627+
CommonStats.getShardLevelStats(new IndicesQueryCache(Settings.EMPTY), shard, new CommonStatsFlags(), 0L),
16281628
shard.commitStats(),
16291629
shard.seqNoStats(),
16301630
shard.getRetentionLeaseStats(),

0 commit comments

Comments
 (0)