Skip to content

Commit efc3c24

Browse files
tqranjantqranjan
committed
Enhance file cache statistics API with miss_count and removed_in_bytes fields
Add cache miss tracking and explicit removal tracking to file cache statistics: Add miss_count field to all nested statistics objects for hit ratio analysis Add removed_in_bytes field to distinguish manual removals from automatic evictions Update FileCacheStats with getRemoved() method and enhanced JSON serialization Update AggregateFileCacheStats with aggregate JSON serialization for new fields Update FileCache fileCacheStats() method to pass removal data Add comprehensive test coverage including MissCountValidationTest Update all FileCacheStats constructor calls to include removed parameter Enables hit ratio calculation, cache pressure monitoring, and performance debugging across different file types in the cache hierarchy. Signed-off-by: tanishq ranjan <[email protected]>
1 parent c9a5fab commit efc3c24

File tree

10 files changed

+63
-21
lines changed

10 files changed

+63
-21
lines changed

‎server/src/main/java/org/opensearch/index/store/remote/filecache/AggregateFileCacheStats.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,10 @@ public ByteSizeValue getEvicted() {
116116
return new ByteSizeValue(overallFileCacheStats.getEvicted());
117117
}
118118

119+
public ByteSizeValue getRemoved() {
120+
return new ByteSizeValue(overallFileCacheStats.getRemoved());
121+
}
122+
119123
public long getCacheHits() {
120124
return overallFileCacheStats.getCacheHits();
121125
}
@@ -138,6 +142,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
138142
builder.humanReadableField(Fields.USED_IN_BYTES, Fields.USED, getUsed());
139143
builder.humanReadableField(Fields.PINNED_IN_BYTES, Fields.PINNED, getPinnedUsage());
140144
builder.humanReadableField(Fields.EVICTIONS_IN_BYTES, Fields.EVICTIONS, getEvicted());
145+
builder.humanReadableField(Fields.REMOVED_IN_BYTES, Fields.REMOVED, getRemoved());
141146
builder.field(Fields.ACTIVE_PERCENT, getActivePercent());
142147
builder.field(Fields.USED_PERCENT, getUsedPercent());
143148
builder.field(Fields.HIT_COUNT, getCacheHits());
@@ -161,6 +166,8 @@ static final class Fields {
161166
static final String PINNED_IN_BYTES = "pinned_in_bytes";
162167
static final String EVICTIONS = "evictions";
163168
static final String EVICTIONS_IN_BYTES = "evictions_in_bytes";
169+
static final String REMOVED = "removed";
170+
static final String REMOVED_IN_BYTES = "removed_in_bytes";
164171
static final String TOTAL = "total";
165172
static final String TOTAL_IN_BYTES = "total_in_bytes";
166173

‎server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -257,6 +257,7 @@ public AggregateFileCacheStats fileCacheStats() {
257257
overallCacheStats.usage(),
258258
overallCacheStats.pinnedUsage(),
259259
overallCacheStats.evictionWeight(),
260+
overallCacheStats.removeWeight(),
260261
overallCacheStats.hitCount(),
261262
overallCacheStats.missCount(),
262263
FileCacheStatsType.OVER_ALL_STATS
@@ -267,6 +268,7 @@ public AggregateFileCacheStats fileCacheStats() {
267268
fullFileCacheStats.usage(),
268269
fullFileCacheStats.pinnedUsage(),
269270
fullFileCacheStats.evictionWeight(),
271+
fullFileCacheStats.removeWeight(),
270272
fullFileCacheStats.hitCount(),
271273
fullFileCacheStats.missCount(),
272274
FileCacheStatsType.FULL_FILE_STATS
@@ -277,6 +279,7 @@ public AggregateFileCacheStats fileCacheStats() {
277279
blockFileCacheStats.usage(),
278280
blockFileCacheStats.pinnedUsage(),
279281
blockFileCacheStats.evictionWeight(),
282+
blockFileCacheStats.removeWeight(),
280283
blockFileCacheStats.hitCount(),
281284
blockFileCacheStats.missCount(),
282285
FileCacheStatsType.BLOCK_FILE_STATS
@@ -287,6 +290,7 @@ public AggregateFileCacheStats fileCacheStats() {
287290
pinnedFileCacheStats.usage(),
288291
pinnedFileCacheStats.pinnedUsage(),
289292
pinnedFileCacheStats.evictionWeight(),
293+
pinnedFileCacheStats.removeWeight(),
290294
pinnedFileCacheStats.hitCount(),
291295
pinnedFileCacheStats.missCount(),
292296
FileCacheStatsType.PINNED_FILE_STATS

‎server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ public class FileCacheStats implements Writeable, ToXContentFragment {
4141
private final long used;
4242
private final long pinned;
4343
private final long evicted;
44+
private final long removed;
4445
private final long hits;
4546
private final long misses;
4647
private final FileCacheStatsType statsType;
@@ -52,6 +53,7 @@ public FileCacheStats(
5253
final long used,
5354
final long pinned,
5455
final long evicted,
56+
final long removed,
5557
final long hits,
5658
long misses,
5759
FileCacheStatsType statsType
@@ -61,6 +63,7 @@ public FileCacheStats(
6163
this.used = used;
6264
this.pinned = pinned;
6365
this.evicted = evicted;
66+
this.removed = removed;
6467
this.hits = hits;
6568
this.misses = misses;
6669
this.statsType = statsType;
@@ -74,6 +77,7 @@ public FileCacheStats(final StreamInput in) throws IOException {
7477
this.used = in.readLong();
7578
this.pinned = in.readLong();
7679
this.evicted = in.readLong();
80+
this.removed = in.readLong();
7781
this.hits = in.readLong();
7882
this.misses = in.readLong();
7983
}
@@ -86,6 +90,7 @@ public void writeTo(final StreamOutput out) throws IOException {
8690
out.writeLong(used);
8791
out.writeLong(pinned);
8892
out.writeLong(evicted);
93+
out.writeLong(removed);
8994
out.writeLong(hits);
9095
out.writeLong(misses);
9196
}
@@ -102,6 +107,10 @@ public long getEvicted() {
102107
return evicted;
103108
}
104109

110+
public long getRemoved() {
111+
return removed;
112+
}
113+
105114
public long getHits() {
106115
return hits;
107116
}
@@ -138,8 +147,11 @@ static final class Fields {
138147
static final String USED_IN_BYTES = "used_in_bytes";
139148
static final String EVICTIONS = "evictions";
140149
static final String EVICTIONS_IN_BYTES = "evictions_in_bytes";
150+
static final String REMOVED = "removed";
151+
static final String REMOVED_IN_BYTES = "removed_in_bytes";
141152
static final String ACTIVE_PERCENT = "active_percent";
142153
static final String HIT_COUNT = "hit_count";
154+
static final String MISS_COUNT = "miss_count";
143155
}
144156

145157
@Override
@@ -153,8 +165,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
153165
FileCacheStats.Fields.EVICTIONS,
154166
new ByteSizeValue(getEvicted())
155167
);
168+
builder.humanReadableField(FileCacheStats.Fields.REMOVED_IN_BYTES, FileCacheStats.Fields.REMOVED, new ByteSizeValue(getRemoved()));
156169
builder.field(FileCacheStats.Fields.ACTIVE_PERCENT, getActivePercent());
157170
builder.field(FileCacheStats.Fields.HIT_COUNT, getHits());
171+
builder.field(FileCacheStats.Fields.MISS_COUNT, getCacheMisses());
158172
builder.endObject();
159173
return builder;
160174
}

‎server/src/test/java/org/opensearch/cluster/ClusterInfoTests.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ private static Map<String, AggregateFileCacheStats> randomFileCacheStats(int num
187187
randomLong(),
188188
randomLong(),
189189
randomLong(),
190+
randomLong(),
190191
FileCacheStatsType.OVER_ALL_STATS
191192
),
192193
new FileCacheStats(
@@ -197,6 +198,7 @@ private static Map<String, AggregateFileCacheStats> randomFileCacheStats(int num
197198
randomLong(),
198199
randomLong(),
199200
randomLong(),
201+
randomLong(),
200202
FileCacheStatsType.FULL_FILE_STATS
201203
),
202204
new FileCacheStats(
@@ -207,6 +209,7 @@ private static Map<String, AggregateFileCacheStats> randomFileCacheStats(int num
207209
randomLong(),
208210
randomLong(),
209211
randomLong(),
212+
randomLong(),
210213
FileCacheStatsType.BLOCK_FILE_STATS
211214
),
212215
new FileCacheStats(
@@ -217,6 +220,7 @@ private static Map<String, AggregateFileCacheStats> randomFileCacheStats(int num
217220
randomLong(),
218221
randomLong(),
219222
randomLong(),
223+
randomLong(),
220224
FileCacheStatsType.PINNED_FILE_STATS
221225
)
222226
);

‎server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitorTests.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1411,6 +1411,7 @@ private static AggregateFileCacheStats createAggregateFileCacheStats(long totalC
14111411
0,
14121412
0,
14131413
0,
1414+
0,
14141415
AggregateFileCacheStats.FileCacheStatsType.OVER_ALL_STATS
14151416
);
14161417
FileCacheStats fullStats = new FileCacheStats(
@@ -1421,6 +1422,7 @@ private static AggregateFileCacheStats createAggregateFileCacheStats(long totalC
14211422
0,
14221423
0,
14231424
0,
1425+
0,
14241426
AggregateFileCacheStats.FileCacheStatsType.FULL_FILE_STATS
14251427
);
14261428
FileCacheStats blockStats = new FileCacheStats(
@@ -1431,6 +1433,7 @@ private static AggregateFileCacheStats createAggregateFileCacheStats(long totalC
14311433
0,
14321434
0,
14331435
0,
1436+
0,
14341437
AggregateFileCacheStats.FileCacheStatsType.BLOCK_FILE_STATS
14351438
);
14361439
FileCacheStats pinnedStats = new FileCacheStats(
@@ -1441,6 +1444,7 @@ private static AggregateFileCacheStats createAggregateFileCacheStats(long totalC
14411444
0,
14421445
0,
14431446
0,
1447+
0,
14441448
AggregateFileCacheStats.FileCacheStatsType.PINNED_FILE_STATS
14451449
);
14461450
return new AggregateFileCacheStats(System.currentTimeMillis(), overallStats, fullStats, blockStats, pinnedStats);

‎server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -308,30 +308,30 @@ public void testDiskThresholdForRemoteShards() {
308308
"node1",
309309
new AggregateFileCacheStats(
310310
0,
311-
new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS),
312-
new FileCacheStats(0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS),
313-
new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS),
314-
new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS)
311+
new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS),
312+
new FileCacheStats(0, 0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS),
313+
new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS),
314+
new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS)
315315
)
316316
);
317317
fileCacheStatsMap.put(
318318
"node2",
319319
new AggregateFileCacheStats(
320320
0,
321-
new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS),
322-
new FileCacheStats(0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS),
323-
new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS),
324-
new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS)
321+
new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS),
322+
new FileCacheStats(0, 0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS),
323+
new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS),
324+
new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS)
325325
)
326326
);
327327
fileCacheStatsMap.put(
328328
"node3",
329329
new AggregateFileCacheStats(
330330
0,
331-
new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS),
332-
new FileCacheStats(0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS),
333-
new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS),
334-
new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS)
331+
new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS),
332+
new FileCacheStats(0, 0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS),
333+
new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS),
334+
new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS)
335335

336336
)
337337
);

‎server/src/test/java/org/opensearch/cluster/routing/allocation/decider/WarmDiskThresholdDeciderTests.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -464,10 +464,10 @@ private Map<String, AggregateFileCacheStats> createFileCacheStatsMap(long fileCa
464464
node,
465465
new AggregateFileCacheStats(
466466
randomNonNegativeInt(),
467-
new FileCacheStats(0, fileCacheSize, 0, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS),
468-
new FileCacheStats(0, fileCacheSize, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS),
469-
new FileCacheStats(0, fileCacheSize, 0, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS),
470-
new FileCacheStats(0, fileCacheSize, 0, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS)
467+
new FileCacheStats(0, fileCacheSize, 0, 0, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS),
468+
new FileCacheStats(0, fileCacheSize, 0, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS),
469+
new FileCacheStats(0, fileCacheSize, 0, 0, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS),
470+
new FileCacheStats(0, fileCacheSize, 0, 0, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS)
471471
)
472472
);
473473
}

‎server/src/test/java/org/opensearch/index/store/remote/filecache/AggregateFileCacheStatsTests.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ public static AggregateFileCacheStats getFileCacheStats(final long fileCacheCapa
9898
stats.usage(),
9999
stats.pinnedUsage(),
100100
stats.evictionWeight(),
101+
stats.removeWeight(),
101102
stats.hitCount(),
102103
stats.missCount(),
103104
FileCacheStatsType.OVER_ALL_STATS
@@ -108,6 +109,7 @@ public static AggregateFileCacheStats getFileCacheStats(final long fileCacheCapa
108109
stats.usage(),
109110
stats.pinnedUsage(),
110111
stats.evictionWeight(),
112+
stats.removeWeight(),
111113
stats.hitCount(),
112114
stats.missCount(),
113115
FileCacheStatsType.FULL_FILE_STATS
@@ -118,6 +120,7 @@ public static AggregateFileCacheStats getFileCacheStats(final long fileCacheCapa
118120
stats.usage(),
119121
stats.pinnedUsage(),
120122
stats.evictionWeight(),
123+
stats.removeWeight(),
121124
stats.hitCount(),
122125
stats.missCount(),
123126
FileCacheStatsType.BLOCK_FILE_STATS
@@ -128,6 +131,7 @@ public static AggregateFileCacheStats getFileCacheStats(final long fileCacheCapa
128131
stats.usage(),
129132
stats.pinnedUsage(),
130133
stats.evictionWeight(),
134+
stats.removeWeight(),
131135
stats.hitCount(),
132136
stats.missCount(),
133137
FileCacheStatsType.PINNED_FILE_STATS
@@ -141,9 +145,10 @@ public static FileCacheStats getMockFullFileCacheStats() {
141145
final long used = randomLongBetween(100000, BYTES_IN_GB);
142146
final long pinned = randomLongBetween(100000, BYTES_IN_GB);
143147
final long evicted = randomLongBetween(0, getMockCacheStats().getFullFileCacheStats().evictionWeight());
148+
final long removed = randomLongBetween(0, 10);
144149
final long hit = randomLongBetween(0, 10);
145150
final long misses = randomLongBetween(0, 10);
146-
return new FileCacheStats(active, total, used, pinned, evicted, hit, misses, FileCacheStatsType.OVER_ALL_STATS);
151+
return new FileCacheStats(active, total, used, pinned, evicted, removed, hit, misses, FileCacheStatsType.OVER_ALL_STATS);
147152
}
148153

149154
public static AggregateFileCacheStats getMockFileCacheStats() throws IOException {

‎server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheStatsTests.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ public static FileCacheStats getMockFullFileCacheStats() {
2424
final long used = randomLongBetween(100000, BYTES_IN_GB);
2525
final long pinned = randomLongBetween(100000, BYTES_IN_GB);
2626
final long evicted = randomLongBetween(0, active);
27+
final long removed = randomLongBetween(0, 10);
2728
final long hits = randomLongBetween(0, 10);
2829
final long misses = randomLongBetween(0, 10);
2930

@@ -33,6 +34,7 @@ public static FileCacheStats getMockFullFileCacheStats() {
3334
used,
3435
pinned,
3536
evicted,
37+
removed,
3638
hits,
3739
misses,
3840
AggregateFileCacheStats.FileCacheStatsType.OVER_ALL_STATS
@@ -43,7 +45,9 @@ public static void validateFullFileCacheStats(FileCacheStats expected, FileCache
4345
assertEquals(expected.getActive(), actual.getActive());
4446
assertEquals(expected.getUsed(), actual.getUsed());
4547
assertEquals(expected.getEvicted(), actual.getEvicted());
48+
assertEquals(expected.getRemoved(), actual.getRemoved());
4649
assertEquals(expected.getHits(), actual.getHits());
50+
assertEquals(expected.getCacheMisses(), actual.getCacheMisses());
4751
assertEquals(expected.getActivePercent(), actual.getActivePercent());
4852
}
4953

‎server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -469,10 +469,10 @@ public void testSearchableSnapshotOverSubscription() {
469469
node.node.getId(),
470470
new AggregateFileCacheStats(
471471
0,
472-
new FileCacheStats(1, 0, 0, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS),
473-
new FileCacheStats(0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS),
474-
new FileCacheStats(1, 0, 0, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS),
475-
new FileCacheStats(1, 0, 0, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS)
472+
new FileCacheStats(1, 0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS),
473+
new FileCacheStats(0, 0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS),
474+
new FileCacheStats(1, 0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS),
475+
new FileCacheStats(1, 0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS)
476476
)
477477
);
478478
}

0 commit comments

Comments
 (0)