diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java index e72a78e187f67..dffc75002c877 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java @@ -756,6 +756,7 @@ private static AggregateFileCacheStats setAggregateFileCacheStats(long active, l 0, 0, 0, + 0, AggregateFileCacheStats.FileCacheStatsType.OVER_ALL_STATS ); FileCacheStats fullStats = new FileCacheStats( @@ -766,6 +767,7 @@ private static AggregateFileCacheStats setAggregateFileCacheStats(long active, l 0, 0, 0, + 0, AggregateFileCacheStats.FileCacheStatsType.FULL_FILE_STATS ); FileCacheStats blockStats = new FileCacheStats( @@ -776,6 +778,7 @@ private static AggregateFileCacheStats setAggregateFileCacheStats(long active, l 0, 0, 0, + 0, AggregateFileCacheStats.FileCacheStatsType.BLOCK_FILE_STATS ); FileCacheStats pinnedStats = new FileCacheStats( @@ -786,6 +789,7 @@ private static AggregateFileCacheStats setAggregateFileCacheStats(long active, l 0, 0, 0, + 0, AggregateFileCacheStats.FileCacheStatsType.PINNED_FILE_STATS ); return new AggregateFileCacheStats(System.currentTimeMillis(), overallStats, fullStats, blockStats, pinnedStats); diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/AggregateFileCacheStats.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/AggregateFileCacheStats.java index 6322ad58ead6d..02436a5b119c1 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/filecache/AggregateFileCacheStats.java +++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/AggregateFileCacheStats.java @@ -116,6 +116,10 @@ public ByteSizeValue getEvicted() { return new ByteSizeValue(overallFileCacheStats.getEvicted()); } + public ByteSizeValue getRemoved() { + return new ByteSizeValue(overallFileCacheStats.getRemoved()); + } + public long getCacheHits() { return overallFileCacheStats.getCacheHits(); } @@ -138,6 +142,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.humanReadableField(Fields.USED_IN_BYTES, Fields.USED, getUsed()); builder.humanReadableField(Fields.PINNED_IN_BYTES, Fields.PINNED, getPinnedUsage()); builder.humanReadableField(Fields.EVICTIONS_IN_BYTES, Fields.EVICTIONS, getEvicted()); + builder.humanReadableField(Fields.REMOVED_IN_BYTES, Fields.REMOVED, getRemoved()); builder.field(Fields.ACTIVE_PERCENT, getActivePercent()); builder.field(Fields.USED_PERCENT, getUsedPercent()); builder.field(Fields.HIT_COUNT, getCacheHits()); @@ -161,6 +166,8 @@ static final class Fields { static final String PINNED_IN_BYTES = "pinned_in_bytes"; static final String EVICTIONS = "evictions"; static final String EVICTIONS_IN_BYTES = "evictions_in_bytes"; + static final String REMOVED = "removed"; + static final String REMOVED_IN_BYTES = "removed_in_bytes"; static final String TOTAL = "total"; static final String TOTAL_IN_BYTES = "total_in_bytes"; diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java index e9df347a8a71a..d6ef5ff655f60 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java +++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java @@ -257,6 +257,7 @@ public AggregateFileCacheStats fileCacheStats() { overallCacheStats.usage(), overallCacheStats.pinnedUsage(), overallCacheStats.evictionWeight(), + overallCacheStats.removeWeight(), overallCacheStats.hitCount(), overallCacheStats.missCount(), FileCacheStatsType.OVER_ALL_STATS @@ -267,6 +268,7 @@ public AggregateFileCacheStats fileCacheStats() { fullFileCacheStats.usage(), fullFileCacheStats.pinnedUsage(), fullFileCacheStats.evictionWeight(), + fullFileCacheStats.removeWeight(), fullFileCacheStats.hitCount(), fullFileCacheStats.missCount(), FileCacheStatsType.FULL_FILE_STATS @@ -277,6 +279,7 @@ public AggregateFileCacheStats fileCacheStats() { blockFileCacheStats.usage(), blockFileCacheStats.pinnedUsage(), blockFileCacheStats.evictionWeight(), + blockFileCacheStats.removeWeight(), blockFileCacheStats.hitCount(), blockFileCacheStats.missCount(), FileCacheStatsType.BLOCK_FILE_STATS @@ -287,6 +290,7 @@ public AggregateFileCacheStats fileCacheStats() { pinnedFileCacheStats.usage(), pinnedFileCacheStats.pinnedUsage(), pinnedFileCacheStats.evictionWeight(), + pinnedFileCacheStats.removeWeight(), pinnedFileCacheStats.hitCount(), pinnedFileCacheStats.missCount(), FileCacheStatsType.PINNED_FILE_STATS diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java index ace4593c4c7ec..1cbd1bfc5d3e2 100644 --- a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java +++ b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheStats.java @@ -8,6 +8,7 @@ package org.opensearch.index.store.remote.filecache; +import org.opensearch.Version; import org.opensearch.common.annotation.InternalApi; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; @@ -41,6 +42,7 @@ public class FileCacheStats implements Writeable, ToXContentFragment { private final long used; private final long pinned; private final long evicted; + private final long removed; private final long hits; private final long misses; private final FileCacheStatsType statsType; @@ -52,6 +54,7 @@ public FileCacheStats( final long used, final long pinned, final long evicted, + final long removed, final long hits, long misses, FileCacheStatsType statsType @@ -61,6 +64,7 @@ public FileCacheStats( this.used = used; this.pinned = pinned; this.evicted = evicted; + this.removed = removed; this.hits = hits; this.misses = misses; this.statsType = statsType; @@ -75,7 +79,14 @@ public FileCacheStats(final StreamInput in) throws IOException { this.pinned = in.readLong(); this.evicted = in.readLong(); this.hits = in.readLong(); - this.misses = in.readLong(); + + if (in.getVersion().onOrAfter(Version.V_3_4_0)) { + this.removed = in.readLong(); + this.misses = in.readLong(); + } else { + this.removed = 0L; + this.misses = 0L; + } } @Override @@ -87,7 +98,11 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeLong(pinned); out.writeLong(evicted); out.writeLong(hits); - out.writeLong(misses); + + if (out.getVersion().onOrAfter(Version.V_3_4_0)) { + out.writeLong(removed); + out.writeLong(misses); + } } public long getActive() { @@ -102,6 +117,10 @@ public long getEvicted() { return evicted; } + public long getRemoved() { + return removed; + } + public long getHits() { return hits; } @@ -136,10 +155,14 @@ static final class Fields { static final String USED = "used"; static final String PINNED = "pinned"; static final String USED_IN_BYTES = "used_in_bytes"; + static final String PINNED_IN_BYTES = "pinned_in_bytes"; static final String EVICTIONS = "evictions"; static final String EVICTIONS_IN_BYTES = "evictions_in_bytes"; + static final String REMOVED = "removed"; + static final String REMOVED_IN_BYTES = "removed_in_bytes"; static final String ACTIVE_PERCENT = "active_percent"; static final String HIT_COUNT = "hit_count"; + static final String MISS_COUNT = "miss_count"; } @Override @@ -147,14 +170,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(statsType.toString()); builder.humanReadableField(FileCacheStats.Fields.ACTIVE_IN_BYTES, FileCacheStats.Fields.ACTIVE, new ByteSizeValue(getActive())); builder.humanReadableField(FileCacheStats.Fields.USED_IN_BYTES, FileCacheStats.Fields.USED, new ByteSizeValue(getUsed())); - builder.humanReadableField(FileCacheStats.Fields.USED_IN_BYTES, Fields.PINNED, new ByteSizeValue(getPinnedUsage())); + builder.humanReadableField(FileCacheStats.Fields.PINNED_IN_BYTES, Fields.PINNED, new ByteSizeValue(getPinnedUsage())); builder.humanReadableField( FileCacheStats.Fields.EVICTIONS_IN_BYTES, FileCacheStats.Fields.EVICTIONS, new ByteSizeValue(getEvicted()) ); + builder.humanReadableField(FileCacheStats.Fields.REMOVED_IN_BYTES, FileCacheStats.Fields.REMOVED, new ByteSizeValue(getRemoved())); builder.field(FileCacheStats.Fields.ACTIVE_PERCENT, getActivePercent()); builder.field(FileCacheStats.Fields.HIT_COUNT, getHits()); + builder.field(FileCacheStats.Fields.MISS_COUNT, getCacheMisses()); builder.endObject(); return builder; } diff --git a/server/src/test/java/org/opensearch/cluster/ClusterInfoTests.java b/server/src/test/java/org/opensearch/cluster/ClusterInfoTests.java index 6fe33479565f8..b0d1b2c8c7ffe 100644 --- a/server/src/test/java/org/opensearch/cluster/ClusterInfoTests.java +++ b/server/src/test/java/org/opensearch/cluster/ClusterInfoTests.java @@ -187,6 +187,7 @@ private static Map randomFileCacheStats(int num randomLong(), randomLong(), randomLong(), + randomLong(), FileCacheStatsType.OVER_ALL_STATS ), new FileCacheStats( @@ -197,6 +198,7 @@ private static Map randomFileCacheStats(int num randomLong(), randomLong(), randomLong(), + randomLong(), FileCacheStatsType.FULL_FILE_STATS ), new FileCacheStats( @@ -207,6 +209,7 @@ private static Map randomFileCacheStats(int num randomLong(), randomLong(), randomLong(), + randomLong(), FileCacheStatsType.BLOCK_FILE_STATS ), new FileCacheStats( @@ -217,6 +220,7 @@ private static Map randomFileCacheStats(int num randomLong(), randomLong(), randomLong(), + randomLong(), FileCacheStatsType.PINNED_FILE_STATS ) ); diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitorTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitorTests.java index 5b5c777251ba3..6761f9f64a28b 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitorTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitorTests.java @@ -1411,6 +1411,7 @@ private static AggregateFileCacheStats createAggregateFileCacheStats(long totalC 0, 0, 0, + 0, AggregateFileCacheStats.FileCacheStatsType.OVER_ALL_STATS ); FileCacheStats fullStats = new FileCacheStats( @@ -1421,6 +1422,7 @@ private static AggregateFileCacheStats createAggregateFileCacheStats(long totalC 0, 0, 0, + 0, AggregateFileCacheStats.FileCacheStatsType.FULL_FILE_STATS ); FileCacheStats blockStats = new FileCacheStats( @@ -1431,6 +1433,7 @@ private static AggregateFileCacheStats createAggregateFileCacheStats(long totalC 0, 0, 0, + 0, AggregateFileCacheStats.FileCacheStatsType.BLOCK_FILE_STATS ); FileCacheStats pinnedStats = new FileCacheStats( @@ -1441,6 +1444,7 @@ private static AggregateFileCacheStats createAggregateFileCacheStats(long totalC 0, 0, 0, + 0, AggregateFileCacheStats.FileCacheStatsType.PINNED_FILE_STATS ); return new AggregateFileCacheStats(System.currentTimeMillis(), overallStats, fullStats, blockStats, pinnedStats); diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index a4f0c010ee154..5a8e0c142ca61 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -308,30 +308,30 @@ public void testDiskThresholdForRemoteShards() { "node1", new AggregateFileCacheStats( 0, - new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS), - new FileCacheStats(0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS), - new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS), - new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS) + new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS), + new FileCacheStats(0, 0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS), + new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS), + new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS) ) ); fileCacheStatsMap.put( "node2", new AggregateFileCacheStats( 0, - new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS), - new FileCacheStats(0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS), - new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS), - new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS) + new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS), + new FileCacheStats(0, 0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS), + new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS), + new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS) ) ); fileCacheStatsMap.put( "node3", new AggregateFileCacheStats( 0, - new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS), - new FileCacheStats(0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS), - new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS), - new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS) + new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS), + new FileCacheStats(0, 0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS), + new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS), + new FileCacheStats(0, 0, 1000, 0, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS) ) ); diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/WarmDiskThresholdDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/WarmDiskThresholdDeciderTests.java index 8fdea89865aa7..944949e82d3d0 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/WarmDiskThresholdDeciderTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/WarmDiskThresholdDeciderTests.java @@ -464,10 +464,10 @@ private Map createFileCacheStatsMap(long fileCa node, new AggregateFileCacheStats( randomNonNegativeInt(), - new FileCacheStats(0, fileCacheSize, 0, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS), - new FileCacheStats(0, fileCacheSize, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS), - new FileCacheStats(0, fileCacheSize, 0, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS), - new FileCacheStats(0, fileCacheSize, 0, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS) + new FileCacheStats(0, fileCacheSize, 0, 0, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS), + new FileCacheStats(0, fileCacheSize, 0, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS), + new FileCacheStats(0, fileCacheSize, 0, 0, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS), + new FileCacheStats(0, fileCacheSize, 0, 0, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS) ) ); } diff --git a/server/src/test/java/org/opensearch/index/store/remote/filecache/AggregateFileCacheStatsTests.java b/server/src/test/java/org/opensearch/index/store/remote/filecache/AggregateFileCacheStatsTests.java index 25dc2bdc412ef..6a433e751c315 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/filecache/AggregateFileCacheStatsTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/filecache/AggregateFileCacheStatsTests.java @@ -98,6 +98,7 @@ public static AggregateFileCacheStats getFileCacheStats(final long fileCacheCapa stats.usage(), stats.pinnedUsage(), stats.evictionWeight(), + stats.removeWeight(), stats.hitCount(), stats.missCount(), FileCacheStatsType.OVER_ALL_STATS @@ -108,6 +109,7 @@ public static AggregateFileCacheStats getFileCacheStats(final long fileCacheCapa stats.usage(), stats.pinnedUsage(), stats.evictionWeight(), + stats.removeWeight(), stats.hitCount(), stats.missCount(), FileCacheStatsType.FULL_FILE_STATS @@ -118,6 +120,7 @@ public static AggregateFileCacheStats getFileCacheStats(final long fileCacheCapa stats.usage(), stats.pinnedUsage(), stats.evictionWeight(), + stats.removeWeight(), stats.hitCount(), stats.missCount(), FileCacheStatsType.BLOCK_FILE_STATS @@ -128,6 +131,7 @@ public static AggregateFileCacheStats getFileCacheStats(final long fileCacheCapa stats.usage(), stats.pinnedUsage(), stats.evictionWeight(), + stats.removeWeight(), stats.hitCount(), stats.missCount(), FileCacheStatsType.PINNED_FILE_STATS @@ -141,9 +145,10 @@ public static FileCacheStats getMockFullFileCacheStats() { final long used = randomLongBetween(100000, BYTES_IN_GB); final long pinned = randomLongBetween(100000, BYTES_IN_GB); final long evicted = randomLongBetween(0, getMockCacheStats().getFullFileCacheStats().evictionWeight()); + final long removed = randomLongBetween(0, 10); final long hit = randomLongBetween(0, 10); final long misses = randomLongBetween(0, 10); - return new FileCacheStats(active, total, used, pinned, evicted, hit, misses, FileCacheStatsType.OVER_ALL_STATS); + return new FileCacheStats(active, total, used, pinned, evicted, removed, hit, misses, FileCacheStatsType.OVER_ALL_STATS); } public static AggregateFileCacheStats getMockFileCacheStats() throws IOException { diff --git a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheStatsTests.java b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheStatsTests.java index abae2c94e91d8..469d00f022149 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheStatsTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/filecache/FileCacheStatsTests.java @@ -24,6 +24,7 @@ public static FileCacheStats getMockFullFileCacheStats() { final long used = randomLongBetween(100000, BYTES_IN_GB); final long pinned = randomLongBetween(100000, BYTES_IN_GB); final long evicted = randomLongBetween(0, active); + final long removed = randomLongBetween(0, 10); final long hits = randomLongBetween(0, 10); final long misses = randomLongBetween(0, 10); @@ -33,6 +34,7 @@ public static FileCacheStats getMockFullFileCacheStats() { used, pinned, evicted, + removed, hits, misses, AggregateFileCacheStats.FileCacheStatsType.OVER_ALL_STATS @@ -43,7 +45,9 @@ public static void validateFullFileCacheStats(FileCacheStats expected, FileCache assertEquals(expected.getActive(), actual.getActive()); assertEquals(expected.getUsed(), actual.getUsed()); assertEquals(expected.getEvicted(), actual.getEvicted()); + assertEquals(expected.getRemoved(), actual.getRemoved()); assertEquals(expected.getHits(), actual.getHits()); + assertEquals(expected.getCacheMisses(), actual.getCacheMisses()); assertEquals(expected.getActivePercent(), actual.getActivePercent()); } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index a936d4ce79ec2..19b21ea1dd487 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -469,10 +469,10 @@ public void testSearchableSnapshotOverSubscription() { node.node.getId(), new AggregateFileCacheStats( 0, - new FileCacheStats(1, 0, 0, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS), - new FileCacheStats(0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS), - new FileCacheStats(1, 0, 0, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS), - new FileCacheStats(1, 0, 0, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS) + new FileCacheStats(1, 0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.OVER_ALL_STATS), + new FileCacheStats(0, 0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.FULL_FILE_STATS), + new FileCacheStats(1, 0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.BLOCK_FILE_STATS), + new FileCacheStats(1, 0, 0, 0, 0, 0, 0, 0, FileCacheStatsType.PINNED_FILE_STATS) ) ); }