diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheMetrics.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheMetrics.java index e7f279a50aa3b..528e1225455aa 100644 --- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheMetrics.java +++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheMetrics.java @@ -27,9 +27,11 @@ public class BlobCacheMetrics { public static final String CACHE_POPULATION_SOURCE_ATTRIBUTE_KEY = "source"; public static final String LUCENE_FILE_EXTENSION_ATTRIBUTE_KEY = "file_extension"; public static final String NON_LUCENE_EXTENSION_TO_RECORD = "other"; + public static final String BLOB_CACHE_COUNT_OF_EVICTED_REGIONS_TOTAL = "es.blob_cache.count_of_evicted_regions.total"; private final LongCounter cacheMissCounter; private final LongCounter evictedCountNonZeroFrequency; + private final LongCounter totalEvictedCount; private final LongHistogram cacheMissLoadTimes; private final DoubleHistogram cachePopulationThroughput; private final LongCounter cachePopulationBytes; @@ -66,6 +68,11 @@ public BlobCacheMetrics(MeterRegistry meterRegistry) { "The number of times a cache entry was evicted where the frequency was not zero", "entries" ), + meterRegistry.registerLongCounter( + BLOB_CACHE_COUNT_OF_EVICTED_REGIONS_TOTAL, + "The number of times a cache entry was evicted, irrespective of the frequency", + "entries" + ), meterRegistry.registerLongHistogram( "es.blob_cache.cache_miss_load_times.histogram", "The time in milliseconds for populating entries in the blob store resulting from a cache miss, expressed as a histogram.", @@ -92,6 +99,7 @@ public BlobCacheMetrics(MeterRegistry meterRegistry) { BlobCacheMetrics( LongCounter cacheMissCounter, LongCounter evictedCountNonZeroFrequency, + LongCounter totalEvictedCount, LongHistogram cacheMissLoadTimes, DoubleHistogram cachePopulationThroughput, LongCounter cachePopulationBytes, @@ -99,6 +107,7 @@ public BlobCacheMetrics(MeterRegistry meterRegistry) { ) { this.cacheMissCounter = cacheMissCounter; this.evictedCountNonZeroFrequency = evictedCountNonZeroFrequency; + this.totalEvictedCount = totalEvictedCount; this.cacheMissLoadTimes = cacheMissLoadTimes; this.cachePopulationThroughput = cachePopulationThroughput; this.cachePopulationBytes = cachePopulationBytes; @@ -115,6 +124,10 @@ public LongCounter getEvictedCountNonZeroFrequency() { return evictedCountNonZeroFrequency; } + public LongCounter getTotalEvictedCount() { + return totalEvictedCount; + } + public LongHistogram getCacheMissLoadTimes() { return cacheMissLoadTimes; } diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java index 3fc66d504a76e..6ef4262f393c5 100644 --- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java +++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java @@ -922,6 +922,7 @@ boolean tryEvict() { if (refCount() <= 1 && evict()) { logger.trace("evicted {} with channel offset {}", regionKey, physicalStartOffset()); blobCacheService.evictCount.increment(); + blobCacheService.blobCacheMetrics.getTotalEvictedCount().increment(); decRef(); return true; } @@ -933,6 +934,7 @@ boolean tryEvictNoDecRef() { if (refCount() <= 1 && evict()) { logger.trace("evicted and take {} with channel offset {}", regionKey, physicalStartOffset()); blobCacheService.evictCount.increment(); + blobCacheService.blobCacheMetrics.getTotalEvictedCount().increment(); return true; } @@ -944,6 +946,7 @@ public boolean forceEvict() { if (evict()) { logger.trace("force evicted {} with channel offset {}", regionKey, physicalStartOffset()); blobCacheService.evictCount.increment(); + blobCacheService.blobCacheMetrics.getTotalEvictedCount().increment(); decRef(); return true; } diff --git a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java index 1b3335d47b1f0..3a64430e7c3d2 100644 --- a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java +++ b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java @@ -34,6 +34,8 @@ import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.node.NodeRoleSettings; +import org.elasticsearch.telemetry.InstrumentType; +import org.elasticsearch.telemetry.RecordingMeterRegistry; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -57,6 +59,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.elasticsearch.blobcache.BlobCacheMetrics.BLOB_CACHE_COUNT_OF_EVICTED_REGIONS_TOTAL; import static org.elasticsearch.node.Node.NODE_NAME_SETTING; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -88,6 +91,8 @@ public void testBasicEviction() throws IOException { .put("path.home", createTempDir()) .build(); final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue(); + RecordingMeterRegistry recordingMeterRegistry = new RecordingMeterRegistry(); + BlobCacheMetrics metrics = new BlobCacheMetrics(recordingMeterRegistry); try ( NodeEnvironment environment = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings)); var cacheService = new SharedBlobCacheService<>( @@ -95,7 +100,7 @@ public void testBasicEviction() throws IOException { settings, taskQueue.getThreadPool(), taskQueue.getThreadPool().executor(ThreadPool.Names.GENERIC), - BlobCacheMetrics.NOOP + metrics ) ) { final var cacheKey = generateCacheKey(); @@ -114,9 +119,17 @@ public void testBasicEviction() throws IOException { assertTrue(tryEvict(region1)); } assertEquals(3, cacheService.freeRegionCount()); + // one eviction should be reflected in the telemetry for total count of evicted regions + assertThat( + recordingMeterRegistry.getRecorder() + .getMeasurements(InstrumentType.LONG_COUNTER, BLOB_CACHE_COUNT_OF_EVICTED_REGIONS_TOTAL) + .size(), + is(1) + ); synchronized (cacheService) { assertFalse(tryEvict(region1)); } + assertEquals(3, cacheService.freeRegionCount()); final var bytesReadFuture = new PlainActionFuture(); region0.populateAndRead( @@ -144,6 +157,14 @@ public void testBasicEviction() throws IOException { assertTrue(tryEvict(region2)); } assertEquals(5, cacheService.freeRegionCount()); + // another 2 evictions should bump our total evictions telemetry at 3 + assertThat( + recordingMeterRegistry.getRecorder() + .getMeasurements(InstrumentType.LONG_COUNTER, BLOB_CACHE_COUNT_OF_EVICTED_REGIONS_TOTAL) + .size(), + is(3) + ); + assertTrue(bytesReadFuture.isDone()); assertEquals(Integer.valueOf(1), bytesReadFuture.actionGet()); }