Register a blob cache long counter metric for total evicted regions (elastic#131862)

Currently we only track the number of evicted _used_ regions (i.e. freq > 1).
This adds another metric, `es.blob_cache.count_of_evicted_regions.total`, that tracks
the total number of evicted regions, irrespective of frequency.

Author: andreidan

x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheMetrics.java

@@ -27,9 +27,11 @@ public class BlobCacheMetrics {
     public static final String CACHE_POPULATION_SOURCE_ATTRIBUTE_KEY = "source";
     public static final String LUCENE_FILE_EXTENSION_ATTRIBUTE_KEY = "file_extension";
     public static final String NON_LUCENE_EXTENSION_TO_RECORD = "other";
+    public static final String BLOB_CACHE_COUNT_OF_EVICTED_REGIONS_TOTAL = "es.blob_cache.count_of_evicted_regions.total";
 
     private final LongCounter cacheMissCounter;
     private final LongCounter evictedCountNonZeroFrequency;
+    private final LongCounter totalEvictedCount;
     private final LongHistogram cacheMissLoadTimes;
     private final DoubleHistogram cachePopulationThroughput;
     private final LongCounter cachePopulationBytes;
@@ -66,6 +68,11 @@ public BlobCacheMetrics(MeterRegistry meterRegistry) {
                 "The number of times a cache entry was evicted where the frequency was not zero",
                 "entries"
             ),
+            meterRegistry.registerLongCounter(
+                BLOB_CACHE_COUNT_OF_EVICTED_REGIONS_TOTAL,
+                "The number of times a cache entry was evicted, irrespective of the frequency",
+                "entries"
+            ),
             meterRegistry.registerLongHistogram(
                 "es.blob_cache.cache_miss_load_times.histogram",
                 "The time in milliseconds for populating entries in the blob store resulting from a cache miss, expressed as a histogram.",
@@ -92,13 +99,15 @@ public BlobCacheMetrics(MeterRegistry meterRegistry) {
     BlobCacheMetrics(
         LongCounter cacheMissCounter,
         LongCounter evictedCountNonZeroFrequency,
+        LongCounter totalEvictedCount,
         LongHistogram cacheMissLoadTimes,
         DoubleHistogram cachePopulationThroughput,
         LongCounter cachePopulationBytes,
         LongCounter cachePopulationTime
     ) {
         this.cacheMissCounter = cacheMissCounter;
         this.evictedCountNonZeroFrequency = evictedCountNonZeroFrequency;
+        this.totalEvictedCount = totalEvictedCount;
         this.cacheMissLoadTimes = cacheMissLoadTimes;
         this.cachePopulationThroughput = cachePopulationThroughput;
         this.cachePopulationBytes = cachePopulationBytes;
@@ -115,6 +124,10 @@ public LongCounter getEvictedCountNonZeroFrequency() {
         return evictedCountNonZeroFrequency;
     }
 
+    public LongCounter getTotalEvictedCount() {
+        return totalEvictedCount;
+    }
+
     public LongHistogram getCacheMissLoadTimes() {
         return cacheMissLoadTimes;
     }

x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java

@@ -922,6 +922,7 @@ boolean tryEvict() {
             if (refCount() <= 1 && evict()) {
                 logger.trace("evicted {} with channel offset {}", regionKey, physicalStartOffset());
                 blobCacheService.evictCount.increment();
+                blobCacheService.blobCacheMetrics.getTotalEvictedCount().increment();
                 decRef();
                 return true;
             }
@@ -933,6 +934,7 @@ boolean tryEvictNoDecRef() {
             if (refCount() <= 1 && evict()) {
                 logger.trace("evicted and take {} with channel offset {}", regionKey, physicalStartOffset());
                 blobCacheService.evictCount.increment();
+                blobCacheService.blobCacheMetrics.getTotalEvictedCount().increment();
                 return true;
             }
 
@@ -944,6 +946,7 @@ public boolean forceEvict() {
             if (evict()) {
                 logger.trace("force evicted {} with channel offset {}", regionKey, physicalStartOffset());
                 blobCacheService.evictCount.increment();
+                blobCacheService.blobCacheMetrics.getTotalEvictedCount().increment();
                 decRef();
                 return true;
             }

x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java

@@ -34,6 +34,8 @@
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.node.NodeRoleSettings;
+import org.elasticsearch.telemetry.InstrumentType;
+import org.elasticsearch.telemetry.RecordingMeterRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -57,6 +59,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static org.elasticsearch.blobcache.BlobCacheMetrics.BLOB_CACHE_COUNT_OF_EVICTED_REGIONS_TOTAL;
 import static org.elasticsearch.node.Node.NODE_NAME_SETTING;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -88,14 +91,16 @@ public void testBasicEviction() throws IOException {
             .put("path.home", createTempDir())
             .build();
         final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue();
+        RecordingMeterRegistry recordingMeterRegistry = new RecordingMeterRegistry();
+        BlobCacheMetrics metrics = new BlobCacheMetrics(recordingMeterRegistry);
         try (
             NodeEnvironment environment = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings));
             var cacheService = new SharedBlobCacheService<>(
                 environment,
                 settings,
                 taskQueue.getThreadPool(),
                 taskQueue.getThreadPool().executor(ThreadPool.Names.GENERIC),
-                BlobCacheMetrics.NOOP
+                metrics
             )
         ) {
             final var cacheKey = generateCacheKey();
@@ -114,9 +119,17 @@ public void testBasicEviction() throws IOException {
                 assertTrue(tryEvict(region1));
             }
             assertEquals(3, cacheService.freeRegionCount());
+            // one eviction should be reflected in the telemetry for total count of evicted regions
+            assertThat(
+                recordingMeterRegistry.getRecorder()
+                    .getMeasurements(InstrumentType.LONG_COUNTER, BLOB_CACHE_COUNT_OF_EVICTED_REGIONS_TOTAL)
+                    .size(),
+                is(1)
+            );
             synchronized (cacheService) {
                 assertFalse(tryEvict(region1));
             }
+
             assertEquals(3, cacheService.freeRegionCount());
             final var bytesReadFuture = new PlainActionFuture<Integer>();
             region0.populateAndRead(
@@ -144,6 +157,14 @@ public void testBasicEviction() throws IOException {
                 assertTrue(tryEvict(region2));
             }
             assertEquals(5, cacheService.freeRegionCount());
+            // another 2 evictions should bump our total evictions telemetry at 3
+            assertThat(
+                recordingMeterRegistry.getRecorder()
+                    .getMeasurements(InstrumentType.LONG_COUNTER, BLOB_CACHE_COUNT_OF_EVICTED_REGIONS_TOTAL)
+                    .size(),
+                is(3)
+            );
+
             assertTrue(bytesReadFuture.isDone());
             assertEquals(Integer.valueOf(1), bytesReadFuture.actionGet());
         }