Add a took time histogram for search triggered cache miss to blob cache metrics (elastic#135257)

This adds a es.blob_cache.search_origin.download_took_time.total histogram metric
to track the times we spend downloading from remote storage, but only when triggered
by a search operation.

Author: andreidan

x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheMetrics.java

@@ -31,6 +31,7 @@ public class BlobCacheMetrics {
     public static final String LUCENE_FILE_EXTENSION_ATTRIBUTE_KEY = "file_extension";
     public static final String NON_LUCENE_EXTENSION_TO_RECORD = "other";
     public static final String BLOB_CACHE_COUNT_OF_EVICTED_REGIONS_TOTAL = "es.blob_cache.count_of_evicted_regions.total";
+    public static final String SEARCH_ORIGIN_REMOTE_STORAGE_DOWNLOAD_TOOK_TIME = "es.blob_cache.search_origin.download_took_time.total";
 
     private final LongCounter cacheMissCounter;
     private final LongCounter evictedCountNonZeroFrequency;
@@ -43,6 +44,7 @@ public class BlobCacheMetrics {
     private final LongAdder missCount = new LongAdder();
     private final LongAdder readCount = new LongAdder();
     private final LongCounter epochChanges;
+    private final LongHistogram searchOriginDownloadTime;
 
     public enum CachePopulationReason {
         /**
@@ -100,7 +102,12 @@ public BlobCacheMetrics(MeterRegistry meterRegistry) {
                 "The time spent copying data into the cache",
                 "milliseconds"
             ),
-            meterRegistry.registerLongCounter("es.blob_cache.epoch.total", "The epoch changes of the LFU cache", "count")
+            meterRegistry.registerLongCounter("es.blob_cache.epoch.total", "The epoch changes of the LFU cache", "count"),
+            meterRegistry.registerLongHistogram(
+                SEARCH_ORIGIN_REMOTE_STORAGE_DOWNLOAD_TOOK_TIME,
+                "The distribution of time in millis taken to download data from remote storage for search requests",
+                "milliseconds"
+            )
         );
 
         meterRegistry.registerLongGauge(
@@ -137,7 +144,8 @@ public BlobCacheMetrics(MeterRegistry meterRegistry) {
         DoubleHistogram cachePopulationThroughput,
         LongCounter cachePopulationBytes,
         LongCounter cachePopulationTime,
-        LongCounter epochChanges
+        LongCounter epochChanges,
+        LongHistogram searchOriginDownloadTime
     ) {
         this.cacheMissCounter = cacheMissCounter;
         this.evictedCountNonZeroFrequency = evictedCountNonZeroFrequency;
@@ -147,6 +155,7 @@ public BlobCacheMetrics(MeterRegistry meterRegistry) {
         this.cachePopulationBytes = cachePopulationBytes;
         this.cachePopulationTime = cachePopulationTime;
         this.epochChanges = epochChanges;
+        this.searchOriginDownloadTime = searchOriginDownloadTime;
     }
 
     public static final BlobCacheMetrics NOOP = new BlobCacheMetrics(TelemetryProvider.NOOP.getMeterRegistry());
@@ -167,6 +176,10 @@ public LongHistogram getCacheMissLoadTimes() {
         return cacheMissLoadTimes;
     }
 
+    public LongHistogram getSearchOriginDownloadTime() {
+        return searchOriginDownloadTime;
+    }
+
     /**
      * Record the various cache population metrics after a chunk is copied to the cache
      *

x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java

@@ -420,6 +420,10 @@ public BlobCacheMetrics getBlobCacheMetrics() {
         return blobCacheMetrics;
     }
 
+    public ThreadPool getThreadPool() {
+        return threadPool;
+    }
+
     public int getRangeSize() {
         return rangeSize;
     }