Fix naming, record shard duration as histogram

Author: nicktindall

server/src/main/java/org/elasticsearch/repositories/SnapshotMetrics.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.repositories;
 
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
+import org.elasticsearch.telemetry.metric.DoubleHistogram;
 import org.elasticsearch.telemetry.metric.LongCounter;
 import org.elasticsearch.telemetry.metric.LongGauge;
 import org.elasticsearch.telemetry.metric.LongWithAttributes;
@@ -21,9 +22,10 @@
 import java.util.function.Supplier;
 
 public record SnapshotMetrics(
-    LongCounter snapshotsStartedCounter,
-    LongCounter snapshotsCompletedCounter,
-    LongGauge snapshotsInProgressGauge,
+    LongCounter snapshotsShardsStartedCounter,
+    LongCounter snapshotsShardsCompletedCounter,
+    LongGauge snapshotShardsInProgressGauge,
+    DoubleHistogram snapshotShardsDurationHistogram,
     LongCounter snapshotBlobsUploadedCounter,
     LongCounter snapshotBytesUploadedCounter,
     LongCounter snapshotUploadDurationCounter,
@@ -34,9 +36,10 @@ public record SnapshotMetrics(
 
     public static final SnapshotMetrics NOOP = new SnapshotMetrics(MeterRegistry.NOOP, List::of);
 
-    public static final String SNAPSHOTS_STARTED = "es.repositories.snapshots.started.total";
-    public static final String SNAPSHOTS_COMPLETED = "es.repositories.snapshots.completed.total";
-    public static final String SNAPSHOTS_IN_PROGRESS = "es.repositories.snapshots.current";
+    public static final String SNAPSHOT_SHARDS_STARTED = "es.repositories.snapshots.shards.started.total";
+    public static final String SNAPSHOT_SHARDS_COMPLETED = "es.repositories.snapshots.shards.completed.total";
+    public static final String SNAPSHOT_SHARDS_IN_PROGRESS = "es.repositories.snapshots.shards.current";
+    public static final String SNAPSHOT_SHARDS_DURATION = "es.repositories.snapshots.shards.duration.histogram";
     public static final String SNAPSHOT_BLOBS_UPLOADED = "es.repositories.snapshots.blobs.uploaded.total";
     public static final String SNAPSHOT_BYTES_UPLOADED = "es.repositories.snapshots.upload.bytes.total";
     public static final String SNAPSHOT_UPLOAD_DURATION = "es.repositories.snapshots.upload.upload_time.total";
@@ -46,14 +49,15 @@ public record SnapshotMetrics(
 
     public SnapshotMetrics(MeterRegistry meterRegistry, Supplier<Collection<LongWithAttributes>> shardSnapshotsInProgressObserver) {
         this(
-            meterRegistry.registerLongCounter(SNAPSHOTS_STARTED, "shard snapshots started", "unit"),
-            meterRegistry.registerLongCounter(SNAPSHOTS_COMPLETED, "shard snapshots completed", "unit"),
+            meterRegistry.registerLongCounter(SNAPSHOT_SHARDS_STARTED, "shard snapshots started", "unit"),
+            meterRegistry.registerLongCounter(SNAPSHOT_SHARDS_COMPLETED, "shard snapshots completed", "unit"),
             meterRegistry.registerLongsGauge(
-                SNAPSHOTS_IN_PROGRESS,
+                SNAPSHOT_SHARDS_IN_PROGRESS,
                 "shard snapshots in progress",
                 "unit",
                 shardSnapshotsInProgressObserver
             ),
+            meterRegistry.registerDoubleHistogram(SNAPSHOT_SHARDS_DURATION, "shard snapshots duration", "s"),
             meterRegistry.registerLongCounter(SNAPSHOT_BLOBS_UPLOADED, "snapshot blobs uploaded", "unit"),
             meterRegistry.registerLongCounter(SNAPSHOT_BYTES_UPLOADED, "snapshot bytes uploaded", "bytes"),
             meterRegistry.registerLongCounter(SNAPSHOT_UPLOAD_DURATION, "snapshot upload duration", "ns"),

server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java

@@ -3214,8 +3214,13 @@ public void snapshotShard(SnapshotShardContext context) {
     }
 
     private void doSnapshotShard(SnapshotShardContext context) {
+        final long startTimeInMillis = threadPool.absoluteTimeInMillis();
         blobStoreSnapshotMetrics.shardSnapshotStarted();
-        context.addListener(ActionListener.running(blobStoreSnapshotMetrics::shardSnapshotCompleted));
+        context.addListener(
+            ActionListener.running(
+                () -> blobStoreSnapshotMetrics.shardSnapshotCompleted(threadPool.absoluteTimeInMillis() - startTimeInMillis)
+            )
+        );
         if (isReadOnly()) {
             context.onFailure(new RepositoryException(metadata.name(), "cannot snapshot shard on a readonly repository"));
             return;
@@ -3225,7 +3230,6 @@ private void doSnapshotShard(SnapshotShardContext context) {
         final SnapshotId snapshotId = context.snapshotId();
         final IndexShardSnapshotStatus snapshotStatus = context.status();
         snapshotStatus.updateStatusDescription("snapshot task runner: setting up shard snapshot");
-        final long startTime = threadPool.absoluteTimeInMillis();
         try {
             final ShardGeneration generation = snapshotStatus.generation();
             final BlobContainer shardContainer = shardContainer(context.indexId(), shardId);
@@ -3347,7 +3351,7 @@ private void doSnapshotShard(SnapshotShardContext context) {
 
             snapshotStatus.updateStatusDescription("snapshot task runner: starting shard snapshot");
             snapshotStatus.moveToStarted(
-                startTime,
+                startTimeInMillis,
                 indexIncrementalFileCount,
                 indexTotalNumberOfFiles,
                 indexIncrementalSize,

server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreSnapshotMetrics.java

@@ -66,13 +66,14 @@ public void incrementNumberOfBlobsUploaded() {
     }
 
     public void shardSnapshotStarted() {
-        snapshotMetrics.snapshotsStartedCounter().increment();
+        snapshotMetrics.snapshotsShardsStartedCounter().increment();
         numberOfShardSnapshotsStarted.inc();
         shardSnapshotsInProgress.inc();
     }
 
-    public void shardSnapshotCompleted() {
-        snapshotMetrics.snapshotsCompletedCounter().increment();
+    public void shardSnapshotCompleted(long durationInMillis) {
+        snapshotMetrics.snapshotsShardsCompletedCounter().increment();
+        snapshotMetrics.snapshotShardsDurationHistogram().record(durationInMillis / 1_000f);
         numberOfShardSnapshotsCompleted.inc();
         shardSnapshotsInProgress.dec();
     }