Skip to content

Commit 0a19687

Browse files
nicktindallnicktindall
committed
Populate attributes consistently, add shard stage and snapshot state
1 parent d05eb17 commit 0a19687

File tree

3 files changed

+86
-16
lines changed

3 files changed

+86
-16
lines changed

‎server/src/internalClusterTest/java/org/elasticsearch/repositories/SnapshotMetricsIT.java‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,32 +12,42 @@
1212
import org.elasticsearch.action.admin.indices.stats.IndexStats;
1313
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
1414
import org.elasticsearch.action.admin.indices.stats.ShardStats;
15+
import org.elasticsearch.cluster.metadata.ProjectId;
1516
import org.elasticsearch.common.settings.Settings;
1617
import org.elasticsearch.common.unit.ByteSizeValue;
1718
import org.elasticsearch.common.util.CollectionUtils;
19+
import org.elasticsearch.common.util.Maps;
1820
import org.elasticsearch.core.TimeValue;
21+
import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
1922
import org.elasticsearch.plugins.Plugin;
2023
import org.elasticsearch.plugins.PluginsService;
2124
import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
25+
import org.elasticsearch.snapshots.SnapshotState;
26+
import org.elasticsearch.telemetry.InstrumentType;
2227
import org.elasticsearch.telemetry.Measurement;
28+
import org.elasticsearch.telemetry.RecordingMeterRegistry;
2329
import org.elasticsearch.telemetry.TestTelemetryPlugin;
2430
import org.elasticsearch.test.ESIntegTestCase;
2531
import org.elasticsearch.threadpool.ThreadPool;
2632
import org.hamcrest.Matcher;
2733

2834
import java.util.Collection;
2935
import java.util.List;
36+
import java.util.Map;
3037
import java.util.stream.Stream;
3138
import java.util.stream.StreamSupport;
3239

3340
import static org.elasticsearch.threadpool.ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING;
3441
import static org.hamcrest.Matchers.allOf;
42+
import static org.hamcrest.Matchers.empty;
3543
import static org.hamcrest.Matchers.equalTo;
3644
import static org.hamcrest.Matchers.everyItem;
3745
import static org.hamcrest.Matchers.greaterThan;
46+
import static org.hamcrest.Matchers.hasEntry;
3847
import static org.hamcrest.Matchers.hasItem;
3948
import static org.hamcrest.Matchers.hasSize;
4049
import static org.hamcrest.Matchers.lessThan;
50+
import static org.hamcrest.Matchers.not;
4151

4252
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
4353
public class SnapshotMetricsIT extends AbstractSnapshotIntegTestCase {
@@ -170,6 +180,65 @@ public void testSnapshotAPMMetrics() throws Exception {
170180

171181
// assert we throttled on restore
172182
assertThat(getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOT_RESTORE_THROTTLE_DURATION), greaterThan(0L));
183+
184+
// assert appropriate attributes are present
185+
final Map<String, Object> expectedAttrs = Map.of(
186+
"project_id",
187+
ProjectId.DEFAULT.id(),
188+
"repo_name",
189+
repositoryName,
190+
"repo_type",
191+
"mock"
192+
);
193+
final Map<String, Object> expectedAttrsWithShardStage = Maps.copyMapWithAddedEntry(
194+
expectedAttrs,
195+
"stage",
196+
IndexShardSnapshotStatus.Stage.DONE.name()
197+
);
198+
final Map<String, Object> expectedAttrsWithSnapshotState = Maps.copyMapWithAddedEntry(
199+
expectedAttrs,
200+
"state",
201+
SnapshotState.SUCCESS.name()
202+
);
203+
assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOTS_STARTED, expectedAttrs);
204+
assertMetricsHaveAttributes(InstrumentType.LONG_GAUGE, SnapshotMetrics.SNAPSHOTS_IN_PROGRESS, expectedAttrs);
205+
assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOTS_COMPLETED, expectedAttrsWithSnapshotState);
206+
assertMetricsHaveAttributes(InstrumentType.DOUBLE_HISTOGRAM, SnapshotMetrics.SNAPSHOT_DURATION, expectedAttrsWithSnapshotState);
207+
208+
assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOT_SHARDS_STARTED, expectedAttrs);
209+
assertMetricsHaveAttributes(InstrumentType.LONG_GAUGE, SnapshotMetrics.SNAPSHOT_SHARDS_IN_PROGRESS, expectedAttrs);
210+
assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOT_SHARDS_COMPLETED, expectedAttrsWithShardStage);
211+
assertMetricsHaveAttributes(InstrumentType.DOUBLE_HISTOGRAM, SnapshotMetrics.SNAPSHOT_SHARDS_DURATION, expectedAttrsWithShardStage);
212+
213+
assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOT_RESTORE_THROTTLE_DURATION, expectedAttrs);
214+
assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOT_CREATE_THROTTLE_DURATION, expectedAttrs);
215+
assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOT_UPLOAD_READ_DURATION, expectedAttrs);
216+
assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOT_UPLOAD_DURATION, expectedAttrs);
217+
assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOT_BYTES_UPLOADED, expectedAttrs);
218+
assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOT_BLOBS_UPLOADED, expectedAttrs);
219+
}
220+
221+
private static void assertMetricsHaveAttributes(
222+
InstrumentType instrumentType,
223+
String metricName,
224+
Map<String, Object> expectedAttributes
225+
) {
226+
final List<Measurement> clusterMeasurements = getClusterMeasurements(instrumentType, metricName);
227+
assertThat(clusterMeasurements, not(empty()));
228+
clusterMeasurements.forEach(recordingMetric -> {
229+
for (Map.Entry<String, Object> entry : expectedAttributes.entrySet()) {
230+
assertThat(recordingMetric.attributes(), hasEntry(entry.getKey(), entry.getValue()));
231+
}
232+
});
233+
}
234+
235+
private static List<Measurement> getClusterMeasurements(InstrumentType instrumentType, String metricName) {
236+
return allTestTelemetryPlugins().flatMap(
237+
testTelemetryPlugin -> ((RecordingMeterRegistry) testTelemetryPlugin.getTelemetryProvider(Settings.EMPTY).getMeterRegistry())
238+
.getRecorder()
239+
.getMeasurements(instrumentType, metricName)
240+
.stream()
241+
).toList();
173242
}
174243

175244
private static void assertDoubleHistogramMetrics(String metricName, Matcher<? super List<Double>> matcher) {

‎server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreSnapshotMetrics.java‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.elasticsearch.cluster.metadata.ProjectId;
1313
import org.elasticsearch.cluster.metadata.RepositoryMetadata;
1414
import org.elasticsearch.common.metrics.CounterMetric;
15+
import org.elasticsearch.common.util.Maps;
1516
import org.elasticsearch.core.Nullable;
1617
import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
1718
import org.elasticsearch.repositories.RepositoriesStats;
@@ -40,7 +41,7 @@ public BlobStoreSnapshotMetrics(@Nullable ProjectId projectId, RepositoryMetadat
4041
}
4142

4243
public void incrementSnapshotRateLimitingTimeInNanos(long throttleTimeNanos) {
43-
snapshotMetrics.snapshotCreateThrottleDurationCounter().incrementBy(throttleTimeNanos);
44+
snapshotMetrics.snapshotCreateThrottleDurationCounter().incrementBy(throttleTimeNanos, metricAttributes);
4445
snapshotRateLimitingTimeInNanos.inc(throttleTimeNanos);
4546
}
4647

@@ -49,7 +50,7 @@ public long snapshotRateLimitingTimeInNanos() {
4950
}
5051

5152
public void incrementRestoreRateLimitingTimeInNanos(long throttleTimeNanos) {
52-
snapshotMetrics.snapshotRestoreThrottleDurationCounter().incrementBy(throttleTimeNanos);
53+
snapshotMetrics.snapshotRestoreThrottleDurationCounter().incrementBy(throttleTimeNanos, metricAttributes);
5354
restoreRateLimitingTimeInNanos.inc(throttleTimeNanos);
5455
}
5556

@@ -58,34 +59,33 @@ public long restoreRateLimitingTimeInNanos() {
5859
}
5960

6061
public void incrementCountersForPartUpload(long partSizeInBytes, long partWriteTimeNanos) {
61-
snapshotMetrics.snapshotBytesUploadedCounter().incrementBy(partSizeInBytes);
62-
snapshotMetrics.snapshotUploadDurationCounter().incrementBy(partWriteTimeNanos);
62+
snapshotMetrics.snapshotBytesUploadedCounter().incrementBy(partSizeInBytes, metricAttributes);
63+
snapshotMetrics.snapshotUploadDurationCounter().incrementBy(partWriteTimeNanos, metricAttributes);
6364
numberOfBytesUploaded.inc(partSizeInBytes);
6465
uploadTimeInNanos.inc(partWriteTimeNanos);
6566
}
6667

6768
public void incrementNumberOfBlobsUploaded() {
68-
snapshotMetrics.snapshotBlobsUploadedCounter().increment();
69+
snapshotMetrics.snapshotBlobsUploadedCounter().incrementBy(1, metricAttributes);
6970
numberOfBlobsUploaded.inc();
7071
}
7172

7273
public void shardSnapshotStarted() {
73-
snapshotMetrics.snapshotsShardsStartedCounter().increment();
74+
snapshotMetrics.snapshotsShardsStartedCounter().incrementBy(1, metricAttributes);
7475
numberOfShardSnapshotsStarted.inc();
7576
shardSnapshotsInProgress.inc();
7677
}
7778

7879
public void shardSnapshotCompleted(IndexShardSnapshotStatus status) {
79-
snapshotMetrics.snapshotsShardsCompletedCounter().increment();
80-
if (status.getStage() == IndexShardSnapshotStatus.Stage.DONE) {
81-
snapshotMetrics.snapshotShardsDurationHistogram().record(status.getTotalTime() / 1_000f);
82-
}
80+
final Map<String, Object> attrsWithStage = Maps.copyMapWithAddedEntry(metricAttributes, "stage", status.getStage().name());
81+
snapshotMetrics.snapshotsShardsCompletedCounter().incrementBy(1, attrsWithStage);
82+
snapshotMetrics.snapshotShardsDurationHistogram().record(status.getTotalTime() / 1_000f, attrsWithStage);
8383
numberOfShardSnapshotsCompleted.inc();
8484
shardSnapshotsInProgress.dec();
8585
}
8686

8787
public void incrementUploadReadTime(long readTimeInNanos) {
88-
snapshotMetrics.snapshotUploadReadDurationCounter().incrementBy(readTimeInNanos);
88+
snapshotMetrics.snapshotUploadReadDurationCounter().incrementBy(readTimeInNanos, metricAttributes);
8989
uploadReadTimeInNanos.inc(readTimeInNanos);
9090
}
9191

‎server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1654,13 +1654,14 @@ protected void doRun() {
16541654
@Override
16551655
public void onResponse(List<ActionListener<SnapshotInfo>> actionListeners) {
16561656
completeListenersIgnoringException(actionListeners, snapshotInfo);
1657-
final Map<String, Object> attributes = SnapshotMetrics.createAttributesMap(
1658-
snapshot.getProjectId(),
1659-
repo.getMetadata()
1657+
final Map<String, Object> attributesWithState = Maps.copyMapWithAddedEntry(
1658+
SnapshotMetrics.createAttributesMap(snapshot.getProjectId(), repo.getMetadata()),
1659+
"state",
1660+
snapshotInfo.state().name()
16601661
);
1661-
snapshotMetrics.snapshotsCompletedCounter().incrementBy(1, attributes);
1662+
snapshotMetrics.snapshotsCompletedCounter().incrementBy(1, attributesWithState);
16621663
snapshotMetrics.snapshotsDurationHistogram()
1663-
.record((snapshotInfo.endTime() - snapshotInfo.startTime()) / 1_000.0, attributes);
1664+
.record((snapshotInfo.endTime() - snapshotInfo.startTime()) / 1_000.0, attributesWithState);
16641665
logger.info("snapshot [{}] completed with state [{}]", snapshot, snapshotInfo.state());
16651666
}
16661667

0 commit comments

Comments
 (0)