Skip to content

Commit 3ba5832

Browse files
nicktindallnicktindall
committed
Publish new stats via node stats
1 parent 9a0e0ec commit 3ba5832

File tree

8 files changed

+111
-17
lines changed

8 files changed

+111
-17
lines changed

‎server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryThrottlingStatsIT.java‎ renamed to ‎server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositorySnapshotStatsIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@
2424
import static org.hamcrest.Matchers.greaterThan;
2525

2626
@ESIntegTestCase.ClusterScope(numDataNodes = 0, scope = ESIntegTestCase.Scope.TEST)
27-
public class RepositoryThrottlingStatsIT extends AbstractSnapshotIntegTestCase {
27+
public class RepositorySnapshotStatsIT extends AbstractSnapshotIntegTestCase {
2828

29-
public void testRepositoryThrottlingStats() throws Exception {
29+
public void testRepositorySnapshotStats() throws Exception {
3030

3131
logger.info("--> starting a node");
3232
internalCluster().startNode();

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,8 @@ static TransportVersion def(int id) {
329329
public static final TransportVersion PROJECT_STATE_REGISTRY_RECORDS_DELETIONS = def(9_113_0_00);
330330
public static final TransportVersion ESQL_SERIALIZE_TIMESERIES_FIELD_TYPE = def(9_114_0_00);
331331
public static final TransportVersion ML_INFERENCE_IBM_WATSONX_COMPLETION_ADDED = def(9_115_0_00);
332+
public static final TransportVersion EXTENDED_SNAPSHOT_STATS_IN_NODE_INFO = def(9_116_0_00);
333+
332334
/*
333335
* STOP! READ THIS FIRST! No, really,
334336
* ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _

‎server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -905,13 +905,7 @@ public List<RepositoryStatsSnapshot> repositoriesStats() {
905905

906906
public RepositoriesStats getRepositoriesThrottlingStats() {
907907
return new RepositoriesStats(
908-
getRepositories().stream()
909-
.collect(
910-
Collectors.toMap(
911-
r -> r.getMetadata().name(),
912-
r -> new RepositoriesStats.ThrottlingStats(r.getRestoreThrottleTimeInNanos(), r.getSnapshotThrottleTimeInNanos())
913-
)
914-
)
908+
getRepositories().stream().collect(Collectors.toMap(r -> r.getMetadata().name(), Repository::getSnapshotStats))
915909
);
916910
}
917911

‎server/src/main/java/org/elasticsearch/repositories/RepositoriesStats.java‎

Lines changed: 80 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.elasticsearch.common.io.stream.StreamInput;
1414
import org.elasticsearch.common.io.stream.StreamOutput;
1515
import org.elasticsearch.common.io.stream.Writeable;
16+
import org.elasticsearch.common.unit.ByteSizeValue;
1617
import org.elasticsearch.core.TimeValue;
1718
import org.elasticsearch.xcontent.ToXContentFragment;
1819
import org.elasticsearch.xcontent.ToXContentObject;
@@ -26,17 +27,17 @@
2627

2728
public class RepositoriesStats implements Writeable, ToXContentFragment {
2829

29-
private final Map<String, ThrottlingStats> repositoryThrottlingStats;
30+
private final Map<String, SnapshotStats> repositoryThrottlingStats;
3031

3132
public RepositoriesStats(StreamInput in) throws IOException {
3233
if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) {
33-
repositoryThrottlingStats = in.readMap(ThrottlingStats::new);
34+
repositoryThrottlingStats = in.readMap(SnapshotStats::readFrom);
3435
} else {
3536
repositoryThrottlingStats = new HashMap<>();
3637
}
3738
}
3839

39-
public RepositoriesStats(Map<String, ThrottlingStats> repositoryThrottlingStats) {
40+
public RepositoriesStats(Map<String, SnapshotStats> repositoryThrottlingStats) {
4041
this.repositoryThrottlingStats = new HashMap<>(repositoryThrottlingStats);
4142
}
4243

@@ -53,14 +54,44 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
5354
return builder;
5455
}
5556

56-
public Map<String, ThrottlingStats> getRepositoryThrottlingStats() {
57+
public Map<String, SnapshotStats> getRepositoryThrottlingStats() {
5758
return Collections.unmodifiableMap(repositoryThrottlingStats);
5859
}
5960

60-
public record ThrottlingStats(long totalReadThrottledNanos, long totalWriteThrottledNanos) implements ToXContentObject, Writeable {
61+
public record SnapshotStats(
62+
long shardSnapshotsStarted,
63+
long shardSnapshotsCompleted,
64+
long shardSnapshotsInProgress,
65+
long totalReadThrottledNanos,
66+
long totalWriteThrottledNanos,
67+
long numberOfBlobsUploaded,
68+
long numberOfBytesUploaded,
69+
long totalUploadTimeInNanos,
70+
long totalUploadReadTimeInNanos
71+
) implements ToXContentObject, Writeable {
6172

62-
ThrottlingStats(StreamInput in) throws IOException {
63-
this(in.readVLong(), in.readVLong());
73+
public static SnapshotStats readFrom(StreamInput in) throws IOException {
74+
final long totalReadThrottledNanos = in.readVLong();
75+
final long totalWriteThrottledNanos = in.readVLong();
76+
if (in.getTransportVersion().onOrAfter(TransportVersions.EXTENDED_SNAPSHOT_STATS_IN_NODE_INFO)) {
77+
return new SnapshotStats(
78+
in.readLong(),
79+
in.readLong(),
80+
in.readLong(),
81+
totalReadThrottledNanos,
82+
totalWriteThrottledNanos,
83+
in.readLong(),
84+
in.readLong(),
85+
in.readLong(),
86+
in.readLong()
87+
);
88+
} else {
89+
return new SnapshotStats(totalReadThrottledNanos, totalWriteThrottledNanos);
90+
}
91+
}
92+
93+
public SnapshotStats(long totalReadThrottledNanos, long totalWriteThrottledNanos) {
94+
this(-1, -1, -1, totalReadThrottledNanos, totalWriteThrottledNanos, -1, -1, -1, -1);
6495
}
6596

6697
@Override
@@ -72,6 +103,39 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
72103
}
73104
builder.field("total_read_throttled_time_nanos", totalReadThrottledNanos);
74105
builder.field("total_write_throttled_time_nanos", totalWriteThrottledNanos);
106+
if (shardSnapshotsStarted != -1) {
107+
builder.field("shard_snapshots_started", shardSnapshotsStarted);
108+
}
109+
if (shardSnapshotsCompleted != -1) {
110+
builder.field("shard_snapshots_completed", shardSnapshotsCompleted);
111+
}
112+
if (shardSnapshotsInProgress != -1) {
113+
builder.field("shard_snapshots_in_progress", shardSnapshotsInProgress);
114+
}
115+
if (numberOfBlobsUploaded != -1) {
116+
builder.field("blobs_uploaded", numberOfBlobsUploaded);
117+
}
118+
if (numberOfBytesUploaded != -1) {
119+
if (builder.humanReadable()) {
120+
builder.field("bytes_uploaded", ByteSizeValue.ofBytes(numberOfBytesUploaded));
121+
} else {
122+
builder.field("bytes_uploaded", numberOfBytesUploaded);
123+
}
124+
}
125+
if (totalUploadTimeInNanos != -1) {
126+
if (builder.humanReadable()) {
127+
builder.field("total_upload_time", TimeValue.timeValueNanos(totalUploadTimeInNanos));
128+
} else {
129+
builder.field("total_upload_time_in_nanos", totalUploadTimeInNanos);
130+
}
131+
}
132+
if (totalUploadReadTimeInNanos != -1) {
133+
if (builder.humanReadable()) {
134+
builder.field("total_read_time", TimeValue.timeValueNanos(totalUploadReadTimeInNanos));
135+
} else {
136+
builder.field("total_read_time_in_nanos", totalUploadReadTimeInNanos);
137+
}
138+
}
75139
builder.endObject();
76140
return builder;
77141
}
@@ -80,6 +144,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
80144
public void writeTo(StreamOutput out) throws IOException {
81145
out.writeVLong(totalReadThrottledNanos);
82146
out.writeVLong(totalWriteThrottledNanos);
147+
if (out.getTransportVersion().onOrAfter(TransportVersions.EXTENDED_SNAPSHOT_STATS_IN_NODE_INFO)) {
148+
out.writeLong(shardSnapshotsStarted);
149+
out.writeLong(shardSnapshotsCompleted);
150+
out.writeLong(shardSnapshotsInProgress);
151+
out.writeLong(numberOfBlobsUploaded);
152+
out.writeLong(numberOfBytesUploaded);
153+
out.writeLong(totalUploadTimeInNanos);
154+
out.writeLong(totalUploadReadTimeInNanos);
155+
}
83156
}
84157
}
85158
}

‎server/src/main/java/org/elasticsearch/repositories/Repository.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -356,4 +356,8 @@ static boolean assertSnapshotMetaThread() {
356356
default LongWithAttributes getShardSnapshotsInProgress() {
357357
return null;
358358
}
359+
360+
default RepositoriesStats.SnapshotStats getSnapshotStats() {
361+
return new RepositoriesStats.SnapshotStats(getRestoreThrottleTimeInNanos(), getSnapshotThrottleTimeInNanos());
362+
}
359363
}

‎server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@
110110
import org.elasticsearch.repositories.IndexId;
111111
import org.elasticsearch.repositories.IndexMetaDataGenerations;
112112
import org.elasticsearch.repositories.RepositoriesService;
113+
import org.elasticsearch.repositories.RepositoriesStats;
113114
import org.elasticsearch.repositories.Repository;
114115
import org.elasticsearch.repositories.RepositoryData;
115116
import org.elasticsearch.repositories.RepositoryData.SnapshotDetails;
@@ -4226,4 +4227,9 @@ protected Set<String> getExtraUsageFeatures() {
42264227
public LongWithAttributes getShardSnapshotsInProgress() {
42274228
return blobStoreSnapshotMetrics.getShardSnapshotsInProgress();
42284229
}
4230+
4231+
@Override
4232+
public RepositoriesStats.SnapshotStats getSnapshotStats() {
4233+
return blobStoreSnapshotMetrics.getSnapshotStats();
4234+
}
42294235
}

‎server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreSnapshotMetrics.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.elasticsearch.cluster.metadata.RepositoryMetadata;
1414
import org.elasticsearch.common.metrics.CounterMetric;
1515
import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
16+
import org.elasticsearch.repositories.RepositoriesStats;
1617
import org.elasticsearch.repositories.SnapshotMetrics;
1718
import org.elasticsearch.telemetry.metric.LongWithAttributes;
1819

@@ -90,4 +91,18 @@ public void incrementUploadReadTime(long readTimeInNanos) {
9091
public LongWithAttributes getShardSnapshotsInProgress() {
9192
return new LongWithAttributes(shardSnapshotsInProgress.count(), metricAttributes);
9293
}
94+
95+
public RepositoriesStats.SnapshotStats getSnapshotStats() {
96+
return new RepositoriesStats.SnapshotStats(
97+
numberOfShardSnapshotsStarted.count(),
98+
numberOfShardSnapshotsCompleted.count(),
99+
shardSnapshotsInProgress.count(),
100+
restoreRateLimitingTimeInNanos.count(),
101+
snapshotRateLimitingTimeInNanos.count(),
102+
numberOfBlobsUploaded.count(),
103+
numberOfBytesUploaded.count(),
104+
uploadTimeInNanos.count(),
105+
uploadReadTimeInNanos.count()
106+
);
107+
}
93108
}

‎server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1069,7 +1069,7 @@ public static NodeStats createNodeStats() {
10691069
);
10701070
}
10711071
RepositoriesStats repositoriesStats = new RepositoriesStats(
1072-
Map.of("test-repository", new RepositoriesStats.ThrottlingStats(100, 200))
1072+
Map.of("test-repository", new RepositoriesStats.SnapshotStats(100, 200))
10731073
);
10741074
NodeAllocationStats nodeAllocationStats = new NodeAllocationStats(
10751075
randomIntBetween(0, 10000),

0 commit comments

Comments
 (0)