Skip to content

Commit 818c259

Browse files
nicktindallnicktindall
committed
Fist pass on tests
1 parent 6e22dc6 commit 818c259

File tree

7 files changed

+208
-39
lines changed

7 files changed

+208
-39
lines changed

‎server/src/internalClusterTest/java/org/elasticsearch/repositories/InvalidRepositoryIT.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,16 @@ public UnstableRepository(
5959
BigArrays bigArrays,
6060
RecoverySettings recoverySettings
6161
) {
62-
super(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings);
62+
super(
63+
projectId,
64+
metadata,
65+
environment,
66+
namedXContentRegistry,
67+
clusterService,
68+
bigArrays,
69+
recoverySettings,
70+
SnapshotMetrics.NOOP
71+
);
6372
List<String> unstableNodes = UNSTABLE_NODES.get(metadata.settings());
6473
if (unstableNodes.contains(clusterService.getNodeName())) {
6574
throw new RepositoryException(metadata.name(), "Failed to create repository: current node is not stable");

‎server/src/internalClusterTest/java/org/elasticsearch/repositories/SnapshotMetricsIT.java‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.repositories;
11+
12+
import org.elasticsearch.common.util.CollectionUtils;
13+
import org.elasticsearch.plugins.Plugin;
14+
import org.elasticsearch.plugins.PluginsService;
15+
import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
16+
import org.elasticsearch.snapshots.mockstore.MockRepository;
17+
import org.elasticsearch.telemetry.Measurement;
18+
import org.elasticsearch.telemetry.TestTelemetryPlugin;
19+
import org.hamcrest.Matcher;
20+
21+
import java.util.Collection;
22+
import java.util.List;
23+
import java.util.stream.Stream;
24+
import java.util.stream.StreamSupport;
25+
26+
import static org.hamcrest.Matchers.allOf;
27+
import static org.hamcrest.Matchers.equalTo;
28+
import static org.hamcrest.Matchers.everyItem;
29+
import static org.hamcrest.Matchers.greaterThan;
30+
import static org.hamcrest.Matchers.hasItem;
31+
import static org.hamcrest.Matchers.lessThan;
32+
33+
public class SnapshotMetricsIT extends AbstractSnapshotIntegTestCase {
34+
35+
@Override
36+
protected Collection<Class<? extends Plugin>> nodePlugins() {
37+
return CollectionUtils.appendToCopy(super.nodePlugins(), TestTelemetryPlugin.class);
38+
}
39+
40+
public void testUpdateRepository() throws Exception {
41+
final String repositoryName = randomIdentifier();
42+
43+
createRepository(repositoryName, "mock");
44+
45+
final String indexName = randomIdentifier();
46+
final int numShards = randomIntBetween(1, 10);
47+
final int numReplicas = randomIntBetween(0, 1);
48+
createIndex(indexName, numShards, numReplicas);
49+
50+
indexRandom(true, indexName, randomIntBetween(100, 300));
51+
52+
// Block the snapshot to test "snapshot shards in progress"
53+
MockRepository repository = asInstanceOf(MockRepository.class, getRepositoryOnMaster(repositoryName));
54+
repository.blockOnDataFiles();
55+
final String snapshotName = randomIdentifier();
56+
final long beforeCreateSnapshotNanos = System.nanoTime();
57+
try {
58+
clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repositoryName, snapshotName)
59+
.setIndices(indexName)
60+
.setWaitForCompletion(false)
61+
.get();
62+
63+
waitForBlockOnAnyDataNode(repositoryName);
64+
collectMetrics();
65+
assertShardsInProgressMetricIs(hasItem(greaterThan(0L)));
66+
} finally {
67+
repository.unblock();
68+
}
69+
70+
// wait for snapshot to finish to test the other metrics
71+
awaitNumberOfSnapshotsInProgress(0);
72+
final long snapshotElapsedTime = System.nanoTime() - beforeCreateSnapshotNanos;
73+
collectMetrics();
74+
75+
// sanity check blobs and bytes metrics
76+
assertThat(getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOT_BLOBS_UPLOADED), greaterThan(0L));
77+
assertThat(getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOT_BYTES_UPLOADED), greaterThan(0L));
78+
79+
// sanity check duration values
80+
final long upperBoundTimeSpentOnSnapshotThings = internalCluster().numDataNodes() * snapshotElapsedTime;
81+
assertThat(
82+
getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOT_UPLOAD_DURATION),
83+
allOf(greaterThan(0L), lessThan(upperBoundTimeSpentOnSnapshotThings))
84+
);
85+
assertThat(
86+
getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOT_UPLOAD_READ_DURATION),
87+
allOf(greaterThan(0L), lessThan(upperBoundTimeSpentOnSnapshotThings))
88+
);
89+
90+
assertThat(getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOT_SHARDS_STARTED), equalTo((long) numShards));
91+
assertThat(getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOT_SHARDS_COMPLETED), equalTo((long) numShards));
92+
93+
assertShardsInProgressMetricIs(everyItem(equalTo(0L)));
94+
}
95+
96+
private static void assertShardsInProgressMetricIs(Matcher<? super List<Long>> matcher) {
97+
final List<Long> values = allTestTelemetryPlugins().map(testTelemetryPlugin -> {
98+
final List<Measurement> longGaugeMeasurement = testTelemetryPlugin.getLongGaugeMeasurement(
99+
SnapshotMetrics.SNAPSHOT_SHARDS_IN_PROGRESS
100+
);
101+
return longGaugeMeasurement.getLast().getLong();
102+
}).toList();
103+
assertThat(values, matcher);
104+
}
105+
106+
private static void collectMetrics() {
107+
allTestTelemetryPlugins().forEach(TestTelemetryPlugin::collect);
108+
}
109+
110+
private long getTotalClusterLongCounterValue(String metricName) {
111+
return allTestTelemetryPlugins().flatMap(testTelemetryPlugin -> testTelemetryPlugin.getLongCounterMeasurement(metricName).stream())
112+
.mapToLong(Measurement::getLong)
113+
.sum();
114+
}
115+
116+
private static Stream<TestTelemetryPlugin> allTestTelemetryPlugins() {
117+
return StreamSupport.stream(internalCluster().getDataNodeInstances(PluginsService.class).spliterator(), false)
118+
.flatMap(pluginsService -> pluginsService.filterPlugins(TestTelemetryPlugin.class));
119+
}
120+
}

‎server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.elasticsearch.repositories.RepositoriesService;
2727
import org.elasticsearch.repositories.Repository;
2828
import org.elasticsearch.repositories.RepositoryData;
29+
import org.elasticsearch.repositories.SnapshotMetrics;
2930
import org.elasticsearch.rest.RestStatus;
3031
import org.elasticsearch.snapshots.mockstore.MockRepository;
3132
import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -185,7 +186,16 @@ public CountingMockRepository(
185186
BigArrays bigArrays,
186187
RecoverySettings recoverySettings
187188
) {
188-
super(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings);
189+
super(
190+
projectId,
191+
metadata,
192+
environment,
193+
namedXContentRegistry,
194+
clusterService,
195+
bigArrays,
196+
recoverySettings,
197+
SnapshotMetrics.NOOP
198+
);
189199
}
190200

191201
@Override

‎server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
package org.elasticsearch.repositories;
1111

1212
import org.elasticsearch.client.internal.node.NodeClient;
13+
import org.elasticsearch.cluster.metadata.ProjectId;
14+
import org.elasticsearch.cluster.metadata.RepositoryMetadata;
1315
import org.elasticsearch.cluster.service.ClusterService;
1416
import org.elasticsearch.common.settings.Settings;
1517
import org.elasticsearch.common.util.BigArrays;
@@ -52,18 +54,22 @@ public RepositoriesModule(
5254
) {
5355
final RepositoriesMetrics repositoriesMetrics = new RepositoriesMetrics(telemetryProvider.getMeterRegistry());
5456
Map<String, Repository.Factory> factories = new HashMap<>();
55-
factories.put(
56-
FsRepository.TYPE,
57-
(projectId, metadata) -> new FsRepository(
58-
projectId,
59-
metadata,
60-
env,
61-
namedXContentRegistry,
62-
clusterService,
63-
bigArrays,
64-
recoverySettings
65-
)
66-
);
57+
factories.put(FsRepository.TYPE, new Repository.SnapshotMetricsFactory() {
58+
59+
@Override
60+
public Repository create(ProjectId projectId, RepositoryMetadata metadata, SnapshotMetrics snapshotMetrics) {
61+
return new FsRepository(
62+
projectId,
63+
metadata,
64+
env,
65+
namedXContentRegistry,
66+
clusterService,
67+
bigArrays,
68+
recoverySettings,
69+
snapshotMetrics
70+
);
71+
}
72+
});
6773

6874
for (RepositoryPlugin repoPlugin : repoPlugins) {
6975
Map<String, Repository.Factory> newRepoTypes = repoPlugin.getRepositories(

‎server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -85,16 +85,23 @@ public FsRepository(
8585
BigArrays bigArrays,
8686
RecoverySettings recoverySettings
8787
) {
88-
super(
89-
projectId,
90-
metadata,
91-
namedXContentRegistry,
92-
clusterService,
93-
bigArrays,
94-
recoverySettings,
95-
BlobPath.EMPTY,
96-
SnapshotMetrics.NOOP
97-
);
88+
this(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings, SnapshotMetrics.NOOP);
89+
}
90+
91+
/**
92+
* Constructs a shared file system repository.
93+
*/
94+
public FsRepository(
95+
ProjectId projectId,
96+
RepositoryMetadata metadata,
97+
Environment environment,
98+
NamedXContentRegistry namedXContentRegistry,
99+
ClusterService clusterService,
100+
BigArrays bigArrays,
101+
RecoverySettings recoverySettings,
102+
SnapshotMetrics snapshotMetrics
103+
) {
104+
super(projectId, metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, BlobPath.EMPTY, snapshotMetrics);
98105
this.environment = environment;
99106
String location = REPOSITORIES_LOCATION_SETTING.get(metadata.settings());
100107
if (location.isEmpty()) {

‎test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java‎

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import org.elasticsearch.plugins.RepositoryPlugin;
4141
import org.elasticsearch.repositories.RepositoriesMetrics;
4242
import org.elasticsearch.repositories.Repository;
43+
import org.elasticsearch.repositories.SnapshotMetrics;
4344
import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
4445
import org.elasticsearch.repositories.fs.FsRepository;
4546
import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -88,18 +89,22 @@ public Map<String, Repository.Factory> getRepositories(
8889
RecoverySettings recoverySettings,
8990
RepositoriesMetrics repositoriesMetrics
9091
) {
91-
return Collections.singletonMap(
92-
"mock",
93-
(projectId, metadata) -> new MockRepository(
94-
projectId,
95-
metadata,
96-
env,
97-
namedXContentRegistry,
98-
clusterService,
99-
bigArrays,
100-
recoverySettings
101-
)
102-
);
92+
return Collections.singletonMap("mock", new SnapshotMetricsFactory() {
93+
94+
@Override
95+
public Repository create(ProjectId projectId, RepositoryMetadata metadata, SnapshotMetrics snapshotMetrics) {
96+
return new MockRepository(
97+
projectId,
98+
metadata,
99+
env,
100+
namedXContentRegistry,
101+
clusterService,
102+
bigArrays,
103+
recoverySettings,
104+
snapshotMetrics
105+
);
106+
}
107+
});
103108
}
104109

105110
@Override
@@ -191,7 +196,8 @@ public MockRepository(
191196
NamedXContentRegistry namedXContentRegistry,
192197
ClusterService clusterService,
193198
BigArrays bigArrays,
194-
RecoverySettings recoverySettings
199+
RecoverySettings recoverySettings,
200+
SnapshotMetrics snapshotMetrics
195201
) {
196202
super(
197203
projectId,
@@ -200,7 +206,8 @@ public MockRepository(
200206
namedXContentRegistry,
201207
clusterService,
202208
bigArrays,
203-
recoverySettings
209+
recoverySettings,
210+
snapshotMetrics
204211
);
205212
randomControlIOExceptionRate = metadata.settings().getAsDouble("random_control_io_exception_rate", 0.0);
206213
randomDataFileIOExceptionRate = metadata.settings().getAsDouble("random_data_file_io_exception_rate", 0.0);

‎x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotDiskThresholdIntegTests.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.elasticsearch.repositories.RepositoriesMetrics;
4040
import org.elasticsearch.repositories.RepositoriesService;
4141
import org.elasticsearch.repositories.Repository;
42+
import org.elasticsearch.repositories.SnapshotMetrics;
4243
import org.elasticsearch.repositories.fs.FsRepository;
4344
import org.elasticsearch.snapshots.SnapshotId;
4445
import org.elasticsearch.snapshots.SnapshotState;
@@ -407,7 +408,16 @@ public CustomMockRepository(
407408
BigArrays bigArrays,
408409
RecoverySettings recoverySettings
409410
) {
410-
super(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings);
411+
super(
412+
projectId,
413+
metadata,
414+
environment,
415+
namedXContentRegistry,
416+
clusterService,
417+
bigArrays,
418+
recoverySettings,
419+
SnapshotMetrics.NOOP
420+
);
411421
}
412422

413423
private void unlockRestore() {

0 commit comments

Comments
 (0)