Skip to content

Commit ca0f449

Browse files
grcevskigrcevski
authored
[TEST] Make FileSettingsService test more reliable (#89468) (#89521)
1 parent b9bf023 commit ca0f449

File tree

2 files changed

+49
-32
lines changed

2 files changed

+49
-32
lines changed

‎server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java‎

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.elasticsearch.cluster.service.ClusterService;
2121
import org.elasticsearch.common.settings.Settings;
2222
import org.elasticsearch.core.Strings;
23+
import org.elasticsearch.core.Tuple;
2324
import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction;
2425
import org.elasticsearch.test.ESIntegTestCase;
2526

@@ -35,7 +36,6 @@
3536
import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING;
3637
import static org.elasticsearch.test.NodeRoles.dataOnlyNode;
3738
import static org.hamcrest.Matchers.allOf;
38-
import static org.hamcrest.Matchers.contains;
3939
import static org.hamcrest.Matchers.containsString;
4040
import static org.hamcrest.Matchers.equalTo;
4141
import static org.hamcrest.Matchers.hasSize;
@@ -92,33 +92,36 @@ private void writeJSONFile(String node, String json) throws Exception {
9292
Files.move(tempFilePath, fileSettingsService.operatorSettingsFile(), StandardCopyOption.ATOMIC_MOVE);
9393
}
9494

95-
private CountDownLatch setupClusterStateListener(String node) {
95+
private Tuple<CountDownLatch, AtomicLong> setupClusterStateListener(String node) {
9696
ClusterService clusterService = internalCluster().clusterService(node);
9797
CountDownLatch savedClusterState = new CountDownLatch(1);
98+
AtomicLong metadataVersion = new AtomicLong(-1);
9899
clusterService.addListener(new ClusterStateListener() {
99100
@Override
100101
public void clusterChanged(ClusterChangedEvent event) {
101102
ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
102103
if (reservedState != null) {
103104
ReservedStateHandlerMetadata handlerMetadata = reservedState.handlers().get(ReservedClusterSettingsAction.NAME);
104-
if (handlerMetadata == null) {
105-
fail("Should've found cluster settings in this metadata");
105+
if (handlerMetadata != null && handlerMetadata.keys().contains("indices.recovery.max_bytes_per_sec")) {
106+
clusterService.removeListener(this);
107+
metadataVersion.set(event.state().metadata().version());
108+
savedClusterState.countDown();
106109
}
107-
assertThat(handlerMetadata.keys(), contains("indices.recovery.max_bytes_per_sec"));
108-
clusterService.removeListener(this);
109-
savedClusterState.countDown();
110110
}
111111
}
112112
});
113113

114-
return savedClusterState;
114+
return new Tuple<>(savedClusterState, metadataVersion);
115115
}
116116

117-
private void assertClusterStateSaveOK(CountDownLatch savedClusterState) throws Exception {
117+
private void assertClusterStateSaveOK(CountDownLatch savedClusterState, AtomicLong metadataVersion) throws Exception {
118118
boolean awaitSuccessful = savedClusterState.await(20, TimeUnit.SECONDS);
119119
assertTrue(awaitSuccessful);
120120

121-
final ClusterStateResponse clusterStateResponse = client().admin().cluster().state(new ClusterStateRequest()).actionGet();
121+
final ClusterStateResponse clusterStateResponse = client().admin()
122+
.cluster()
123+
.state(new ClusterStateRequest().waitForMetadataVersion(metadataVersion.get()))
124+
.actionGet();
122125

123126
assertThat(
124127
clusterStateResponse.getState().metadata().persistentSettings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()),
@@ -155,7 +158,7 @@ public void testSettingsApplied() throws Exception {
155158
assertFalse(dataFileSettingsService.watching());
156159

157160
writeJSONFile(masterNode, testJSON);
158-
assertClusterStateSaveOK(savedClusterState);
161+
assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2());
159162
}
160163

161164
public void testSettingsAppliedOnStart() throws Exception {
@@ -180,37 +183,42 @@ public void testSettingsAppliedOnStart() throws Exception {
180183
assertTrue(masterFileSettingsService.watching());
181184
assertFalse(dataFileSettingsService.watching());
182185

183-
assertClusterStateSaveOK(savedClusterState);
186+
assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2());
184187
}
185188

186-
private CountDownLatch setupClusterStateListenerForError(String node) {
189+
private Tuple<CountDownLatch, AtomicLong> setupClusterStateListenerForError(String node) {
187190
ClusterService clusterService = internalCluster().clusterService(node);
188191
CountDownLatch savedClusterState = new CountDownLatch(1);
192+
AtomicLong metadataVersion = new AtomicLong(-1);
189193
clusterService.addListener(new ClusterStateListener() {
190194
@Override
191195
public void clusterChanged(ClusterChangedEvent event) {
192196
ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
193-
if (reservedState != null) {
197+
if (reservedState != null && reservedState.errorMetadata() != null) {
194198
assertEquals(ReservedStateErrorMetadata.ErrorKind.PARSING, reservedState.errorMetadata().errorKind());
195199
assertThat(reservedState.errorMetadata().errors(), allOf(notNullValue(), hasSize(1)));
196200
assertThat(
197201
reservedState.errorMetadata().errors().get(0),
198202
containsString("Missing handler definition for content key [not_cluster_settings]")
199203
);
200204
clusterService.removeListener(this);
205+
metadataVersion.set(event.state().metadata().version());
201206
savedClusterState.countDown();
202207
}
203208
}
204209
});
205210

206-
return savedClusterState;
211+
return new Tuple<>(savedClusterState, metadataVersion);
207212
}
208213

209-
private void assertClusterStateNotSaved(CountDownLatch savedClusterState) throws Exception {
214+
private void assertClusterStateNotSaved(CountDownLatch savedClusterState, AtomicLong metadataVersion) throws Exception {
210215
boolean awaitSuccessful = savedClusterState.await(20, TimeUnit.SECONDS);
211216
assertTrue(awaitSuccessful);
212217

213-
final ClusterStateResponse clusterStateResponse = client().admin().cluster().state(new ClusterStateRequest()).actionGet();
218+
final ClusterStateResponse clusterStateResponse = client().admin()
219+
.cluster()
220+
.state(new ClusterStateRequest().waitForMetadataVersion(metadataVersion.get()))
221+
.actionGet();
214222

215223
assertThat(clusterStateResponse.getState().metadata().persistentSettings().get("search.allow_expensive_queries"), nullValue());
216224

@@ -240,6 +248,6 @@ public void testErrorSaved() throws Exception {
240248
assertFalse(dataFileSettingsService.watching());
241249

242250
writeJSONFile(masterNode, testErrorJSON);
243-
assertClusterStateNotSaved(savedClusterState);
251+
assertClusterStateNotSaved(savedClusterState.v1(), savedClusterState.v2());
244252
}
245253
}

‎server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/SnaphotsAndFileSettingsIT.java‎

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.elasticsearch.common.settings.Settings;
2121
import org.elasticsearch.core.Strings;
2222
import org.elasticsearch.core.TimeValue;
23+
import org.elasticsearch.core.Tuple;
2324
import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction;
2425
import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
2526
import org.elasticsearch.snapshots.SnapshotState;
@@ -85,9 +86,10 @@ private void writeJSONFile(String node, String json) throws Exception {
8586
Files.move(tempFilePath, fileSettingsService.operatorSettingsFile(), StandardCopyOption.ATOMIC_MOVE);
8687
}
8788

88-
private CountDownLatch setupClusterStateListener(String node) {
89+
private Tuple<CountDownLatch, AtomicLong> setupClusterStateListener(String node) {
8990
ClusterService clusterService = internalCluster().clusterService(node);
9091
CountDownLatch savedClusterState = new CountDownLatch(1);
92+
AtomicLong metadataVersion = new AtomicLong(-1);
9193
clusterService.addListener(new ClusterStateListener() {
9294
@Override
9395
public void clusterChanged(ClusterChangedEvent event) {
@@ -99,20 +101,21 @@ public void clusterChanged(ClusterChangedEvent event) {
99101
}
100102
if (handlerMetadata.keys().contains("indices.recovery.max_bytes_per_sec")) {
101103
clusterService.removeListener(this);
104+
metadataVersion.set(event.state().metadata().version());
102105
savedClusterState.countDown();
103106
}
104107
}
105108
}
106109
});
107110

108-
return savedClusterState;
111+
return new Tuple<>(savedClusterState, metadataVersion);
109112
}
110113

111-
private ClusterStateResponse assertClusterStateSaveOK(CountDownLatch savedClusterState) throws Exception {
114+
private ClusterStateResponse assertClusterStateSaveOK(CountDownLatch savedClusterState, AtomicLong metadataVersion) throws Exception {
112115
boolean awaitSuccessful = savedClusterState.await(20, TimeUnit.SECONDS);
113116
assertTrue(awaitSuccessful);
114117

115-
return clusterAdmin().state(new ClusterStateRequest()).actionGet();
118+
return clusterAdmin().state(new ClusterStateRequest().waitForMetadataVersion(metadataVersion.get())).actionGet();
116119
}
117120

118121
public void testRestoreWithRemovedFileSettings() throws Exception {
@@ -138,7 +141,7 @@ public void testRestoreWithRemovedFileSettings() throws Exception {
138141

139142
logger.info("--> write some file based settings, putting some reserved state");
140143
writeJSONFile(masterNode, testFileSettingsJSON);
141-
final ClusterStateResponse savedStateResponse = assertClusterStateSaveOK(savedClusterState);
144+
final ClusterStateResponse savedStateResponse = assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2());
142145
assertThat(
143146
savedStateResponse.getState().metadata().persistentSettings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()),
144147
equalTo("50mb")
@@ -195,26 +198,29 @@ public void testRestoreWithRemovedFileSettings() throws Exception {
195198
}
196199
}
197200

198-
private CountDownLatch removedReservedClusterStateListener(String node) {
201+
private Tuple<CountDownLatch, AtomicLong> removedReservedClusterStateListener(String node) {
199202
ClusterService clusterService = internalCluster().clusterService(node);
200203
CountDownLatch savedClusterState = new CountDownLatch(1);
204+
AtomicLong metadataVersion = new AtomicLong(-1);
201205
clusterService.addListener(new ClusterStateListener() {
202206
@Override
203207
public void clusterChanged(ClusterChangedEvent event) {
204208
ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
205209
if (reservedState != null && reservedState.version() == 0L) {
206210
clusterService.removeListener(this);
211+
metadataVersion.set(event.state().metadata().version());
207212
savedClusterState.countDown();
208213
}
209214
}
210215
});
211216

212-
return savedClusterState;
217+
return new Tuple<>(savedClusterState, metadataVersion);
213218
}
214219

215-
private CountDownLatch cleanedClusterStateListener(String node) {
220+
private Tuple<CountDownLatch, AtomicLong> cleanedClusterStateListener(String node) {
216221
ClusterService clusterService = internalCluster().clusterService(node);
217222
CountDownLatch savedClusterState = new CountDownLatch(1);
223+
AtomicLong metadataVersion = new AtomicLong(-1);
218224
clusterService.addListener(new ClusterStateListener() {
219225
@Override
220226
public void clusterChanged(ClusterChangedEvent event) {
@@ -226,13 +232,14 @@ public void clusterChanged(ClusterChangedEvent event) {
226232
}
227233
if (handlerMetadata.keys().isEmpty()) {
228234
clusterService.removeListener(this);
235+
metadataVersion.set(event.state().metadata().version());
229236
savedClusterState.countDown();
230237
}
231238
}
232239
}
233240
});
234241

235-
return savedClusterState;
242+
return new Tuple<>(savedClusterState, metadataVersion);
236243
}
237244

238245
public void testRestoreWithPersistedFileSettings() throws Exception {
@@ -258,7 +265,7 @@ public void testRestoreWithPersistedFileSettings() throws Exception {
258265

259266
logger.info("--> write some file based settings, putting some reserved state");
260267
writeJSONFile(masterNode, testFileSettingsJSON);
261-
final ClusterStateResponse savedStateResponse = assertClusterStateSaveOK(savedClusterState);
268+
final ClusterStateResponse savedStateResponse = assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2());
262269
assertThat(
263270
savedStateResponse.getState().metadata().persistentSettings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()),
264271
equalTo("50mb")
@@ -289,12 +296,14 @@ public void testRestoreWithPersistedFileSettings() throws Exception {
289296
// cluster state for file based settings, but instead we reset the version to 0 and 'touch' the operator file
290297
// so that it gets re-processed.
291298
logger.info("--> reserved state version will be reset to 0, because of snapshot restore");
292-
assertTrue(removedReservedState.await(20, TimeUnit.SECONDS));
299+
assertTrue(removedReservedState.v1().await(20, TimeUnit.SECONDS));
293300

294301
logger.info("--> reserved state would be restored");
295-
assertTrue(restoredReservedState.await(20, TimeUnit.SECONDS));
302+
assertTrue(restoredReservedState.v1().await(20, TimeUnit.SECONDS));
296303

297-
final ClusterStateResponse clusterStateResponse = clusterAdmin().state(new ClusterStateRequest().metadata(true)).actionGet();
304+
final ClusterStateResponse clusterStateResponse = clusterAdmin().state(
305+
new ClusterStateRequest().metadata(true).waitForMetadataVersion(restoredReservedState.v2().get())
306+
).actionGet();
298307

299308
assertNotNull(clusterStateResponse.getState().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE));
300309

@@ -313,7 +322,7 @@ public void testRestoreWithPersistedFileSettings() throws Exception {
313322

314323
logger.info("--> clear the file based settings");
315324
writeJSONFile(masterNode, emptyFileSettingsJSON);
316-
assertClusterStateSaveOK(cleanupReservedState);
325+
assertClusterStateSaveOK(cleanupReservedState.v1(), cleanupReservedState.v2());
317326
} finally {
318327
// cleanup
319328
assertAcked(

0 commit comments

Comments
 (0)