Skip to content

Commit 7b7cd1f

Browse files
tlrxtlrx
authored
Support recovery for closed shard in N-2 version (#121141)
Allow shard recovery for shards in version N-2 that have been verified before being closed, but not verified as read-only, in 7.x or 8.x. Reopening such closed indices automatically adds an index.blocks.write. Requires #120595 for the 8.x changes. Closes ES-10320 Closes #121170 Closes #121171
1 parent 49a20c1 commit 7b7cd1f

File tree

7 files changed

+184
-68
lines changed

7 files changed

+184
-68
lines changed

‎muted-tests.yml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -305,12 +305,6 @@ tests:
305305
- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
306306
method: testProfileIndexAutoCreation
307307
issue: https://github.com/elastic/elasticsearch/issues/120987
308-
- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
309-
method: testRestoreIndex {p0=[9.0.0, 9.0.0, 8.18.0]}
310-
issue: https://github.com/elastic/elasticsearch/issues/121170
311-
- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
312-
method: testRestoreIndex {p0=[9.0.0, 8.18.0, 8.18.0]}
313-
issue: https://github.com/elastic/elasticsearch/issues/121171
314308
- class: org.elasticsearch.xpack.security.FileSettingsRoleMappingsRestartIT
315309
method: testFileSettingsReprocessedOnRestartWithoutVersionChange
316310
issue: https://github.com/elastic/elasticsearch/issues/120964

‎qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartLuceneIndexCompatibilityIT.java‎

Lines changed: 76 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,10 @@
99

1010
package org.elasticsearch.lucene;
1111

12-
import org.elasticsearch.client.ResponseException;
1312
import org.elasticsearch.cluster.metadata.IndexMetadata;
1413
import org.elasticsearch.common.settings.Settings;
1514
import org.elasticsearch.index.IndexSettings;
15+
import org.elasticsearch.index.translog.Translog;
1616
import org.elasticsearch.repositories.fs.FsRepository;
1717
import org.elasticsearch.test.cluster.util.Version;
1818

@@ -22,10 +22,10 @@
2222
import static org.elasticsearch.cluster.metadata.MetadataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING;
2323
import static org.elasticsearch.cluster.metadata.MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING;
2424
import static org.hamcrest.Matchers.contains;
25-
import static org.hamcrest.Matchers.containsString;
2625
import static org.hamcrest.Matchers.either;
2726
import static org.hamcrest.Matchers.equalTo;
2827
import static org.hamcrest.Matchers.is;
28+
import static org.hamcrest.Matchers.not;
2929

3030
public class FullClusterRestartLuceneIndexCompatibilityIT extends FullClusterRestartIndexCompatibilityTestCase {
3131

@@ -51,7 +51,6 @@ public void testIndexUpgrade() throws Exception {
5151
Settings.builder()
5252
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
5353
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, randomInt(2))
54-
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
5554
.build()
5655
);
5756
indexDocs(index, numDocs);
@@ -111,12 +110,10 @@ public void testIndexUpgrade() throws Exception {
111110
.putNull(IndexMetadata.APIBlock.WRITE.settingName())
112111
.putNull(IndexMetadata.APIBlock.READ_ONLY.settingName())
113112
);
114-
logger.debug("--> but attempts to re-opening [{}] should fail due to the missing block", index);
115-
var ex = expectThrows(ResponseException.class, () -> openIndex(index));
116-
assertThat(ex.getMessage(), containsString("must be marked as read-only"));
117113

118-
// TODO this could be randomized once we support recovering verified-before-close closed indices with no write/ro block
119-
addIndexBlock(index, IndexMetadata.APIBlock.WRITE);
114+
assertThat(indexBlocks(index), contains(INDEX_CLOSED_BLOCK));
115+
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(true));
116+
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
120117
}
121118

122119
var block = indexBlocks(index).stream().filter(c -> c.equals(INDEX_WRITE_BLOCK) || c.equals(INDEX_READ_ONLY_BLOCK)).findFirst();
@@ -128,11 +125,11 @@ public void testIndexUpgrade() throws Exception {
128125
.putNull(IndexMetadata.APIBlock.READ_ONLY.settingName())
129126
.put(IndexMetadata.APIBlock.WRITE.settingName(), true)
130127
);
131-
}
132128

133-
assertThat(indexBlocks(index), isClosed ? contains(INDEX_CLOSED_BLOCK, INDEX_WRITE_BLOCK) : contains(INDEX_WRITE_BLOCK));
134-
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(isClosed));
135-
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
129+
assertThat(indexBlocks(index), isClosed ? contains(INDEX_CLOSED_BLOCK, INDEX_WRITE_BLOCK) : contains(INDEX_WRITE_BLOCK));
130+
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(isClosed));
131+
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
132+
}
136133

137134
var numberOfReplicas = getNumberOfReplicas(index);
138135
if (0 < numberOfReplicas) {
@@ -173,6 +170,71 @@ public void testIndexUpgrade() throws Exception {
173170
}
174171
}
175172

173+
/**
174+
* Creates an index on N-2, closes it on N-1 (without marking it as read-only), then upgrades to N.
175+
*/
176+
public void testClosedIndexUpgrade() throws Exception {
177+
final String index = suffix("index");
178+
final int numDocs = 2437;
179+
180+
if (isFullyUpgradedTo(VERSION_MINUS_2)) {
181+
createIndex(
182+
client(),
183+
index,
184+
Settings.builder()
185+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
186+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, randomInt(2))
187+
.put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), randomFrom(Translog.Durability.values()))
188+
.build()
189+
);
190+
indexDocs(index, numDocs);
191+
return;
192+
}
193+
194+
assertThat(indexVersion(index), equalTo(VERSION_MINUS_2));
195+
ensureGreen(index);
196+
197+
if (isIndexClosed(index) == false) {
198+
assertDocCount(client(), index, numDocs);
199+
}
200+
201+
if (isFullyUpgradedTo(VERSION_MINUS_1)) {
202+
logger.debug("--> [{}] closing index before upgrade without adding a read_only/write block", index);
203+
closeIndex(index);
204+
205+
assertThat(indexBlocks(index), contains(INDEX_CLOSED_BLOCK));
206+
assertThat(indexBlocks(index), not(contains(INDEX_WRITE_BLOCK)));
207+
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(true));
208+
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(false));
209+
return;
210+
}
211+
212+
if (isFullyUpgradedTo(VERSION_CURRENT)) {
213+
assertThat(indexBlocks(index), contains(INDEX_CLOSED_BLOCK));
214+
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(true));
215+
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(false));
216+
217+
logger.debug("--> re-opening index [{}] will add a write block", index);
218+
openIndex(index);
219+
ensureGreen(index);
220+
221+
assertThat(indexBlocks(index), contains(INDEX_WRITE_BLOCK));
222+
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(false));
223+
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
224+
assertDocCount(client(), index, numDocs);
225+
226+
logger.debug("--> closing index [{}]", index);
227+
closeIndex(index);
228+
ensureGreen(index);
229+
230+
assertThat(indexBlocks(index), contains(INDEX_CLOSED_BLOCK, INDEX_WRITE_BLOCK));
231+
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(true));
232+
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
233+
234+
deleteIndex(index);
235+
}
236+
}
237+
176238
/**
177239
* Creates an index on N-2, marks as read-only on N-1 and creates a snapshot, then restores the snapshot on N.
178240
*/
@@ -190,11 +252,7 @@ public void testRestoreIndex() throws Exception {
190252
createIndex(
191253
client(),
192254
index,
193-
Settings.builder()
194-
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
195-
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
196-
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
197-
.build()
255+
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
198256
);
199257

200258
logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);
@@ -272,11 +330,7 @@ public void testRestoreIndexOverClosedIndex() throws Exception {
272330
createIndex(
273331
client(),
274332
index,
275-
Settings.builder()
276-
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
277-
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
278-
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
279-
.build()
333+
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
280334
);
281335

282336
logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);

‎qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartSearchableSnapshotIndexCompatibilityIT.java‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111

1212
import org.elasticsearch.cluster.metadata.IndexMetadata;
1313
import org.elasticsearch.common.settings.Settings;
14-
import org.elasticsearch.index.IndexSettings;
1514
import org.elasticsearch.repositories.fs.FsRepository;
1615
import org.elasticsearch.test.cluster.util.Version;
1716

@@ -46,11 +45,7 @@ public void testSearchableSnapshot() throws Exception {
4645
createIndex(
4746
client(),
4847
index,
49-
Settings.builder()
50-
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
51-
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
52-
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
53-
.build()
48+
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
5449
);
5550

5651
logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);
@@ -125,11 +120,7 @@ public void testSearchableSnapshotUpgrade() throws Exception {
125120
createIndex(
126121
client(),
127122
index,
128-
Settings.builder()
129-
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
130-
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
131-
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
132-
.build()
123+
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
133124
);
134125

135126
logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);

‎qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeLuceneIndexCompatibilityTestCase.java‎

Lines changed: 86 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.elasticsearch.cluster.metadata.IndexMetadata;
1515
import org.elasticsearch.common.settings.Settings;
1616
import org.elasticsearch.index.IndexSettings;
17+
import org.elasticsearch.index.translog.Translog;
1718
import org.elasticsearch.repositories.fs.FsRepository;
1819
import org.elasticsearch.test.cluster.util.Version;
1920

@@ -54,11 +55,7 @@ public void testIndexUpgrade() throws Exception {
5455
createIndex(
5556
client(),
5657
index,
57-
Settings.builder()
58-
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
59-
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
60-
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
61-
.build()
58+
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
6259
);
6360
indexDocs(index, numDocs);
6461
return;
@@ -181,6 +178,75 @@ public void testIndexUpgrade() throws Exception {
181178
}
182179
}
183180

181+
/**
182+
* Creates an index on N-2, closes it on N-1 and then upgrades the cluster.
183+
*/
184+
public void testClosedIndexUpgrade() throws Exception {
185+
final String index = suffix("closed-rolling-upgraded");
186+
final int numDocs = 1543;
187+
188+
if (isFullyUpgradedTo(VERSION_MINUS_2)) {
189+
createIndex(
190+
client(),
191+
index,
192+
Settings.builder()
193+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
194+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
195+
.put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), randomFrom(Translog.Durability.values()))
196+
.build()
197+
);
198+
indexDocs(index, numDocs);
199+
return;
200+
}
201+
202+
assertThat(indexVersion(index), equalTo(VERSION_MINUS_2));
203+
ensureGreen(index);
204+
205+
if (isIndexClosed(index) == false) {
206+
assertDocCount(client(), index, numDocs);
207+
}
208+
209+
if (isFullyUpgradedTo(VERSION_MINUS_1)) {
210+
logger.debug("--> closing index [{}]", index);
211+
closeIndex(index);
212+
213+
assertThat(indexBlocks(index), contains(INDEX_CLOSED_BLOCK));
214+
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(true));
215+
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(false));
216+
return;
217+
}
218+
219+
if (nodesVersions().values().stream().anyMatch(v -> v.onOrAfter(VERSION_CURRENT))) {
220+
long upgradedNodes = nodesVersions().values().stream().filter(v -> v.onOrAfter(VERSION_CURRENT)).count();
221+
if (upgradedNodes == 1) {
222+
// Mixed cluster with 1 of the 3 nodes upgraded: the index hasn't been reopened yet
223+
assertThat(indexBlocks(index), contains(INDEX_CLOSED_BLOCK));
224+
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(true));
225+
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(false));
226+
227+
} else {
228+
// Index has been reopened at least once, it should have an additional write block and the verified-read-only setting
229+
assertThat(indexBlocks(index), contains(INDEX_CLOSED_BLOCK, INDEX_WRITE_BLOCK));
230+
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(true));
231+
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
232+
}
233+
234+
openIndex(index);
235+
ensureGreen(index);
236+
237+
assertThat(indexBlocks(index), contains(INDEX_WRITE_BLOCK));
238+
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(false));
239+
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
240+
assertDocCount(client(), index, numDocs);
241+
242+
updateRandomIndexSettings(index);
243+
updateRandomMappings(index);
244+
245+
closeIndex(index);
246+
ensureGreen(index);
247+
}
248+
}
249+
184250
/**
185251
* Creates an index on N-2, marks as read-only on N-1 and creates a snapshot, then restores the snapshot during rolling upgrades to N.
186252
*/
@@ -198,11 +264,7 @@ public void testRestoreIndex() throws Exception {
198264
createIndex(
199265
client(),
200266
index,
201-
Settings.builder()
202-
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
203-
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
204-
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
205-
.build()
267+
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
206268
);
207269

208270
logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);
@@ -253,19 +315,29 @@ public void testRestoreIndex() throws Exception {
253315
closeIndex(restoredIndex);
254316
ensureGreen(restoredIndex);
255317

318+
assertThat(indexBlocks(restoredIndex), contains(INDEX_CLOSED_BLOCK, INDEX_WRITE_BLOCK));
319+
assertIndexSetting(restoredIndex, VERIFIED_BEFORE_CLOSE_SETTING, is(true));
320+
assertIndexSetting(restoredIndex, VERIFIED_READ_ONLY_SETTING, is(true));
321+
256322
logger.debug("--> write API block can be removed on a closed index: INDEX_CLOSED_BLOCK already blocks writes");
257323
updateIndexSettings(restoredIndex, Settings.builder().putNull(IndexMetadata.APIBlock.WRITE.settingName()));
258324

259-
logger.debug("--> but attempts to re-opening [{}] should fail due to the missing block", restoredIndex);
260-
ex = expectThrows(ResponseException.class, () -> openIndex(restoredIndex));
261-
assertThat(ex.getMessage(), containsString("must be marked as read-only"));
325+
assertThat(indexBlocks(restoredIndex), contains(INDEX_CLOSED_BLOCK));
326+
assertIndexSetting(restoredIndex, VERIFIED_BEFORE_CLOSE_SETTING, is(true));
327+
assertIndexSetting(restoredIndex, VERIFIED_READ_ONLY_SETTING, is(true));
262328

263-
addIndexBlock(restoredIndex, IndexMetadata.APIBlock.WRITE);
329+
if (randomBoolean()) {
330+
addIndexBlock(restoredIndex, IndexMetadata.APIBlock.WRITE);
331+
}
264332

265333
logger.debug("--> re-opening restored index [{}]", restoredIndex);
266334
openIndex(restoredIndex);
267335
ensureGreen(restoredIndex);
268336

337+
assertThat(indexBlocks(restoredIndex), contains(INDEX_WRITE_BLOCK));
338+
assertIndexSetting(restoredIndex, VERIFIED_BEFORE_CLOSE_SETTING, is(false));
339+
assertIndexSetting(restoredIndex, VERIFIED_READ_ONLY_SETTING, is(true));
340+
269341
assertDocCount(client(), restoredIndex, numDocs);
270342

271343
logger.debug("--> deleting restored index [{}]", restoredIndex);

‎qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeSearchableSnapshotIndexCompatibilityIT.java‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
import org.elasticsearch.client.ResponseException;
1414
import org.elasticsearch.cluster.metadata.IndexMetadata;
1515
import org.elasticsearch.common.settings.Settings;
16-
import org.elasticsearch.index.IndexSettings;
1716
import org.elasticsearch.repositories.fs.FsRepository;
1817
import org.elasticsearch.test.cluster.util.Version;
1918

@@ -51,11 +50,7 @@ public void testMountSearchableSnapshot() throws Exception {
5150
createIndex(
5251
client(),
5352
index,
54-
Settings.builder()
55-
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
56-
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
57-
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
58-
.build()
53+
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
5954
);
6055

6156
logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);
@@ -122,11 +117,7 @@ public void testSearchableSnapshotUpgrade() throws Exception {
122117
createIndex(
123118
client(),
124119
index,
125-
Settings.builder()
126-
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
127-
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
128-
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
129-
.build()
120+
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
130121
);
131122

132123
logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);

0 commit comments

Comments
 (0)