Skip to content

Commit d49517b

Browse files
lukewhitinglukewhiting
authored
[8.x] Report Deprecated Indices That Are Flagged To Ignore Migration Reinde… (elastic#120709)
* Report Deprecated Indices That Are Flagged To Ignore Migration Reindex As A Warning (elastic#120629) * Add block state matching option to deprecation check predicate * Add new deprecation checks to warn on old indices with ignore reindex flag * Test for new deprecation checks * Update docs/changelog/120629.yaml * PR Changes - Remove leftover comment that's no longer true (cherry picked from commit 45ae071) * Update notice wording for 8.x
1 parent 6d2106d commit d49517b

File tree

10 files changed

+186
-21
lines changed

10 files changed

+186
-21
lines changed

‎docs/changelog/120629.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 120629
2+
summary: Report Deprecated Indices That Are Flagged To Ignore Migration Reindex As
3+
A Warning
4+
area: Data streams
5+
type: enhancement
6+
issues: []

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -20,29 +20,38 @@ public class DeprecatedIndexPredicate {
2020

2121
public static final IndexVersion MINIMUM_WRITEABLE_VERSION_AFTER_UPGRADE = IndexVersions.V_8_0_0;
2222

23-
/*
23+
/**
2424
* This predicate allows through only indices that were created with a previous lucene version, meaning that they need to be reindexed
25-
* in order to be writable in the _next_ lucene version.
25+
* in order to be writable in the _next_ lucene version. It excludes searchable snapshots as they are not writable.
2626
*
2727
* It ignores searchable snapshots as they are not writable.
28+
*
29+
* @param metadata the cluster metadata
30+
* @param filterToBlockedStatus if true, only indices that are write blocked will be returned,
31+
* if false, only those without a block are returned
32+
* @return a predicate that returns true for indices that need to be reindexed
2833
*/
29-
public static Predicate<Index> getReindexRequiredPredicate(Metadata metadata) {
34+
public static Predicate<Index> getReindexRequiredPredicate(Metadata metadata, boolean filterToBlockedStatus) {
3035
return index -> {
3136
IndexMetadata indexMetadata = metadata.index(index);
32-
return reindexRequired(indexMetadata);
37+
return reindexRequired(indexMetadata, filterToBlockedStatus);
3338
};
3439
}
3540

36-
public static boolean reindexRequired(IndexMetadata indexMetadata) {
41+
/**
42+
* This method check if the indices that were created with a previous lucene version, meaning that they need to be reindexed
43+
* in order to be writable in the _next_ lucene version. It excludes searchable snapshots as they are not writable.
44+
*
45+
* @param indexMetadata the index metadata
46+
* @param filterToBlockedStatus if true, only indices that are write blocked will be returned,
47+
* if false, only those without a block are returned
48+
* @return a predicate that returns true for indices that need to be reindexed
49+
*/
50+
public static boolean reindexRequired(IndexMetadata indexMetadata, boolean filterToBlockedStatus) {
3751
return creationVersionBeforeMinimumWritableVersion(indexMetadata)
3852
&& isNotSearchableSnapshot(indexMetadata)
3953
&& isNotClosed(indexMetadata)
40-
&& isNotVerifiedReadOnly(indexMetadata);
41-
}
42-
43-
private static boolean isNotVerifiedReadOnly(IndexMetadata indexMetadata) {
44-
// no need to check blocks.
45-
return MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.get(indexMetadata.getSettings()) == false;
54+
&& matchBlockedStatus(indexMetadata, filterToBlockedStatus);
4655
}
4756

4857
private static boolean isNotSearchableSnapshot(IndexMetadata indexMetadata) {
@@ -57,4 +66,7 @@ private static boolean isNotClosed(IndexMetadata indexMetadata) {
5766
return indexMetadata.getState().equals(IndexMetadata.State.CLOSE) == false;
5867
}
5968

69+
private static boolean matchBlockedStatus(IndexMetadata indexMetadata, boolean filterToBlockedStatus) {
70+
return MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.get(indexMetadata.getSettings()) == filterToBlockedStatus;
71+
}
6072
}

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecks.java‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,7 @@ public class DataStreamDeprecationChecks {
2424
static DeprecationIssue oldIndicesCheck(DataStream dataStream, ClusterState clusterState) {
2525
List<Index> backingIndices = dataStream.getIndices();
2626

27-
Set<String> indicesNeedingUpgrade = backingIndices.stream()
28-
.filter(DeprecatedIndexPredicate.getReindexRequiredPredicate(clusterState.metadata()))
29-
.map(Index::getName)
30-
.collect(Collectors.toUnmodifiableSet());
27+
Set<String> indicesNeedingUpgrade = getReindexRequiredIndices(backingIndices, clusterState, false);
3128

3229
if (indicesNeedingUpgrade.isEmpty() == false) {
3330
return new DeprecationIssue(
@@ -47,4 +44,40 @@ static DeprecationIssue oldIndicesCheck(DataStream dataStream, ClusterState clus
4744

4845
return null;
4946
}
47+
48+
static DeprecationIssue ignoredOldIndicesCheck(DataStream dataStream, ClusterState clusterState) {
49+
List<Index> backingIndices = dataStream.getIndices();
50+
51+
Set<String> ignoredIndices = getReindexRequiredIndices(backingIndices, clusterState, true);
52+
53+
if (ignoredIndices.isEmpty() == false) {
54+
return new DeprecationIssue(
55+
DeprecationIssue.Level.WARNING,
56+
"Old data stream with a compatibility version < 8.0 Have Been Ignored",
57+
"https://www.elastic.co/guide/en/elasticsearch/reference/current/migrating-8.0.html#breaking-changes-8.0",
58+
"This data stream has read only backing indices that were created before Elasticsearch 8.0.0 and have been marked as "
59+
+ "OK to remain read-only after upgrade",
60+
false,
61+
ofEntries(
62+
entry("reindex_required", true),
63+
entry("total_backing_indices", backingIndices.size()),
64+
entry("ignored_indices_requiring_upgrade_count", ignoredIndices.size()),
65+
entry("ignored_indices_requiring_upgrade", ignoredIndices)
66+
)
67+
);
68+
}
69+
70+
return null;
71+
}
72+
73+
private static Set<String> getReindexRequiredIndices(
74+
List<Index> backingIndices,
75+
ClusterState clusterState,
76+
boolean filterToBlockedStatus
77+
) {
78+
return backingIndices.stream()
79+
.filter(DeprecatedIndexPredicate.getReindexRequiredPredicate(clusterState.metadata(), filterToBlockedStatus))
80+
.map(Index::getName)
81+
.collect(Collectors.toUnmodifiableSet());
82+
}
5083
}

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ private DeprecationChecks() {}
9595

9696
static List<BiFunction<IndexMetadata, ClusterState, DeprecationIssue>> INDEX_SETTINGS_CHECKS = List.of(
9797
IndexDeprecationChecks::oldIndicesCheck,
98+
IndexDeprecationChecks::ignoredOldIndicesCheck,
9899
IndexDeprecationChecks::translogRetentionSettingCheck,
99100
IndexDeprecationChecks::checkIndexDataPath,
100101
IndexDeprecationChecks::storeTypeSettingCheck,
@@ -103,7 +104,8 @@ private DeprecationChecks() {}
103104
);
104105

105106
static List<BiFunction<DataStream, ClusterState, DeprecationIssue>> DATA_STREAM_CHECKS = List.of(
106-
DataStreamDeprecationChecks::oldIndicesCheck
107+
DataStreamDeprecationChecks::oldIndicesCheck,
108+
DataStreamDeprecationChecks::ignoredOldIndicesCheck
107109
);
108110

109111
/**

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ static DeprecationIssue oldIndicesCheck(IndexMetadata indexMetadata, ClusterStat
3636
// TODO: this check needs to be revised. It's trivially true right now.
3737
IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion();
3838
// We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks
39-
if (DeprecatedIndexPredicate.reindexRequired(indexMetadata) && isNotDataStreamIndex(indexMetadata, clusterState)) {
39+
if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, false) && isNotDataStreamIndex(indexMetadata, clusterState)) {
4040
return new DeprecationIssue(
4141
DeprecationIssue.Level.CRITICAL,
4242
"Old index with a compatibility version < 8.0",
@@ -49,6 +49,24 @@ static DeprecationIssue oldIndicesCheck(IndexMetadata indexMetadata, ClusterStat
4949
return null;
5050
}
5151

52+
static DeprecationIssue ignoredOldIndicesCheck(IndexMetadata indexMetadata, ClusterState clusterState) {
53+
IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion();
54+
// We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks
55+
if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, true) && isNotDataStreamIndex(indexMetadata, clusterState)) {
56+
return new DeprecationIssue(
57+
DeprecationIssue.Level.WARNING,
58+
"Old index with a compatibility version < 8.0 Has Been Ignored",
59+
"https://www.elastic.co/guide/en/elasticsearch/reference/current/migrating-8.0.html#breaking-changes-8.0",
60+
"This read-only index has version: "
61+
+ currentCompatibilityVersion.toReleaseVersion()
62+
+ " and will be supported as read-only in 9.0",
63+
false,
64+
Collections.singletonMap("reindex_required", true)
65+
);
66+
}
67+
return null;
68+
}
69+
5270
private static boolean isNotDataStreamIndex(IndexMetadata indexMetadata, ClusterState clusterState) {
5371
return clusterState.metadata().findDataStreams(indexMetadata.getIndex().getName()).isEmpty();
5472
}

‎x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecksTests.java‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.elasticsearch.cluster.metadata.DataStreamOptions;
1414
import org.elasticsearch.cluster.metadata.IndexMetadata;
1515
import org.elasticsearch.cluster.metadata.Metadata;
16+
import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
1617
import org.elasticsearch.common.settings.Settings;
1718
import org.elasticsearch.index.Index;
1819
import org.elasticsearch.index.IndexMode;
@@ -224,4 +225,75 @@ private Index createIndex(
224225
nameToIndexMetadata.put(indexMetadata.getIndex().getName(), indexMetadata);
225226
return indexMetadata.getIndex();
226227
}
228+
229+
public void testOldIndicesIgnoredWarningCheck() {
230+
int oldIndexCount = randomIntBetween(1, 100);
231+
int newIndexCount = randomIntBetween(1, 100);
232+
233+
List<Index> allIndices = new ArrayList<>();
234+
Map<String, IndexMetadata> nameToIndexMetadata = new HashMap<>();
235+
Set<String> expectedIndices = new HashSet<>();
236+
237+
for (int i = 0; i < oldIndexCount; i++) {
238+
Settings.Builder settings = settings(IndexVersion.fromId(7170099));
239+
240+
String indexName = "old-data-stream-index-" + i;
241+
settings.put(MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey(), true);
242+
expectedIndices.add(indexName);
243+
244+
Settings.Builder settingsBuilder = settings;
245+
IndexMetadata oldIndexMetadata = IndexMetadata.builder(indexName)
246+
.settings(settingsBuilder)
247+
.numberOfShards(1)
248+
.numberOfReplicas(0)
249+
.build();
250+
allIndices.add(oldIndexMetadata.getIndex());
251+
nameToIndexMetadata.put(oldIndexMetadata.getIndex().getName(), oldIndexMetadata);
252+
}
253+
254+
for (int i = 0; i < newIndexCount; i++) {
255+
Index newIndex = createNewIndex(i, false, nameToIndexMetadata);
256+
allIndices.add(newIndex);
257+
}
258+
259+
DataStream dataStream = new DataStream(
260+
randomAlphaOfLength(10),
261+
allIndices,
262+
randomLong() * -1L,
263+
Map.of(),
264+
randomBoolean(),
265+
false,
266+
false,
267+
randomBoolean(),
268+
randomFrom(IndexMode.values()),
269+
null,
270+
randomFrom(DataStreamOptions.EMPTY, DataStreamOptions.FAILURE_STORE_DISABLED, DataStreamOptions.FAILURE_STORE_ENABLED, null),
271+
List.of(),
272+
randomBoolean(),
273+
null
274+
);
275+
276+
Metadata metadata = Metadata.builder().indices(nameToIndexMetadata).build();
277+
ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build();
278+
279+
DeprecationIssue expected = new DeprecationIssue(
280+
DeprecationIssue.Level.WARNING,
281+
"Old data stream with a compatibility version < 8.0 Have Been Ignored",
282+
"https://www.elastic.co/guide/en/elasticsearch/reference/current/migrating-8.0.html#breaking-changes-8.0",
283+
"This data stream has read only backing indices that were created before Elasticsearch 8.0.0 and have been marked as "
284+
+ "OK to remain read-only after upgrade",
285+
false,
286+
ofEntries(
287+
entry("reindex_required", true),
288+
entry("total_backing_indices", oldIndexCount + newIndexCount),
289+
entry("ignored_indices_requiring_upgrade_count", expectedIndices.size()),
290+
entry("ignored_indices_requiring_upgrade", expectedIndices)
291+
)
292+
);
293+
294+
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(DATA_STREAM_CHECKS, c -> c.apply(dataStream, clusterState));
295+
296+
assertThat(issues, equalTo(singletonList(expected)));
297+
}
298+
227299
}

‎x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.elasticsearch.cluster.metadata.DataStreamOptions;
1414
import org.elasticsearch.cluster.metadata.IndexMetadata;
1515
import org.elasticsearch.cluster.metadata.Metadata;
16+
import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
1617
import org.elasticsearch.common.collect.ImmutableOpenMap;
1718
import org.elasticsearch.common.settings.Settings;
1819
import org.elasticsearch.index.IndexMode;
@@ -132,6 +133,25 @@ public void testOldIndicesCheckClosedIgnored() {
132133
assertThat(issues, empty());
133134
}
134135

136+
public void testOldIndicesIgnoredWarningCheck() {
137+
IndexVersion createdWith = IndexVersion.fromId(7170099);
138+
Settings.Builder settings = settings(createdWith).put(MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey(), true);
139+
IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build();
140+
ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE)
141+
.metadata(Metadata.builder().put(indexMetadata, true))
142+
.build();
143+
DeprecationIssue expected = new DeprecationIssue(
144+
DeprecationIssue.Level.WARNING,
145+
"Old index with a compatibility version < 8.0 Has Been Ignored",
146+
"https://www.elastic.co/guide/en/elasticsearch/reference/current/migrating-8.0.html#breaking-changes-8.0",
147+
"This read-only index has version: " + createdWith.toReleaseVersion() + " and will be supported as read-only in 9.0",
148+
false,
149+
singletonMap("reindex_required", true)
150+
);
151+
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata, clusterState));
152+
assertEquals(singletonList(expected), issues);
153+
}
154+
135155
public void testTranslogRetentionSettings() {
136156
Settings.Builder settings = settings(IndexVersion.current());
137157
settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue());

‎x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ protected void doExecute(
118118
IndexMetadata sourceIndex = clusterService.state().getMetadata().index(sourceIndexName);
119119
Settings settingsBefore = sourceIndex.getSettings();
120120

121-
var hasOldVersion = DeprecatedIndexPredicate.getReindexRequiredPredicate(clusterService.state().metadata());
121+
var hasOldVersion = DeprecatedIndexPredicate.getReindexRequiredPredicate(clusterService.state().metadata(), false);
122122
if (hasOldVersion.test(sourceIndex.getIndex()) == false) {
123123
logger.warn(
124124
"Migrating index [{}] with version [{}] is unnecessary as its version is not before [{}]",

‎x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ protected void doExecute(Task task, ReindexDataStreamRequest request, ActionList
6868
return;
6969
}
7070
int totalIndices = dataStream.getIndices().size();
71-
int totalIndicesToBeUpgraded = (int) dataStream.getIndices().stream().filter(getReindexRequiredPredicate(metadata)).count();
71+
int totalIndicesToBeUpgraded = (int) dataStream.getIndices().stream().filter(getReindexRequiredPredicate(metadata, false)).count();
7272
ReindexDataStreamTaskParams params = new ReindexDataStreamTaskParams(
7373
sourceDataStreamName,
7474
transportService.getThreadPool().absoluteTimeInMillis(),

‎x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ protected void nodeOperation(
110110
List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = response.getDataStreams();
111111
if (dataStreamInfos.size() == 1) {
112112
DataStream dataStream = dataStreamInfos.get(0).getDataStream();
113-
if (getReindexRequiredPredicate(clusterService.state().metadata()).test(dataStream.getWriteIndex())) {
113+
if (getReindexRequiredPredicate(clusterService.state().metadata(), false).test(dataStream.getWriteIndex())) {
114114
RolloverRequest rolloverRequest = new RolloverRequest(sourceDataStream, null);
115115
rolloverRequest.setParentTask(taskId);
116116
client.execute(
@@ -156,7 +156,9 @@ private void reindexIndices(
156156
TaskId parentTaskId
157157
) {
158158
List<Index> indices = dataStream.getIndices();
159-
List<Index> indicesToBeReindexed = indices.stream().filter(getReindexRequiredPredicate(clusterService.state().metadata())).toList();
159+
List<Index> indicesToBeReindexed = indices.stream()
160+
.filter(getReindexRequiredPredicate(clusterService.state().metadata(), false))
161+
.toList();
160162
final ReindexDataStreamPersistentTaskState updatedState;
161163
if (params.totalIndices() != totalIndicesInDataStream
162164
|| params.totalIndicesToBeUpgraded() != indicesToBeReindexed.size()

0 commit comments

Comments
 (0)