Skip to content

Commit b3424ff

Browse files
masseykemasseyke
authored
Excluding data stream indices from IndexDeprecationChecks.oldIndicesCheck() (#116527)
1 parent f2b3882 commit b3424ff

File tree

5 files changed

+113
-28
lines changed

5 files changed

+113
-28
lines changed

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ private DeprecationChecks() {}
9090
NodeDeprecationChecks::checkWatcherBulkConcurrentRequestsSetting
9191
);
9292

93-
static List<Function<IndexMetadata, DeprecationIssue>> INDEX_SETTINGS_CHECKS = List.of(
93+
static List<BiFunction<IndexMetadata, ClusterState, DeprecationIssue>> INDEX_SETTINGS_CHECKS = List.of(
9494
IndexDeprecationChecks::oldIndicesCheck,
9595
IndexDeprecationChecks::translogRetentionSettingCheck,
9696
IndexDeprecationChecks::checkIndexDataPath,

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,7 @@ public static DeprecationInfoAction.Response from(
274274
IndexNameExpressionResolver indexNameExpressionResolver,
275275
Request request,
276276
NodesDeprecationCheckResponse nodeDeprecationResponse,
277-
List<Function<IndexMetadata, DeprecationIssue>> indexSettingsChecks,
277+
List<BiFunction<IndexMetadata, ClusterState, DeprecationIssue>> indexSettingsChecks,
278278
List<BiFunction<DataStream, ClusterState, DeprecationIssue>> dataStreamChecks,
279279
List<Function<ClusterState, DeprecationIssue>> clusterSettingsChecks,
280280
Map<String, List<DeprecationIssue>> pluginSettingIssues,
@@ -293,7 +293,10 @@ public static DeprecationInfoAction.Response from(
293293
Map<String, List<DeprecationIssue>> indexSettingsIssues = new HashMap<>();
294294
for (String concreteIndex : concreteIndexNames) {
295295
IndexMetadata indexMetadata = stateWithSkippedSettingsRemoved.getMetadata().index(concreteIndex);
296-
List<DeprecationIssue> singleIndexIssues = filterChecks(indexSettingsChecks, c -> c.apply(indexMetadata));
296+
List<DeprecationIssue> singleIndexIssues = filterChecks(
297+
indexSettingsChecks,
298+
c -> c.apply(indexMetadata, stateWithSkippedSettingsRemoved)
299+
);
297300
if (singleIndexIssues.size() > 0) {
298301
indexSettingsIssues.put(concreteIndex, singleIndexIssues);
299302
}

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
*/
77
package org.elasticsearch.xpack.deprecation;
88

9+
import org.elasticsearch.cluster.ClusterState;
910
import org.elasticsearch.cluster.metadata.IndexMetadata;
1011
import org.elasticsearch.cluster.metadata.MappingMetadata;
1112
import org.elasticsearch.common.time.DateFormatter;
@@ -30,14 +31,15 @@
3031
*/
3132
public class IndexDeprecationChecks {
3233

33-
static DeprecationIssue oldIndicesCheck(IndexMetadata indexMetadata) {
34+
static DeprecationIssue oldIndicesCheck(IndexMetadata indexMetadata, ClusterState clusterState) {
3435
// TODO: this check needs to be revised. It's trivially true right now.
3536
IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion();
36-
if (currentCompatibilityVersion.before(IndexVersions.V_7_0_0)) {
37+
// We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks
38+
if (currentCompatibilityVersion.before(IndexVersions.V_8_0_0) && isNotDataStreamIndex(indexMetadata, clusterState)) {
3739
return new DeprecationIssue(
3840
DeprecationIssue.Level.CRITICAL,
39-
"Old index with a compatibility version < 7.0",
40-
"https://www.elastic.co/guide/en/elasticsearch/reference/master/" + "breaking-changes-8.0.html",
41+
"Old index with a compatibility version < 8.0",
42+
"https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html",
4143
"This index has version: " + currentCompatibilityVersion.toReleaseVersion(),
4244
false,
4345
null
@@ -46,7 +48,11 @@ static DeprecationIssue oldIndicesCheck(IndexMetadata indexMetadata) {
4648
return null;
4749
}
4850

49-
static DeprecationIssue translogRetentionSettingCheck(IndexMetadata indexMetadata) {
51+
private static boolean isNotDataStreamIndex(IndexMetadata indexMetadata, ClusterState clusterState) {
52+
return clusterState.metadata().findDataStreams(indexMetadata.getIndex().getName()).isEmpty();
53+
}
54+
55+
static DeprecationIssue translogRetentionSettingCheck(IndexMetadata indexMetadata, ClusterState clusterState) {
5056
final boolean softDeletesEnabled = IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexMetadata.getSettings());
5157
if (softDeletesEnabled) {
5258
if (IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.exists(indexMetadata.getSettings())
@@ -73,7 +79,7 @@ static DeprecationIssue translogRetentionSettingCheck(IndexMetadata indexMetadat
7379
return null;
7480
}
7581

76-
static DeprecationIssue checkIndexDataPath(IndexMetadata indexMetadata) {
82+
static DeprecationIssue checkIndexDataPath(IndexMetadata indexMetadata, ClusterState clusterState) {
7783
if (IndexMetadata.INDEX_DATA_PATH_SETTING.exists(indexMetadata.getSettings())) {
7884
final String message = String.format(
7985
Locale.ROOT,
@@ -88,7 +94,7 @@ static DeprecationIssue checkIndexDataPath(IndexMetadata indexMetadata) {
8894
return null;
8995
}
9096

91-
static DeprecationIssue storeTypeSettingCheck(IndexMetadata indexMetadata) {
97+
static DeprecationIssue storeTypeSettingCheck(IndexMetadata indexMetadata, ClusterState clusterState) {
9298
final String storeType = IndexModule.INDEX_STORE_TYPE_SETTING.get(indexMetadata.getSettings());
9399
if (IndexModule.Type.SIMPLEFS.match(storeType)) {
94100
return new DeprecationIssue(
@@ -105,7 +111,7 @@ static DeprecationIssue storeTypeSettingCheck(IndexMetadata indexMetadata) {
105111
return null;
106112
}
107113

108-
static DeprecationIssue frozenIndexSettingCheck(IndexMetadata indexMetadata) {
114+
static DeprecationIssue frozenIndexSettingCheck(IndexMetadata indexMetadata, ClusterState clusterState) {
109115
Boolean isIndexFrozen = FrozenEngine.INDEX_FROZEN.get(indexMetadata.getSettings());
110116
if (Boolean.TRUE.equals(isIndexFrozen)) {
111117
String indexName = indexMetadata.getIndex().getName();
@@ -195,7 +201,7 @@ static List<String> findInPropertiesRecursively(
195201
return issues;
196202
}
197203

198-
static DeprecationIssue deprecatedCamelCasePattern(IndexMetadata indexMetadata) {
204+
static DeprecationIssue deprecatedCamelCasePattern(IndexMetadata indexMetadata, ClusterState clusterState) {
199205
List<String> fields = new ArrayList<>();
200206
fieldLevelMappingIssue(
201207
indexMetadata,

‎x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,9 @@ public void testFrom() throws IOException {
117117
boolean dataStreamIssueFound = randomBoolean();
118118
DeprecationIssue foundIssue = createTestDeprecationIssue();
119119
List<Function<ClusterState, DeprecationIssue>> clusterSettingsChecks = List.of((s) -> clusterIssueFound ? foundIssue : null);
120-
List<Function<IndexMetadata, DeprecationIssue>> indexSettingsChecks = List.of((idx) -> indexIssueFound ? foundIssue : null);
120+
List<BiFunction<IndexMetadata, ClusterState, DeprecationIssue>> indexSettingsChecks = List.of(
121+
(idx, cs) -> indexIssueFound ? foundIssue : null
122+
);
121123
List<BiFunction<DataStream, ClusterState, DeprecationIssue>> dataStreamChecks = List.of(
122124
(ds, cs) -> dataStreamIssueFound ? foundIssue : null
123125
);
@@ -211,7 +213,7 @@ public void testFromWithMergeableNodeIssues() throws IOException {
211213
DeprecationIssue foundIssue1 = createTestDeprecationIssue(metaMap1);
212214
DeprecationIssue foundIssue2 = createTestDeprecationIssue(foundIssue1, metaMap2);
213215
List<Function<ClusterState, DeprecationIssue>> clusterSettingsChecks = Collections.emptyList();
214-
List<Function<IndexMetadata, DeprecationIssue>> indexSettingsChecks = List.of((idx) -> null);
216+
List<BiFunction<IndexMetadata, ClusterState, DeprecationIssue>> indexSettingsChecks = List.of((idx, cs) -> null);
215217
List<BiFunction<DataStream, ClusterState, DeprecationIssue>> dataStreamChecks = List.of((ds, cs) -> null);
216218

217219
NodesDeprecationCheckResponse nodeDeprecationIssues = new NodesDeprecationCheckResponse(
@@ -276,10 +278,12 @@ public void testRemoveSkippedSettings() throws IOException {
276278
return null;
277279
}));
278280
AtomicReference<Settings> visibleIndexSettings = new AtomicReference<>();
279-
List<Function<IndexMetadata, DeprecationIssue>> indexSettingsChecks = Collections.unmodifiableList(Arrays.asList((idx) -> {
280-
visibleIndexSettings.set(idx.getSettings());
281-
return null;
282-
}));
281+
List<BiFunction<IndexMetadata, ClusterState, DeprecationIssue>> indexSettingsChecks = Collections.unmodifiableList(
282+
Arrays.asList((idx, cs) -> {
283+
visibleIndexSettings.set(idx.getSettings());
284+
return null;
285+
})
286+
);
283287
AtomicInteger backingIndicesCount = new AtomicInteger(0);
284288
List<BiFunction<DataStream, ClusterState, DeprecationIssue>> dataStreamChecks = Collections.unmodifiableList(
285289
Arrays.asList((ds, cs) -> {

‎x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java‎

Lines changed: 82 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,15 @@
77

88
package org.elasticsearch.xpack.deprecation;
99

10+
import org.elasticsearch.cluster.ClusterState;
11+
import org.elasticsearch.cluster.metadata.DataStream;
12+
import org.elasticsearch.cluster.metadata.DataStreamMetadata;
13+
import org.elasticsearch.cluster.metadata.DataStreamOptions;
1014
import org.elasticsearch.cluster.metadata.IndexMetadata;
15+
import org.elasticsearch.cluster.metadata.Metadata;
16+
import org.elasticsearch.common.collect.ImmutableOpenMap;
1117
import org.elasticsearch.common.settings.Settings;
18+
import org.elasticsearch.index.IndexMode;
1219
import org.elasticsearch.index.IndexModule;
1320
import org.elasticsearch.index.IndexSettings;
1421
import org.elasticsearch.index.IndexVersion;
@@ -19,39 +26,89 @@
1926

2027
import java.io.IOException;
2128
import java.util.List;
29+
import java.util.Map;
2230

2331
import static java.util.Collections.singletonList;
2432
import static org.elasticsearch.xpack.deprecation.DeprecationChecks.INDEX_SETTINGS_CHECKS;
2533
import static org.hamcrest.Matchers.empty;
34+
import static org.hamcrest.Matchers.equalTo;
2635
import static org.hamcrest.Matchers.hasItem;
2736
import static org.hamcrest.collection.IsIterableContainingInOrder.contains;
2837

2938
public class IndexDeprecationChecksTests extends ESTestCase {
3039
public void testOldIndicesCheck() {
31-
IndexVersion createdWith = IndexVersion.fromId(1000099);
40+
IndexVersion createdWith = IndexVersion.fromId(7170099);
3241
IndexMetadata indexMetadata = IndexMetadata.builder("test")
3342
.settings(settings(createdWith))
3443
.numberOfShards(1)
3544
.numberOfReplicas(0)
3645
.build();
46+
ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE)
47+
.metadata(Metadata.builder().put(indexMetadata, true))
48+
.build();
3749
DeprecationIssue expected = new DeprecationIssue(
3850
DeprecationIssue.Level.CRITICAL,
39-
"Old index with a compatibility version < 7.0",
40-
"https://www.elastic.co/guide/en/elasticsearch/reference/master/" + "breaking-changes-8.0.html",
51+
"Old index with a compatibility version < 8.0",
52+
"https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html",
4153
"This index has version: " + createdWith.toReleaseVersion(),
4254
false,
4355
null
4456
);
45-
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata));
57+
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata, clusterState));
4658
assertEquals(singletonList(expected), issues);
4759
}
4860

61+
public void testOldIndicesCheckDataStreamIndex() {
62+
IndexVersion createdWith = IndexVersion.fromId(7170099);
63+
IndexMetadata indexMetadata = IndexMetadata.builder(".ds-test")
64+
.settings(settings(createdWith).put("index.hidden", true))
65+
.numberOfShards(1)
66+
.numberOfReplicas(0)
67+
.build();
68+
DataStream dataStream = new DataStream(
69+
randomAlphaOfLength(10),
70+
List.of(indexMetadata.getIndex()),
71+
randomNegativeLong(),
72+
Map.of(),
73+
randomBoolean(),
74+
false,
75+
false,
76+
randomBoolean(),
77+
randomFrom(IndexMode.values()),
78+
null,
79+
randomFrom(DataStreamOptions.EMPTY, DataStreamOptions.FAILURE_STORE_DISABLED, DataStreamOptions.FAILURE_STORE_ENABLED, null),
80+
List.of(),
81+
randomBoolean(),
82+
null
83+
);
84+
ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE)
85+
.metadata(
86+
Metadata.builder()
87+
.put(indexMetadata, true)
88+
.customs(
89+
Map.of(
90+
DataStreamMetadata.TYPE,
91+
new DataStreamMetadata(
92+
ImmutableOpenMap.builder(Map.of("my-data-stream", dataStream)).build(),
93+
ImmutableOpenMap.of()
94+
)
95+
)
96+
)
97+
)
98+
.build();
99+
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata, clusterState));
100+
assertThat(issues.size(), equalTo(0));
101+
}
102+
49103
public void testTranslogRetentionSettings() {
50104
Settings.Builder settings = settings(IndexVersion.current());
51105
settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue());
52106
settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), between(1, 1024) + "b");
53107
IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build();
54-
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata));
108+
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(
109+
INDEX_SETTINGS_CHECKS,
110+
c -> c.apply(indexMetadata, ClusterState.EMPTY_STATE)
111+
);
55112
assertThat(
56113
issues,
57114
contains(
@@ -81,15 +138,21 @@ public void testDefaultTranslogRetentionSettings() {
81138
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false);
82139
}
83140
IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build();
84-
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata));
141+
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(
142+
INDEX_SETTINGS_CHECKS,
143+
c -> c.apply(indexMetadata, ClusterState.EMPTY_STATE)
144+
);
85145
assertThat(issues, empty());
86146
}
87147

88148
public void testIndexDataPathSetting() {
89149
Settings.Builder settings = settings(IndexVersion.current());
90150
settings.put(IndexMetadata.INDEX_DATA_PATH_SETTING.getKey(), createTempDir());
91151
IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build();
92-
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata));
152+
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(
153+
INDEX_SETTINGS_CHECKS,
154+
c -> c.apply(indexMetadata, ClusterState.EMPTY_STATE)
155+
);
93156
final String expectedUrl =
94157
"https://www.elastic.co/guide/en/elasticsearch/reference/7.13/breaking-changes-7.13.html#deprecate-shared-data-path-setting";
95158
assertThat(
@@ -111,7 +174,10 @@ public void testSimpleFSSetting() {
111174
Settings.Builder settings = settings(IndexVersion.current());
112175
settings.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "simplefs");
113176
IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build();
114-
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata));
177+
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(
178+
INDEX_SETTINGS_CHECKS,
179+
c -> c.apply(indexMetadata, ClusterState.EMPTY_STATE)
180+
);
115181
assertThat(
116182
issues,
117183
contains(
@@ -133,7 +199,10 @@ public void testFrozenIndex() {
133199
Settings.Builder settings = settings(IndexVersion.current());
134200
settings.put(FrozenEngine.INDEX_FROZEN.getKey(), true);
135201
IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build();
136-
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata));
202+
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(
203+
INDEX_SETTINGS_CHECKS,
204+
c -> c.apply(indexMetadata, ClusterState.EMPTY_STATE)
205+
);
137206
assertThat(
138207
issues,
139208
contains(
@@ -175,7 +244,10 @@ public void testCamelCaseDeprecation() throws IOException {
175244
false,
176245
null
177246
);
178-
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(simpleIndex));
247+
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(
248+
INDEX_SETTINGS_CHECKS,
249+
c -> c.apply(simpleIndex, ClusterState.EMPTY_STATE)
250+
);
179251
assertThat(issues, hasItem(expected));
180252
}
181253
}

0 commit comments

Comments
 (0)