Skip to content

Commit 7c3d402

Browse files
masseykemasseyke
authored
Adding a deprecation info API warning for data streams with old indices (#116447) (#116626)
* Adding a deprecation info API warning for data streams with old indices (#116447) * removing use of a method not available in 8.x
1 parent f9077a0 commit 7c3d402

File tree

8 files changed

+295
-5
lines changed

8 files changed

+295
-5
lines changed

‎docs/changelog/116447.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 116447
2+
summary: Adding a deprecation info API warning for data streams with old indices
3+
area: Data streams
4+
type: enhancement
5+
issues: []

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,8 @@ static TransportVersion def(int id) {
191191
public static final TransportVersion LOGSDB_TELEMETRY_STATS = def(8_785_00_0);
192192
public static final TransportVersion KQL_QUERY_ADDED = def(8_786_00_0);
193193
public static final TransportVersion ROLE_MONITOR_STATS = def(8_787_00_0);
194-
public static final TransportVersion ADD_COMPATIBILITY_VERSIONS_TO_NODE_INFO = def(8_788_00_0);
194+
public static final TransportVersion DATA_STREAM_INDEX_VERSION_DEPRECATION_CHECK = def(8_788_00_0);
195+
public static final TransportVersion ADD_COMPATIBILITY_VERSIONS_TO_NODE_INFO = def(8_789_00_0);
195196

196197
/*
197198
* STOP! READ THIS FIRST! No, really,

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecks.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.deprecation;
9+
10+
import org.elasticsearch.cluster.ClusterState;
11+
import org.elasticsearch.cluster.metadata.DataStream;
12+
import org.elasticsearch.index.Index;
13+
import org.elasticsearch.index.IndexVersions;
14+
import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
15+
16+
import java.util.List;
17+
18+
import static java.util.Map.entry;
19+
import static java.util.Map.ofEntries;
20+
21+
public class DataStreamDeprecationChecks {
22+
static DeprecationIssue oldIndicesCheck(DataStream dataStream, ClusterState clusterState) {
23+
List<Index> backingIndices = dataStream.getIndices();
24+
boolean hasOldIndices = backingIndices.stream()
25+
.anyMatch(index -> clusterState.metadata().index(index).getCompatibilityVersion().before(IndexVersions.V_8_0_0));
26+
if (hasOldIndices) {
27+
long totalIndices = backingIndices.size();
28+
List<Index> oldIndices = backingIndices.stream()
29+
.filter(index -> clusterState.metadata().index(index).getCompatibilityVersion().before(IndexVersions.V_8_0_0))
30+
.toList();
31+
long totalOldIndices = oldIndices.size();
32+
long totalOldSearchableSnapshots = oldIndices.stream()
33+
.filter(index -> clusterState.metadata().index(index).isSearchableSnapshot())
34+
.count();
35+
long totalOldPartiallyMountedSearchableSnapshots = oldIndices.stream()
36+
.filter(index -> clusterState.metadata().index(index).isPartialSearchableSnapshot())
37+
.count();
38+
long totalOldFullyMountedSearchableSnapshots = totalOldSearchableSnapshots - totalOldPartiallyMountedSearchableSnapshots;
39+
return new DeprecationIssue(
40+
DeprecationIssue.Level.CRITICAL,
41+
"Old data stream with a compatibility version < 8.0",
42+
"https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html",
43+
"This data stream has backing indices that were created before Elasticsearch 8.0.0",
44+
false,
45+
ofEntries(
46+
entry(
47+
"backing_indices",
48+
ofEntries(
49+
entry("count", totalIndices),
50+
entry(
51+
"need_upgrading",
52+
ofEntries(
53+
entry("count", totalOldIndices),
54+
entry(
55+
"searchable_snapshots",
56+
ofEntries(
57+
entry("count", totalOldSearchableSnapshots),
58+
entry("fully_mounted", ofEntries(entry("count", totalOldFullyMountedSearchableSnapshots))),
59+
entry(
60+
"partially_mounted",
61+
ofEntries(entry("count", totalOldPartiallyMountedSearchableSnapshots))
62+
)
63+
)
64+
)
65+
)
66+
)
67+
)
68+
)
69+
)
70+
);
71+
}
72+
return null;
73+
}
74+
}

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
1010
import org.elasticsearch.cluster.ClusterState;
11+
import org.elasticsearch.cluster.metadata.DataStream;
1112
import org.elasticsearch.cluster.metadata.IndexMetadata;
1213
import org.elasticsearch.common.settings.Setting;
1314
import org.elasticsearch.common.settings.Settings;
@@ -16,6 +17,7 @@
1617

1718
import java.util.List;
1819
import java.util.Objects;
20+
import java.util.function.BiFunction;
1921
import java.util.function.Function;
2022
import java.util.stream.Collectors;
2123

@@ -98,6 +100,10 @@ private DeprecationChecks() {}
98100
IndexDeprecationChecks::deprecatedCamelCasePattern
99101
);
100102

103+
static List<BiFunction<DataStream, ClusterState, DeprecationIssue>> DATA_STREAM_CHECKS = List.of(
104+
DataStreamDeprecationChecks::oldIndicesCheck
105+
);
106+
101107
/**
102108
* helper utility function to reduce repeat of running a specific {@link List} of checks.
103109
*

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import org.elasticsearch.action.support.IndicesOptions;
1717
import org.elasticsearch.action.support.master.MasterNodeReadRequest;
1818
import org.elasticsearch.cluster.ClusterState;
19+
import org.elasticsearch.cluster.metadata.DataStream;
1920
import org.elasticsearch.cluster.metadata.IndexMetadata;
2021
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
2122
import org.elasticsearch.cluster.metadata.Metadata;
@@ -42,6 +43,7 @@
4243
import java.util.Map;
4344
import java.util.Objects;
4445
import java.util.Set;
46+
import java.util.function.BiFunction;
4547
import java.util.function.Function;
4648
import java.util.stream.Collectors;
4749

@@ -144,17 +146,23 @@ private static Map<DeprecationIssue, List<String>> getMergedIssuesToNodesMap(
144146
}
145147

146148
public static class Response extends ActionResponse implements ToXContentObject {
147-
static final Set<String> RESERVED_NAMES = Set.of("cluster_settings", "node_settings", "index_settings");
149+
static final Set<String> RESERVED_NAMES = Set.of("cluster_settings", "node_settings", "index_settings", "data_streams");
148150
private final List<DeprecationIssue> clusterSettingsIssues;
149151
private final List<DeprecationIssue> nodeSettingsIssues;
150152
private final Map<String, List<DeprecationIssue>> indexSettingsIssues;
153+
private final Map<String, List<DeprecationIssue>> dataStreamIssues;
151154
private final Map<String, List<DeprecationIssue>> pluginSettingsIssues;
152155

153156
public Response(StreamInput in) throws IOException {
154157
super(in);
155158
clusterSettingsIssues = in.readCollectionAsList(DeprecationIssue::new);
156159
nodeSettingsIssues = in.readCollectionAsList(DeprecationIssue::new);
157160
indexSettingsIssues = in.readMapOfLists(DeprecationIssue::new);
161+
if (in.getTransportVersion().onOrAfter(TransportVersions.DATA_STREAM_INDEX_VERSION_DEPRECATION_CHECK)) {
162+
dataStreamIssues = in.readMapOfLists(DeprecationIssue::new);
163+
} else {
164+
dataStreamIssues = Map.of();
165+
}
158166
if (in.getTransportVersion().before(TransportVersions.V_7_11_0)) {
159167
List<DeprecationIssue> mlIssues = in.readCollectionAsList(DeprecationIssue::new);
160168
pluginSettingsIssues = new HashMap<>();
@@ -168,11 +176,13 @@ public Response(
168176
List<DeprecationIssue> clusterSettingsIssues,
169177
List<DeprecationIssue> nodeSettingsIssues,
170178
Map<String, List<DeprecationIssue>> indexSettingsIssues,
179+
Map<String, List<DeprecationIssue>> dataStreamIssues,
171180
Map<String, List<DeprecationIssue>> pluginSettingsIssues
172181
) {
173182
this.clusterSettingsIssues = clusterSettingsIssues;
174183
this.nodeSettingsIssues = nodeSettingsIssues;
175184
this.indexSettingsIssues = indexSettingsIssues;
185+
this.dataStreamIssues = dataStreamIssues;
176186
Set<String> intersection = Sets.intersection(RESERVED_NAMES, pluginSettingsIssues.keySet());
177187
if (intersection.isEmpty() == false) {
178188
throw new ElasticsearchStatusException(
@@ -205,6 +215,9 @@ public void writeTo(StreamOutput out) throws IOException {
205215
out.writeCollection(clusterSettingsIssues);
206216
out.writeCollection(nodeSettingsIssues);
207217
out.writeMap(indexSettingsIssues, StreamOutput::writeCollection);
218+
if (out.getTransportVersion().onOrAfter(TransportVersions.DATA_STREAM_INDEX_VERSION_DEPRECATION_CHECK)) {
219+
out.writeMap(dataStreamIssues, StreamOutput::writeCollection);
220+
}
208221
if (out.getTransportVersion().before(TransportVersions.V_7_11_0)) {
209222
out.writeCollection(pluginSettingsIssues.getOrDefault("ml_settings", Collections.emptyList()));
210223
} else {
@@ -219,6 +232,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
219232
.array("node_settings", nodeSettingsIssues.toArray())
220233
.field("index_settings")
221234
.map(indexSettingsIssues)
235+
.field("data_streams")
236+
.map(dataStreamIssues)
222237
.mapContents(pluginSettingsIssues)
223238
.endObject();
224239
}
@@ -260,6 +275,7 @@ public static DeprecationInfoAction.Response from(
260275
Request request,
261276
NodesDeprecationCheckResponse nodeDeprecationResponse,
262277
List<Function<IndexMetadata, DeprecationIssue>> indexSettingsChecks,
278+
List<BiFunction<DataStream, ClusterState, DeprecationIssue>> dataStreamChecks,
263279
List<Function<ClusterState, DeprecationIssue>> clusterSettingsChecks,
264280
Map<String, List<DeprecationIssue>> pluginSettingIssues,
265281
List<String> skipTheseDeprecatedSettings
@@ -283,6 +299,19 @@ public static DeprecationInfoAction.Response from(
283299
}
284300
}
285301

302+
List<String> dataStreamNames = indexNameExpressionResolver.dataStreamNames(
303+
state,
304+
IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN
305+
);
306+
Map<String, List<DeprecationIssue>> dataStreamIssues = new HashMap<>();
307+
for (String dataStreamName : dataStreamNames) {
308+
DataStream dataStream = stateWithSkippedSettingsRemoved.metadata().dataStreams().get(dataStreamName);
309+
List<DeprecationIssue> issuesForSingleDataStream = filterChecks(dataStreamChecks, c -> c.apply(dataStream, state));
310+
if (issuesForSingleDataStream.isEmpty() == false) {
311+
dataStreamIssues.put(dataStreamName, issuesForSingleDataStream);
312+
}
313+
}
314+
286315
// WORKAROUND: move transform deprecation issues into cluster_settings
287316
List<DeprecationIssue> transformDeprecations = pluginSettingIssues.remove(
288317
TransformDeprecationChecker.TRANSFORM_DEPRECATION_KEY
@@ -291,7 +320,13 @@ public static DeprecationInfoAction.Response from(
291320
clusterSettingsIssues.addAll(transformDeprecations);
292321
}
293322

294-
return new DeprecationInfoAction.Response(clusterSettingsIssues, nodeSettingsIssues, indexSettingsIssues, pluginSettingIssues);
323+
return new DeprecationInfoAction.Response(
324+
clusterSettingsIssues,
325+
nodeSettingsIssues,
326+
indexSettingsIssues,
327+
dataStreamIssues,
328+
pluginSettingIssues
329+
);
295330
}
296331
}
297332

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import java.util.stream.Collectors;
3737

3838
import static org.elasticsearch.xpack.deprecation.DeprecationChecks.CLUSTER_SETTINGS_CHECKS;
39+
import static org.elasticsearch.xpack.deprecation.DeprecationChecks.DATA_STREAM_CHECKS;
3940
import static org.elasticsearch.xpack.deprecation.DeprecationChecks.INDEX_SETTINGS_CHECKS;
4041

4142
public class TransportDeprecationInfoAction extends TransportMasterNodeReadAction<
@@ -134,6 +135,7 @@ protected final void masterOperation(
134135
request,
135136
response,
136137
INDEX_SETTINGS_CHECKS,
138+
DATA_STREAM_CHECKS,
137139
CLUSTER_SETTINGS_CHECKS,
138140
deprecationIssues,
139141
skipTheseDeprecations

‎x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecksTests.java‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.deprecation;
9+
10+
import org.elasticsearch.cluster.ClusterName;
11+
import org.elasticsearch.cluster.ClusterState;
12+
import org.elasticsearch.cluster.metadata.DataStream;
13+
import org.elasticsearch.cluster.metadata.DataStreamOptions;
14+
import org.elasticsearch.cluster.metadata.IndexMetadata;
15+
import org.elasticsearch.cluster.metadata.Metadata;
16+
import org.elasticsearch.common.settings.Settings;
17+
import org.elasticsearch.index.Index;
18+
import org.elasticsearch.index.IndexMode;
19+
import org.elasticsearch.index.IndexVersion;
20+
import org.elasticsearch.test.ESTestCase;
21+
import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
22+
23+
import java.util.ArrayList;
24+
import java.util.HashMap;
25+
import java.util.List;
26+
import java.util.Map;
27+
28+
import static java.util.Collections.singletonList;
29+
import static org.elasticsearch.xpack.deprecation.DeprecationChecks.DATA_STREAM_CHECKS;
30+
import static org.hamcrest.Matchers.equalTo;
31+
32+
public class DataStreamDeprecationChecksTests extends ESTestCase {
33+
34+
public void testOldIndicesCheck() {
35+
long oldIndexCount = randomIntBetween(1, 100);
36+
long newIndexCount = randomIntBetween(1, 100);
37+
long oldSearchableSnapshotCount = 0;
38+
long oldFullyManagedSearchableSnapshotCount = 0;
39+
long oldPartiallyManagedSearchableSnapshotCount = 0;
40+
List<Index> allIndices = new ArrayList<>();
41+
Map<String, IndexMetadata> nameToIndexMetadata = new HashMap<>();
42+
for (int i = 0; i < oldIndexCount; i++) {
43+
Settings.Builder settingsBuilder = settings(IndexVersion.fromId(7170099));
44+
if (randomBoolean()) {
45+
settingsBuilder.put("index.store.type", "snapshot");
46+
if (randomBoolean()) {
47+
oldFullyManagedSearchableSnapshotCount++;
48+
} else {
49+
settingsBuilder.put("index.store.snapshot.partial", true);
50+
oldPartiallyManagedSearchableSnapshotCount++;
51+
}
52+
oldSearchableSnapshotCount++;
53+
}
54+
IndexMetadata oldIndexMetadata = IndexMetadata.builder("old-data-stream-index-" + i)
55+
.settings(settingsBuilder)
56+
.numberOfShards(1)
57+
.numberOfReplicas(0)
58+
.build();
59+
allIndices.add(oldIndexMetadata.getIndex());
60+
nameToIndexMetadata.put(oldIndexMetadata.getIndex().getName(), oldIndexMetadata);
61+
}
62+
for (int i = 0; i < newIndexCount; i++) {
63+
Settings.Builder settingsBuilder = settings(IndexVersion.current());
64+
if (randomBoolean()) {
65+
settingsBuilder.put("index.store.type", "snapshot");
66+
}
67+
IndexMetadata newIndexMetadata = IndexMetadata.builder("new-data-stream-index-" + i)
68+
.settings(settingsBuilder)
69+
.numberOfShards(1)
70+
.numberOfReplicas(0)
71+
.build();
72+
allIndices.add(newIndexMetadata.getIndex());
73+
nameToIndexMetadata.put(newIndexMetadata.getIndex().getName(), newIndexMetadata);
74+
}
75+
DataStream dataStream = new DataStream(
76+
randomAlphaOfLength(10),
77+
allIndices,
78+
randomNonNegativeLong(),
79+
Map.of(),
80+
randomBoolean(),
81+
false,
82+
false,
83+
randomBoolean(),
84+
randomFrom(IndexMode.values()),
85+
null,
86+
randomFrom(DataStreamOptions.EMPTY, DataStreamOptions.FAILURE_STORE_DISABLED, DataStreamOptions.FAILURE_STORE_ENABLED, null),
87+
List.of(),
88+
randomBoolean(),
89+
null
90+
);
91+
Metadata metadata = Metadata.builder().indices(nameToIndexMetadata).build();
92+
ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build();
93+
DeprecationIssue expected = new DeprecationIssue(
94+
DeprecationIssue.Level.CRITICAL,
95+
"Old data stream with a compatibility version < 8.0",
96+
"https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html",
97+
"This data stream has backing indices that were created before Elasticsearch 8.0.0",
98+
false,
99+
Map.of(
100+
"backing_indices",
101+
Map.of(
102+
"count",
103+
oldIndexCount + newIndexCount,
104+
"need_upgrading",
105+
Map.of(
106+
"count",
107+
oldIndexCount,
108+
"searchable_snapshots",
109+
Map.of(
110+
"count",
111+
oldSearchableSnapshotCount,
112+
"fully_mounted",
113+
Map.of("count", oldFullyManagedSearchableSnapshotCount),
114+
"partially_mounted",
115+
Map.of("count", oldPartiallyManagedSearchableSnapshotCount)
116+
)
117+
)
118+
)
119+
)
120+
);
121+
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(DATA_STREAM_CHECKS, c -> c.apply(dataStream, clusterState));
122+
assertThat(issues, equalTo(singletonList(expected)));
123+
}
124+
}

0 commit comments

Comments
 (0)