Skip to content

Commit 800e312

Browse files
davidkyledavidkyle
authored
[ML] Check if the anomaly results index has been rolled over (#125404) (#125786)
If the v7 index has already been rolled over don't try again
1 parent 968bddc commit 800e312

File tree

6 files changed

+151
-6
lines changed

6 files changed

+151
-6
lines changed

‎docs/changelog/125404.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 125404
2+
summary: Check if the anomaly results index has been rolled over
3+
area: Machine Learning
4+
type: upgrade
5+
issues: []

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -433,7 +433,9 @@ public static boolean hasIndexTemplate(ClusterState state, String templateName,
433433
}
434434

435435
public static boolean has6DigitSuffix(String indexName) {
436-
return HAS_SIX_DIGIT_SUFFIX.test(indexName);
436+
String[] indexParts = indexName.split("-");
437+
String suffix = indexParts[indexParts.length - 1];
438+
return HAS_SIX_DIGIT_SUFFIX.test(suffix);
437439
}
438440

439441
/**

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -385,6 +385,13 @@ public void testIndexIsReadWriteCompatibleInV9() {
385385
assertFalse(MlIndexAndAlias.indexIsReadWriteCompatibleInV9(IndexVersions.V_7_17_0));
386386
}
387387

388+
public void testHas6DigitSuffix() {
389+
assertTrue(MlIndexAndAlias.has6DigitSuffix("index-000001"));
390+
assertFalse(MlIndexAndAlias.has6DigitSuffix("index1"));
391+
assertFalse(MlIndexAndAlias.has6DigitSuffix("index-foo"));
392+
assertFalse(MlIndexAndAlias.has6DigitSuffix("index000001"));
393+
}
394+
388395
private void createIndexAndAliasIfNecessary(ClusterState clusterState) {
389396
MlIndexAndAlias.createIndexAndAliasIfNecessary(
390397
client,

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java‎

Lines changed: 63 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
import org.elasticsearch.ElasticsearchException;
1111
import org.elasticsearch.ElasticsearchStatusException;
12+
import org.elasticsearch.ResourceAlreadyExistsException;
1213
import org.elasticsearch.TransportVersion;
1314
import org.elasticsearch.TransportVersions;
1415
import org.elasticsearch.action.ActionListener;
@@ -37,9 +38,12 @@
3738
import org.elasticsearch.xpack.core.ml.utils.MlStrings;
3839

3940
import java.util.ArrayList;
41+
import java.util.Arrays;
4042
import java.util.List;
4143

4244
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
45+
import static org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX;
46+
import static org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias.has6DigitSuffix;
4347

4448
/**
4549
* Rollover the various .ml-anomalies result indices
@@ -108,6 +112,14 @@ public void runUpdate(ClusterState latestState) {
108112
continue;
109113
}
110114

115+
// Check if this index has already been rolled over
116+
String latestIndex = latestIndexMatchingBaseName(index, expressionResolver, latestState);
117+
118+
if (index.equals(latestIndex) == false) {
119+
logger.debug("index [{}] will not be rolled over as there is a later index [{}]", index, latestIndex);
120+
continue;
121+
}
122+
111123
PlainActionFuture<Boolean> updated = new PlainActionFuture<>();
112124
rollAndUpdateAliases(latestState, index, updated);
113125
try {
@@ -137,7 +149,7 @@ public void runUpdate(ClusterState latestState) {
137149

138150
private void rollAndUpdateAliases(ClusterState clusterState, String index, ActionListener<Boolean> listener) {
139151
// Create an alias specifically for rolling over.
140-
// The ml-anomalies index has aliases for each job anyone
152+
// The ml-anomalies index has aliases for each job, any
141153
// of which could be used but that means one alias is
142154
// treated differently.
143155
// Using a `.` in the alias name avoids any conflicts
@@ -168,9 +180,19 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio
168180
}
169181

170182
private void rollover(String alias, @Nullable String newIndexName, ActionListener<String> listener) {
171-
client.admin().indices().rolloverIndex(new RolloverRequest(alias, newIndexName), listener.delegateFailure((l, response) -> {
172-
l.onResponse(response.getNewIndex());
173-
}));
183+
client.admin()
184+
.indices()
185+
.rolloverIndex(
186+
new RolloverRequest(alias, newIndexName),
187+
ActionListener.wrap(response -> listener.onResponse(response.getNewIndex()), e -> {
188+
if (e instanceof ResourceAlreadyExistsException alreadyExistsException) {
189+
// The destination index already exists possibly because it has been rolled over already.
190+
listener.onResponse(alreadyExistsException.getIndex().getName());
191+
} else {
192+
listener.onFailure(e);
193+
}
194+
})
195+
);
174196
}
175197

176198
private void createAliasForRollover(String indexName, String aliasName, ActionListener<IndicesAliasesResponse> listener) {
@@ -240,4 +262,41 @@ static boolean isAnomaliesReadAlias(String aliasName) {
240262
// which is not a valid job id.
241263
return MlStrings.isValidId(jobIdPart);
242264
}
265+
266+
/**
267+
* Strip any suffix from the index name and find any other indices
268+
* that match the base name. Then return the latest index from the
269+
* matching ones.
270+
*
271+
* @param index The index to check
272+
* @param expressionResolver The expression resolver
273+
* @param latestState The latest cluster state
274+
* @return The latest index that matches the base name of the given index
275+
*/
276+
static String latestIndexMatchingBaseName(String index, IndexNameExpressionResolver expressionResolver, ClusterState latestState) {
277+
String baseIndexName = MlIndexAndAlias.has6DigitSuffix(index)
278+
? index.substring(0, index.length() - FIRST_INDEX_SIX_DIGIT_SUFFIX.length())
279+
: index;
280+
281+
String[] matching = expressionResolver.concreteIndexNames(
282+
latestState,
283+
IndicesOptions.lenientExpandOpenHidden(),
284+
baseIndexName + "*"
285+
);
286+
287+
// This should never happen
288+
assert matching.length > 0 : "No indices matching [" + baseIndexName + "*]";
289+
if (matching.length == 0) {
290+
return index;
291+
}
292+
293+
// Exclude indices that start with the same base name but are a different index
294+
// e.g. .ml-anomalies-foobar should not be included when the index name is
295+
// .ml-anomalies-foo
296+
String[] filtered = Arrays.stream(matching).filter(i -> {
297+
return i.equals(index) || (has6DigitSuffix(i) && i.length() == baseIndexName.length() + FIRST_INDEX_SIX_DIGIT_SUFFIX.length());
298+
}).toArray(String[]::new);
299+
300+
return MlIndexAndAlias.latestIndex(filtered);
301+
}
243302
}

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAutoUpdateService.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,6 @@ public void clusterChanged(ClusterChangedEvent event) {
6666
.filter(action -> action.isAbleToRun(latestState))
6767
.filter(action -> currentlyUpdating.add(action.getName()))
6868
.toList();
69-
// TODO run updates serially
7069
threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME)
7170
.execute(() -> toRun.forEach((action) -> this.runUpdate(action, latestState)));
7271
}

‎x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.elasticsearch.test.ESTestCase;
3333
import org.elasticsearch.threadpool.ThreadPool;
3434
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
35+
import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias;
3536

3637
import java.util.List;
3738
import java.util.Map;
@@ -183,6 +184,78 @@ public void testRunUpdate_LegacyIndex() {
183184
verifyNoMoreInteractions(client);
184185
}
185186

187+
public void testLatestIndexMatchingBaseName_isLatest() {
188+
Metadata.Builder metadata = Metadata.builder();
189+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo", IndexVersion.current(), List.of("job1")));
190+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bar", IndexVersion.current(), List.of("job2")));
191+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bax", IndexVersion.current(), List.of("job3")));
192+
ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"));
193+
csBuilder.metadata(metadata);
194+
195+
var latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName(
196+
".ml-anomalies-custom-foo",
197+
TestIndexNameExpressionResolver.newInstance(),
198+
csBuilder.build()
199+
);
200+
assertEquals(".ml-anomalies-custom-foo", latest);
201+
}
202+
203+
public void testLatestIndexMatchingBaseName_hasLater() {
204+
Metadata.Builder metadata = Metadata.builder();
205+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo", IndexVersion.current(), List.of("job1")));
206+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bar", IndexVersion.current(), List.of("job2")));
207+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-000001", IndexVersion.current(), List.of("job3")));
208+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-000002", IndexVersion.current(), List.of("job4")));
209+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-baz-000001", IndexVersion.current(), List.of("job5")));
210+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-baz-000002", IndexVersion.current(), List.of("job6")));
211+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-baz-000003", IndexVersion.current(), List.of("job7")));
212+
ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"));
213+
csBuilder.metadata(metadata);
214+
var state = csBuilder.build();
215+
216+
assertTrue(MlIndexAndAlias.has6DigitSuffix(".ml-anomalies-custom-foo-000002"));
217+
218+
var latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName(
219+
".ml-anomalies-custom-foo",
220+
TestIndexNameExpressionResolver.newInstance(),
221+
state
222+
);
223+
assertEquals(".ml-anomalies-custom-foo-000002", latest);
224+
225+
latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName(
226+
".ml-anomalies-custom-baz-000001",
227+
TestIndexNameExpressionResolver.newInstance(),
228+
state
229+
);
230+
assertEquals(".ml-anomalies-custom-baz-000003", latest);
231+
}
232+
233+
public void testLatestIndexMatchingBaseName_CollidingIndexNames() {
234+
Metadata.Builder metadata = Metadata.builder();
235+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo", IndexVersion.current(), List.of("job1")));
236+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bar", IndexVersion.current(), List.of("job2")));
237+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foodifferent000001", IndexVersion.current(), List.of("job3")));
238+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-notthisone-000001", IndexVersion.current(), List.of("job4")));
239+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-notthisone-000002", IndexVersion.current(), List.of("job5")));
240+
ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"));
241+
csBuilder.metadata(metadata);
242+
var state = csBuilder.build();
243+
244+
var latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName(
245+
".ml-anomalies-custom-foo",
246+
TestIndexNameExpressionResolver.newInstance(),
247+
state
248+
);
249+
assertEquals(".ml-anomalies-custom-foo", latest);
250+
251+
latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName(
252+
".ml-anomalies-custom-foo-notthisone-000001",
253+
TestIndexNameExpressionResolver.newInstance(),
254+
state
255+
);
256+
assertEquals(".ml-anomalies-custom-foo-notthisone-000002", latest);
257+
}
258+
186259
private record AliasActionMatcher(String aliasName, String index, IndicesAliasesRequest.AliasActions.Type actionType) {
187260
boolean matches(IndicesAliasesRequest.AliasActions aliasAction) {
188261
return aliasAction.actionType() == actionType

0 commit comments

Comments
 (0)