Skip to content

Commit 923a0d4

Browse files
davidkyledavidkyle
authored
[ML] Check if the anomaly results index has been rolled over (#125404) (#125785)
If the v7 index has already been rolled over don't try again
1 parent aa17ec6 commit 923a0d4

File tree

6 files changed

+151
-6
lines changed

6 files changed

+151
-6
lines changed

‎docs/changelog/125404.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 125404
2+
summary: Check if the anomaly results index has been rolled over
3+
area: Machine Learning
4+
type: upgrade
5+
issues: []

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -414,7 +414,9 @@ public static boolean hasIndexTemplate(ClusterState state, String templateName,
414414
}
415415

416416
public static boolean has6DigitSuffix(String indexName) {
417-
return HAS_SIX_DIGIT_SUFFIX.test(indexName);
417+
String[] indexParts = indexName.split("-");
418+
String suffix = indexParts[indexParts.length - 1];
419+
return HAS_SIX_DIGIT_SUFFIX.test(suffix);
418420
}
419421

420422
/**

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -383,6 +383,13 @@ public void testIndexIsReadWriteCompatibleInV9() {
383383
assertFalse(MlIndexAndAlias.indexIsReadWriteCompatibleInV9(IndexVersions.V_7_17_0));
384384
}
385385

386+
public void testHas6DigitSuffix() {
387+
assertTrue(MlIndexAndAlias.has6DigitSuffix("index-000001"));
388+
assertFalse(MlIndexAndAlias.has6DigitSuffix("index1"));
389+
assertFalse(MlIndexAndAlias.has6DigitSuffix("index-foo"));
390+
assertFalse(MlIndexAndAlias.has6DigitSuffix("index000001"));
391+
}
392+
386393
private void createIndexAndAliasIfNecessary(ClusterState clusterState) {
387394
MlIndexAndAlias.createIndexAndAliasIfNecessary(
388395
client,

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java‎

Lines changed: 63 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
import org.elasticsearch.ElasticsearchException;
1111
import org.elasticsearch.ElasticsearchStatusException;
12+
import org.elasticsearch.ResourceAlreadyExistsException;
1213
import org.elasticsearch.TransportVersion;
1314
import org.elasticsearch.TransportVersions;
1415
import org.elasticsearch.action.ActionListener;
@@ -37,9 +38,12 @@
3738
import org.elasticsearch.xpack.core.ml.utils.MlStrings;
3839

3940
import java.util.ArrayList;
41+
import java.util.Arrays;
4042
import java.util.List;
4143

4244
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
45+
import static org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX;
46+
import static org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias.has6DigitSuffix;
4347

4448
/**
4549
* Rollover the various .ml-anomalies result indices
@@ -108,6 +112,14 @@ public void runUpdate(ClusterState latestState) {
108112
continue;
109113
}
110114

115+
// Check if this index has already been rolled over
116+
String latestIndex = latestIndexMatchingBaseName(index, expressionResolver, latestState);
117+
118+
if (index.equals(latestIndex) == false) {
119+
logger.debug("index [{}] will not be rolled over as there is a later index [{}]", index, latestIndex);
120+
continue;
121+
}
122+
111123
PlainActionFuture<Boolean> updated = new PlainActionFuture<>();
112124
rollAndUpdateAliases(latestState, index, updated);
113125
try {
@@ -137,7 +149,7 @@ public void runUpdate(ClusterState latestState) {
137149

138150
private void rollAndUpdateAliases(ClusterState clusterState, String index, ActionListener<Boolean> listener) {
139151
// Create an alias specifically for rolling over.
140-
// The ml-anomalies index has aliases for each job anyone
152+
// The ml-anomalies index has aliases for each job, any
141153
// of which could be used but that means one alias is
142154
// treated differently.
143155
// Using a `.` in the alias name avoids any conflicts
@@ -163,9 +175,19 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio
163175
}
164176

165177
private void rollover(String alias, @Nullable String newIndexName, ActionListener<String> listener) {
166-
client.admin().indices().rolloverIndex(new RolloverRequest(alias, newIndexName), listener.delegateFailure((l, response) -> {
167-
l.onResponse(response.getNewIndex());
168-
}));
178+
client.admin()
179+
.indices()
180+
.rolloverIndex(
181+
new RolloverRequest(alias, newIndexName),
182+
ActionListener.wrap(response -> listener.onResponse(response.getNewIndex()), e -> {
183+
if (e instanceof ResourceAlreadyExistsException alreadyExistsException) {
184+
// The destination index already exists possibly because it has been rolled over already.
185+
listener.onResponse(alreadyExistsException.getIndex().getName());
186+
} else {
187+
listener.onFailure(e);
188+
}
189+
})
190+
);
169191
}
170192

171193
private void createAliasForRollover(String indexName, String aliasName, ActionListener<IndicesAliasesResponse> listener) {
@@ -232,4 +254,41 @@ static boolean isAnomaliesReadAlias(String aliasName) {
232254
// which is not a valid job id.
233255
return MlStrings.isValidId(jobIdPart);
234256
}
257+
258+
/**
259+
* Strip any suffix from the index name and find any other indices
260+
* that match the base name. Then return the latest index from the
261+
* matching ones.
262+
*
263+
* @param index The index to check
264+
* @param expressionResolver The expression resolver
265+
* @param latestState The latest cluster state
266+
* @return The latest index that matches the base name of the given index
267+
*/
268+
static String latestIndexMatchingBaseName(String index, IndexNameExpressionResolver expressionResolver, ClusterState latestState) {
269+
String baseIndexName = MlIndexAndAlias.has6DigitSuffix(index)
270+
? index.substring(0, index.length() - FIRST_INDEX_SIX_DIGIT_SUFFIX.length())
271+
: index;
272+
273+
String[] matching = expressionResolver.concreteIndexNames(
274+
latestState,
275+
IndicesOptions.lenientExpandOpenHidden(),
276+
baseIndexName + "*"
277+
);
278+
279+
// This should never happen
280+
assert matching.length > 0 : "No indices matching [" + baseIndexName + "*]";
281+
if (matching.length == 0) {
282+
return index;
283+
}
284+
285+
// Exclude indices that start with the same base name but are a different index
286+
// e.g. .ml-anomalies-foobar should not be included when the index name is
287+
// .ml-anomalies-foo
288+
String[] filtered = Arrays.stream(matching).filter(i -> {
289+
return i.equals(index) || (has6DigitSuffix(i) && i.length() == baseIndexName.length() + FIRST_INDEX_SIX_DIGIT_SUFFIX.length());
290+
}).toArray(String[]::new);
291+
292+
return MlIndexAndAlias.latestIndex(filtered);
293+
}
235294
}

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAutoUpdateService.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,6 @@ public void clusterChanged(ClusterChangedEvent event) {
6666
.filter(action -> action.isAbleToRun(latestState))
6767
.filter(action -> currentlyUpdating.add(action.getName()))
6868
.toList();
69-
// TODO run updates serially
7069
threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME)
7170
.execute(() -> toRun.forEach((action) -> this.runUpdate(action, latestState)));
7271
}

‎x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.elasticsearch.test.ESTestCase;
3333
import org.elasticsearch.threadpool.ThreadPool;
3434
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
35+
import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias;
3536

3637
import java.util.List;
3738
import java.util.Map;
@@ -179,6 +180,78 @@ public void testRunUpdate_LegacyIndex() {
179180
verifyNoMoreInteractions(client);
180181
}
181182

183+
public void testLatestIndexMatchingBaseName_isLatest() {
184+
Metadata.Builder metadata = Metadata.builder();
185+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo", IndexVersion.current(), List.of("job1")));
186+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bar", IndexVersion.current(), List.of("job2")));
187+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bax", IndexVersion.current(), List.of("job3")));
188+
ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"));
189+
csBuilder.metadata(metadata);
190+
191+
var latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName(
192+
".ml-anomalies-custom-foo",
193+
TestIndexNameExpressionResolver.newInstance(),
194+
csBuilder.build()
195+
);
196+
assertEquals(".ml-anomalies-custom-foo", latest);
197+
}
198+
199+
public void testLatestIndexMatchingBaseName_hasLater() {
200+
Metadata.Builder metadata = Metadata.builder();
201+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo", IndexVersion.current(), List.of("job1")));
202+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bar", IndexVersion.current(), List.of("job2")));
203+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-000001", IndexVersion.current(), List.of("job3")));
204+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-000002", IndexVersion.current(), List.of("job4")));
205+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-baz-000001", IndexVersion.current(), List.of("job5")));
206+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-baz-000002", IndexVersion.current(), List.of("job6")));
207+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-baz-000003", IndexVersion.current(), List.of("job7")));
208+
ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"));
209+
csBuilder.metadata(metadata);
210+
var state = csBuilder.build();
211+
212+
assertTrue(MlIndexAndAlias.has6DigitSuffix(".ml-anomalies-custom-foo-000002"));
213+
214+
var latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName(
215+
".ml-anomalies-custom-foo",
216+
TestIndexNameExpressionResolver.newInstance(),
217+
state
218+
);
219+
assertEquals(".ml-anomalies-custom-foo-000002", latest);
220+
221+
latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName(
222+
".ml-anomalies-custom-baz-000001",
223+
TestIndexNameExpressionResolver.newInstance(),
224+
state
225+
);
226+
assertEquals(".ml-anomalies-custom-baz-000003", latest);
227+
}
228+
229+
public void testLatestIndexMatchingBaseName_CollidingIndexNames() {
230+
Metadata.Builder metadata = Metadata.builder();
231+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo", IndexVersion.current(), List.of("job1")));
232+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bar", IndexVersion.current(), List.of("job2")));
233+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foodifferent000001", IndexVersion.current(), List.of("job3")));
234+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-notthisone-000001", IndexVersion.current(), List.of("job4")));
235+
metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-notthisone-000002", IndexVersion.current(), List.of("job5")));
236+
ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"));
237+
csBuilder.metadata(metadata);
238+
var state = csBuilder.build();
239+
240+
var latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName(
241+
".ml-anomalies-custom-foo",
242+
TestIndexNameExpressionResolver.newInstance(),
243+
state
244+
);
245+
assertEquals(".ml-anomalies-custom-foo", latest);
246+
247+
latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName(
248+
".ml-anomalies-custom-foo-notthisone-000001",
249+
TestIndexNameExpressionResolver.newInstance(),
250+
state
251+
);
252+
assertEquals(".ml-anomalies-custom-foo-notthisone-000002", latest);
253+
}
254+
182255
private record AliasActionMatcher(String aliasName, String index, IndicesAliasesRequest.AliasActions.Type actionType) {
183256
boolean matches(IndicesAliasesRequest.AliasActions aliasAction) {
184257
return aliasAction.actionType() == actionType

0 commit comments

Comments
 (0)