From 9ff0b94864ca4e85b9b1270c94d2746ae28cb823 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Mon, 6 Oct 2025 15:20:36 +1300 Subject: [PATCH 01/42] [ML] 8Manage rollover of AD results indices Add a rollover check for the AD results indices to the nightly ML maintenance task. The concrete AD reults indices now have a six digit suffix. This is necessary to keep track of rollover behaviour and to determine which index is the "latest" in the series. WIP --- .../alias/IndicesAliasesRequestBuilder.java | 2 + .../alias/TransportIndicesAliasesAction.java | 1 + .../core/src/main/java/module-info.java | 1 + .../xpack/core/ml/job/config/Job.java | 59 +++- .../AnomalyDetectorsIndexFields.java | 2 +- .../core/ml/utils/MlAnomaliesIndexUtils.java | 117 +++++++ .../xpack/core/ml/utils/MlIndexAndAlias.java | 52 ++- .../ml/integration/BulkFailureRetryIT.java | 2 +- .../MlDailyMaintenanceServiceIT.java | 2 + .../xpack/ml/integration/MlJobIT.java | 4 +- .../xpack/ml/MachineLearning.java | 1 + .../xpack/ml/MlAnomaliesIndexUpdate.java | 139 +------- .../xpack/ml/MlDailyMaintenanceService.java | 296 ++++++++++++++++-- .../xpack/ml/MlInitializationService.java | 3 + .../xpack/ml/job/JobManager.java | 25 +- .../xpack/ml/MlAnomaliesIndexUpdateTests.java | 25 +- .../ml/MlDailyMaintenanceServiceTests.java | 3 + .../ml/MlInitializationServiceTests.java | 3 + .../test/ml/custom_all_field.yml | 4 +- .../test/ml/get_datafeed_stats.yml | 2 +- .../ml/jobs_get_result_overall_buckets.yml | 22 +- .../rest-api-spec/test/ml/jobs_get_stats.yml | 12 +- .../test/ml/ml_anomalies_default_mappings.yml | 20 +- .../test/ml/upgrade_job_snapshot.yml | 4 +- .../test/mixed_cluster/30_ml_jobs_crud.yml | 2 +- .../test/old_cluster/30_ml_jobs_crud.yml | 12 +- 26 files changed, 606 insertions(+), 209 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlAnomaliesIndexUtils.java diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequestBuilder.java index 3b700384b85a6..62649fbded846 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequestBuilder.java @@ -14,6 +14,7 @@ import org.elasticsearch.client.internal.ElasticsearchClient; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.logging.LogManager; import java.util.Map; @@ -142,6 +143,7 @@ public IndicesAliasesRequestBuilder addAlias(String index, String alias, boolean * @param alias The alias */ public IndicesAliasesRequestBuilder removeAlias(String index, String alias) { + LogManager.getLogger(IndicesAliasesRequestBuilder.class).info("removing alias [{}] from index [{}]", alias, index); request.addAliasAction(AliasActions.remove().index(index).alias(alias)); return this; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java index cbb0392d4c89a..d41bad1d2e234 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java @@ -247,6 +247,7 @@ protected void masterOperation( break; case REMOVE: for (String alias : concreteAliases(action, projectMetadata, index.getName())) { + logger.warn("Adding alias [{}] for index [{}] to remove list", alias, index.getName()); finalActions.add(new AliasAction.Remove(index.getName(), alias, action.mustExist())); numAliasesRemoved++; } diff --git a/x-pack/plugin/core/src/main/java/module-info.java b/x-pack/plugin/core/src/main/java/module-info.java index 83c8e780106a9..bc155a369ca62 100644 --- a/x-pack/plugin/core/src/main/java/module-info.java +++ b/x-pack/plugin/core/src/main/java/module-info.java @@ -26,6 +26,7 @@ requires org.apache.httpcomponents.client5.httpclient5; requires org.apache.httpcomponents.core5.httpcore5; requires org.slf4j; + requires org.elasticsearch.logging; exports org.elasticsearch.index.engine.frozen; exports org.elasticsearch.license; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index e663bbd6800bd..bda3a53abc5cd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -6,9 +6,12 @@ */ package org.elasticsearch.xpack.core.ml.job.config; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.SimpleDiffable; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -18,6 +21,8 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ObjectParser.ValueType; import org.elasticsearch.xcontent.ParseField; @@ -25,11 +30,13 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.common.time.TimeUtils; import org.elasticsearch.xpack.core.ml.MlConfigVersion; +import org.elasticsearch.xpack.core.ml.utils.MlAnomaliesIndexUtils; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias; import org.elasticsearch.xpack.core.ml.utils.MlStrings; import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; @@ -805,6 +812,8 @@ public static class Builder implements Writeable { private boolean allowLazyOpen; private Blocked blocked = Blocked.none(); private DatafeedConfig.Builder datafeedConfig; + private SetOnce clusterState = new SetOnce<>(); + private SetOnce indexNameExpressionResolver = new SetOnce<>(); public Builder() {} @@ -879,6 +888,14 @@ public String getId() { return id; } + private void setClusterState(ClusterState state) { + this.clusterState.set(state); + } + + private void setIndexNameExpressionResolver(IndexNameExpressionResolver indexNameExpressionResolver) { + this.indexNameExpressionResolver.set(indexNameExpressionResolver); + } + public void setJobVersion(MlConfigVersion jobVersion) { this.jobVersion = jobVersion; } @@ -1305,6 +1322,18 @@ public void validateDetectorsAreUnique() { } } + public Job build( + @SuppressWarnings("HiddenField") Date createTime, + ClusterState state, + IndexNameExpressionResolver indexNameExpressionResolver + ) { +// setCreateTime(createTime); +// setJobVersion(MlConfigVersion.CURRENT); + setClusterState(state); + setIndexNameExpressionResolver(indexNameExpressionResolver); + return build(createTime); + } + /** * Builds a job with the given {@code createTime} and the current version. * This should be used when a new job is created as opposed to {@link #build()}. @@ -1313,6 +1342,7 @@ public void validateDetectorsAreUnique() { * @return The job */ public Job build(@SuppressWarnings("HiddenField") Date createTime) { + LogManager.getLogger(Job.class).debug("[ML] building job withe create time: [{}]", createTime); setCreateTime(createTime); setJobVersion(MlConfigVersion.CURRENT); return build(); @@ -1342,13 +1372,40 @@ public Job build() { // Creation time is NOT required in user input, hence validated only on build ExceptionsHelper.requireNonNull(createTime, CREATE_TIME.getPreferredName()); + LogManager.getLogger(Job.class).warn("resultsIndexName: [{}]: ", resultsIndexName); + if (Strings.isNullOrEmpty(resultsIndexName)) { resultsIndexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; + LogManager.getLogger(Job.class).warn("Using default resultsIndexName: [{}]: ", resultsIndexName); + } else if (resultsIndexName.equals(AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT) == false) { - // User-defined names are prepended with "custom" + // User-defined names are prepended with "custom" and end with a 6 digit suffix // Conditional guards against multiple prepending due to updates instead of first creation resultsIndexName = resultsIndexName.startsWith("custom-") ? resultsIndexName : "custom-" + resultsIndexName; } + + LogManager.getLogger(Job.class).warn("Before: [{}]: ", resultsIndexName); + + resultsIndexName = MlIndexAndAlias.indexNameHasSixDigitSuffix(resultsIndexName) + ? resultsIndexName + : resultsIndexName + "-000001"; + + if (indexNameExpressionResolver.get() != null && clusterState.get() != null) { + LogManager.getLogger(Job.class).warn("Getting latest index matching base name: [{}]: ", resultsIndexName); + + String tmpResultsIndexName = MlIndexAndAlias.latestIndexMatchingBaseName( + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + resultsIndexName, + indexNameExpressionResolver.get(), + clusterState.get() + ); + + resultsIndexName = tmpResultsIndexName.substring(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX.length()); + + LogManager.getLogger(Job.class).warn("OBTAINED latest index matching base name: [{}]: ", resultsIndexName); + } + + LogManager.getLogger(Job.class).warn("After: [{}]: ", resultsIndexName); + if (datafeedConfig != null) { if (datafeedConfig.getId() == null) { datafeedConfig.setId(id); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java index 2a0fff86ba494..d36d031a6a4c3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java @@ -14,7 +14,7 @@ public final class AnomalyDetectorsIndexFields { // ".write" rather than simply "write" to avoid the danger of clashing // with the read alias of a job whose name begins with "write-" public static final String RESULTS_INDEX_WRITE_PREFIX = RESULTS_INDEX_PREFIX + ".write-"; - public static final String RESULTS_INDEX_DEFAULT = "shared"; + public static final String RESULTS_INDEX_DEFAULT = "shared-000001"; private AnomalyDetectorsIndexFields() {} } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlAnomaliesIndexUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlAnomaliesIndexUtils.java new file mode 100644 index 0000000000000..1fbdae0201d82 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlAnomaliesIndexUtils.java @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ml.utils; + +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.logging.Logger; +import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; + +public class MlAnomaliesIndexUtils { + public static void rollover(Client client, RolloverRequest rolloverRequest, ActionListener listener) { + client.admin() + .indices() + .rolloverIndex( + rolloverRequest, + ActionListener.wrap(response -> listener.onResponse(response.getNewIndex()), e -> { + if (e instanceof ResourceAlreadyExistsException alreadyExistsException) { + // The destination index already exists possibly because it has been rolled over already. + listener.onResponse(alreadyExistsException.getIndex().getName()); + } else { + listener.onFailure(e); + } + }) + ); + } + + public static void createAliasForRollover( + Logger logger, + Client client, + String indexName, + String aliasName, + ActionListener listener + ) { + logger.warn("creating rollover [{}] alias for [{}]", aliasName, indexName); + client.admin() + .indices() + .prepareAliases( + TimeValue.THIRTY_SECONDS, + TimeValue.THIRTY_SECONDS + ) + .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(indexName).alias(aliasName).isHidden(true)) + .execute(listener); + } + + public static void updateAliases(IndicesAliasesRequestBuilder request, ActionListener listener) { + request.execute(listener.delegateFailure((l, response) -> l.onResponse(Boolean.TRUE))); + } + + public static IndicesAliasesRequestBuilder addIndexAliasesRequests( + IndicesAliasesRequestBuilder aliasRequestBuilder, + String oldIndex, + String newIndex, + ClusterState clusterState + ) { + // Multiple jobs can share the same index each job + // has a read and write alias that needs updating + // after the rollover + var meta = clusterState.metadata().getProject().index(oldIndex); + assert meta != null; + if (meta == null) { + return aliasRequestBuilder; + } + + for (var alias : meta.getAliases().values()) { + if (isAnomaliesWriteAlias(alias.alias())) { + aliasRequestBuilder.addAliasAction( + IndicesAliasesRequest.AliasActions.add().index(newIndex).alias(alias.alias()).isHidden(true).writeIndex(true) + ); + aliasRequestBuilder.addAliasAction(IndicesAliasesRequest.AliasActions.remove().index(oldIndex).alias(alias.alias())); + } else if (isAnomaliesReadAlias(alias.alias())) { + String jobId = AnomalyDetectorsIndex.jobIdFromAlias(alias.alias()); + aliasRequestBuilder.addAliasAction( + IndicesAliasesRequest.AliasActions.add() + .index(newIndex) + .alias(alias.alias()) + .isHidden(true) + .filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), jobId)) + ); + } + } + + return aliasRequestBuilder; + } + + public static boolean isAnomaliesWriteAlias(String aliasName) { + return aliasName.startsWith(AnomalyDetectorsIndexFields.RESULTS_INDEX_WRITE_PREFIX); + } + + static boolean isAnomaliesReadAlias(String aliasName) { + if (aliasName.startsWith(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX) == false) { + return false; + } + + // See {@link AnomalyDetectorsIndex#jobResultsAliasedName} + String jobIdPart = aliasName.substring(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX.length()); + // If this is a write alias it will start with a `.` character + // which is not a valid job id. + return MlStrings.isValidId(jobIdPart); + } + + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java index 2d8b2bbd19410..79420f030dfa1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java @@ -69,7 +69,7 @@ public final class MlIndexAndAlias { public static final String FIRST_INDEX_SIX_DIGIT_SUFFIX = "-000001"; private static final Logger logger = LogManager.getLogger(MlIndexAndAlias.class); - private static final Predicate HAS_SIX_DIGIT_SUFFIX = Pattern.compile("\\d{6}").asMatchPredicate(); + private static final Predicate HAS_SIX_DIGIT_SUFFIX = Pattern.compile("^.*\\d{6}$").asMatchPredicate(); static final Comparator INDEX_NAME_COMPARATOR = (index1, index2) -> { String[] index1Parts = index1.split("-"); @@ -456,4 +456,54 @@ public static String latestIndex(String[] concreteIndices) { public static boolean indexIsReadWriteCompatibleInV9(IndexVersion version) { return version.onOrAfter(IndexVersions.V_8_0_0); } + + /** + * True if the index name ends with a 6 digit suffix, e.g. 000001 + */ + public static boolean indexNameHasSixDigitSuffix(String indexName) { + boolean ret = HAS_SIX_DIGIT_SUFFIX.test(indexName); + logger.warn("indexNameHasSixDigitSuffix [{}] returning [{}]", indexName, ret); + return ret; + } + + /** + * Strip any suffix from the index name and find any other indices + * that match the base name. Then return the latest index from the + * matching ones. + * + * @param index The index to check + * @param expressionResolver The expression resolver + * @param latestState The latest cluster state + * @return The latest index that matches the base name of the given index + */ + public static String latestIndexMatchingBaseName( + String index, + IndexNameExpressionResolver expressionResolver, + ClusterState latestState + ) { + String baseIndexName = MlIndexAndAlias.has6DigitSuffix(index) + ? index.substring(0, index.length() - FIRST_INDEX_SIX_DIGIT_SUFFIX.length()) + : index; + + String[] matching = expressionResolver.concreteIndexNames( + latestState, + IndicesOptions.lenientExpandOpenHidden(), + baseIndexName + "*" + ); + + // This should never happen + assert matching.length > 0 : "No indices matching [" + baseIndexName + "*]"; + if (matching.length == 0) { + return index; + } + + // Exclude indices that start with the same base name but are a different index + // e.g. .ml-anomalies-foobar should not be included when the index name is + // .ml-anomalies-foo + String[] filtered = Arrays.stream(matching).filter(i -> { + return i.equals(index) || (has6DigitSuffix(i) && i.length() == baseIndexName.length() + FIRST_INDEX_SIX_DIGIT_SUFFIX.length()); + }).toArray(String[]::new); + + return MlIndexAndAlias.latestIndex(filtered); + } } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/BulkFailureRetryIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/BulkFailureRetryIT.java index 39b1d711a9206..2d327324466cb 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/BulkFailureRetryIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/BulkFailureRetryIT.java @@ -44,7 +44,7 @@ public class BulkFailureRetryIT extends MlNativeAutodetectIntegTestCase { private final long now = System.currentTimeMillis(); private static final long DAY = Duration.ofDays(1).toMillis(); private final String jobId = "bulk-failure-retry-job"; - private final String resultsIndex = ".ml-anomalies-custom-bulk-failure-retry-job"; + private final String resultsIndex = ".ml-anomalies-custom-bulk-failure-retry-job-000001"; @Before public void putPastDataIntoIndex() { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceIT.java index 4fe3ed61114c3..fdfb0216d5d9f 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceIT.java @@ -8,6 +8,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexVersion; @@ -54,6 +55,7 @@ public void testTriggerDeleteJobsInStateDeletingWithoutDeletionTask() throws Int client(), mock(ClusterService.class), mock(MlAssignmentNotifier.class), + mock(IndexNameExpressionResolver.class), true, true, true diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java index cebcb6631c9bf..1c1acc99f02ed 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java @@ -212,7 +212,7 @@ public void testCreateJobsWithIndexNameOption() throws Exception { "results_index_name" : "%s"}"""; String jobId1 = "create-jobs-with-index-name-option-job-1"; - String indexName = "non-default-index"; + String indexName = "non-default-index-000001"; putJob(jobId1, Strings.format(jobTemplate, indexName)); String jobId2 = "create-jobs-with-index-name-option-job-2"; @@ -406,7 +406,7 @@ public void testCreateJobInCustomSharedIndexUpdatesMapping() throws Exception { // Check the index mapping contains the first by_field_name Request getResultsMappingRequest = new Request( "GET", - AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-shared-index/_mapping" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-shared-index-000001/_mapping" ); getResultsMappingRequest.addParameter("pretty", null); String resultsMappingAfterJob1 = EntityUtils.toString(client().performRequest(getResultsMappingRequest).getEntity()); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 83adee27248be..204dc5c57e4c6 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -1348,6 +1348,7 @@ public Collection createComponents(PluginServices services) { client, adaptiveAllocationsScalerService, mlAssignmentNotifier, + indexNameExpressionResolver, machineLearningExtension.get().isAnomalyDetectionEnabled(), machineLearningExtension.get().isDataFrameAnalyticsEnabled(), machineLearningExtension.get().isNlpEnabled() diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java index 8067fcfde4ab4..bc18f92769f18 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java @@ -9,10 +9,9 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; + import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; @@ -26,24 +25,18 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.core.Nullable; -import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; -import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; +import org.elasticsearch.xpack.core.ml.utils.MlAnomaliesIndexUtils; import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias; -import org.elasticsearch.xpack.core.ml.utils.MlStrings; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import static org.elasticsearch.TransportVersions.ML_ROLLOVER_LEGACY_INDICES; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; -import static org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; -import static org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias.has6DigitSuffix; /** * Rollover the various .ml-anomalies result indices @@ -108,12 +101,15 @@ public void runUpdate(ClusterState latestState) { latestState.metadata().getProject().index(index).getCreationVersion() ); - if (isCompatibleIndexVersion) { + // Ensure the index name is of a format amenable to simplifying maintenance + boolean isCompatibleIndexFormat = MlIndexAndAlias.indexNameHasSixDigitSuffix(index); + + if (isCompatibleIndexVersion && isCompatibleIndexFormat) { continue; } // Check if this index has already been rolled over - String latestIndex = latestIndexMatchingBaseName(index, expressionResolver, latestState); + String latestIndex = MlIndexAndAlias.latestIndexMatchingBaseName(index, expressionResolver, latestState); if (index.equals(latestIndex) == false) { logger.debug("index [{}] will not be rolled over as there is a later index [{}]", index, latestIndex); @@ -172,131 +168,18 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio ).andThen((l, success) -> { rollover(rolloverAlias, newIndexName, l); }).andThen((l, newIndexNameResponse) -> { - addIndexAliasesRequests(aliasRequestBuilder, index, newIndexNameResponse, clusterState); + MlAnomaliesIndexUtils.addIndexAliasesRequests(aliasRequestBuilder, index, newIndexNameResponse, clusterState); // Delete the new alias created for the rollover action aliasRequestBuilder.removeAlias(newIndexNameResponse, rolloverAlias); - updateAliases(aliasRequestBuilder, l); + MlAnomaliesIndexUtils.updateAliases(aliasRequestBuilder, l); }).addListener(listener); } private void rollover(String alias, @Nullable String newIndexName, ActionListener listener) { - client.admin() - .indices() - .rolloverIndex( - new RolloverRequest(alias, newIndexName), - ActionListener.wrap(response -> listener.onResponse(response.getNewIndex()), e -> { - if (e instanceof ResourceAlreadyExistsException alreadyExistsException) { - // The destination index already exists possibly because it has been rolled over already. - listener.onResponse(alreadyExistsException.getIndex().getName()); - } else { - listener.onFailure(e); - } - }) - ); + MlAnomaliesIndexUtils.rollover(client, new RolloverRequest(alias, newIndexName), listener); } private void createAliasForRollover(String indexName, String aliasName, ActionListener listener) { - logger.info("creating alias for rollover [{}]", aliasName); - client.admin() - .indices() - .prepareAliases( - MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT, - MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT - ) - .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(indexName).alias(aliasName).isHidden(true)) - .execute(listener); - } - - private void updateAliases(IndicesAliasesRequestBuilder request, ActionListener listener) { - request.execute(listener.delegateFailure((l, response) -> l.onResponse(Boolean.TRUE))); - } - - IndicesAliasesRequestBuilder addIndexAliasesRequests( - IndicesAliasesRequestBuilder aliasRequestBuilder, - String oldIndex, - String newIndex, - ClusterState clusterState - ) { - // Multiple jobs can share the same index each job - // has a read and write alias that needs updating - // after the rollover - var meta = clusterState.metadata().getProject().index(oldIndex); - assert meta != null; - if (meta == null) { - return aliasRequestBuilder; - } - - for (var alias : meta.getAliases().values()) { - if (isAnomaliesWriteAlias(alias.alias())) { - aliasRequestBuilder.addAliasAction( - IndicesAliasesRequest.AliasActions.add().index(newIndex).alias(alias.alias()).isHidden(true).writeIndex(true) - ); - aliasRequestBuilder.addAliasAction(IndicesAliasesRequest.AliasActions.remove().index(oldIndex).alias(alias.alias())); - } else if (isAnomaliesReadAlias(alias.alias())) { - String jobId = AnomalyDetectorsIndex.jobIdFromAlias(alias.alias()); - aliasRequestBuilder.addAliasAction( - IndicesAliasesRequest.AliasActions.add() - .index(newIndex) - .alias(alias.alias()) - .isHidden(true) - .filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), jobId)) - ); - } - } - - return aliasRequestBuilder; - } - - static boolean isAnomaliesWriteAlias(String aliasName) { - return aliasName.startsWith(AnomalyDetectorsIndexFields.RESULTS_INDEX_WRITE_PREFIX); - } - - static boolean isAnomaliesReadAlias(String aliasName) { - if (aliasName.startsWith(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX) == false) { - return false; - } - - // See {@link AnomalyDetectorsIndex#jobResultsAliasedName} - String jobIdPart = aliasName.substring(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX.length()); - // If this is a write alias it will start with a `.` character - // which is not a valid job id. - return MlStrings.isValidId(jobIdPart); - } - - /** - * Strip any suffix from the index name and find any other indices - * that match the base name. Then return the latest index from the - * matching ones. - * - * @param index The index to check - * @param expressionResolver The expression resolver - * @param latestState The latest cluster state - * @return The latest index that matches the base name of the given index - */ - static String latestIndexMatchingBaseName(String index, IndexNameExpressionResolver expressionResolver, ClusterState latestState) { - String baseIndexName = MlIndexAndAlias.has6DigitSuffix(index) - ? index.substring(0, index.length() - FIRST_INDEX_SIX_DIGIT_SUFFIX.length()) - : index; - - String[] matching = expressionResolver.concreteIndexNames( - latestState, - IndicesOptions.lenientExpandOpenHidden(), - baseIndexName + "*" - ); - - // This should never happen - assert matching.length > 0 : "No indices matching [" + baseIndexName + "*]"; - if (matching.length == 0) { - return index; - } - - // Exclude indices that start with the same base name but are a different index - // e.g. .ml-anomalies-foobar should not be included when the index name is - // .ml-anomalies-foo - String[] filtered = Arrays.stream(matching).filter(i -> { - return i.equals(index) || (has6DigitSuffix(i) && i.length() == baseIndexName.length() + FIRST_INDEX_SIX_DIGIT_SUFFIX.length()); - }).toArray(String[]::new); - - return MlIndexAndAlias.latestIndex(filtered); + MlAnomaliesIndexUtils.createAliasForRollover(logger, client, indexName, aliasName, listener); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index e7e6e713f123f..2bef624a91853 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -6,19 +6,28 @@ */ package org.elasticsearch.xpack.ml; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.admin.indices.rollover.RolloverConditions; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequestBuilder; +import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.logging.LogManager; import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; @@ -40,10 +49,14 @@ import org.elasticsearch.xpack.core.ml.action.GetJobsAction; import org.elasticsearch.xpack.core.ml.action.ResetJobAction; import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.core.ml.utils.MlAnomaliesIndexUtils; +import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias; import org.elasticsearch.xpack.ml.utils.TypedChainTaskExecutor; import java.time.Clock; import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Random; @@ -62,12 +75,12 @@ */ public class MlDailyMaintenanceService implements Releasable { - private static final Logger logger = LogManager.getLogger(MlDailyMaintenanceService.class); + private static final org.elasticsearch.logging.Logger logger = LogManager.getLogger(MlDailyMaintenanceService.class); private static final int MAX_TIME_OFFSET_MINUTES = 120; private final ThreadPool threadPool; - private final Client client; + private final OriginSettingClient client; private final ClusterService clusterService; private final MlAssignmentNotifier mlAssignmentNotifier; @@ -77,6 +90,9 @@ public class MlDailyMaintenanceService implements Releasable { */ private final Supplier schedulerProvider; + private final IndexNameExpressionResolver expressionResolver; + + private final boolean isAnomalyDetectionEnabled; private final boolean isDataFrameAnalyticsEnabled; private final boolean isNlpEnabled; @@ -90,16 +106,18 @@ public class MlDailyMaintenanceService implements Releasable { Client client, ClusterService clusterService, MlAssignmentNotifier mlAssignmentNotifier, - Supplier scheduleProvider, + Supplier schedulerProvider, + IndexNameExpressionResolver expressionResolver, boolean isAnomalyDetectionEnabled, boolean isDataFrameAnalyticsEnabled, boolean isNlpEnabled ) { this.threadPool = Objects.requireNonNull(threadPool); - this.client = Objects.requireNonNull(client); + this.client = new OriginSettingClient(client, ML_ORIGIN); this.clusterService = Objects.requireNonNull(clusterService); this.mlAssignmentNotifier = Objects.requireNonNull(mlAssignmentNotifier); - this.schedulerProvider = Objects.requireNonNull(scheduleProvider); + this.schedulerProvider = Objects.requireNonNull(schedulerProvider); + this.expressionResolver = Objects.requireNonNull(expressionResolver); this.deleteExpiredDataRequestsPerSecond = MachineLearning.NIGHTLY_MAINTENANCE_REQUESTS_PER_SECOND.get(settings); this.isAnomalyDetectionEnabled = isAnomalyDetectionEnabled; this.isDataFrameAnalyticsEnabled = isDataFrameAnalyticsEnabled; @@ -113,6 +131,7 @@ public MlDailyMaintenanceService( Client client, ClusterService clusterService, MlAssignmentNotifier mlAssignmentNotifier, + IndexNameExpressionResolver expressionResolver, boolean isAnomalyDetectionEnabled, boolean isDataFrameAnalyticsEnabled, boolean isNlpEnabled @@ -124,6 +143,7 @@ public MlDailyMaintenanceService( clusterService, mlAssignmentNotifier, () -> delayToNextTime(clusterName), + expressionResolver, isAnomalyDetectionEnabled, isDataFrameAnalyticsEnabled, isNlpEnabled @@ -144,17 +164,28 @@ void setDeleteExpiredDataRequestsPerSecond(float value) { * @param clusterName the cluster name is used to seed the random offset * @return the delay to the next time the maintenance should be triggered */ +// private static TimeValue delayToNextTime(ClusterName clusterName) { +// Random random = new Random(clusterName.hashCode()); +// int minutesOffset = random.ints(0, MAX_TIME_OFFSET_MINUTES).findFirst().getAsInt(); +// +// ZonedDateTime now = ZonedDateTime.now(Clock.systemDefaultZone()); +// ZonedDateTime next = now.plusDays(1).toLocalDate().atStartOfDay(now.getZone()).plusMinutes(30).plusMinutes(minutesOffset); +// return TimeValue.timeValueMillis(next.toInstant().toEpochMilli() - now.toInstant().toEpochMilli()); +// } + private static TimeValue delayToNextTime(ClusterName clusterName) { Random random = new Random(clusterName.hashCode()); - int minutesOffset = random.ints(0, MAX_TIME_OFFSET_MINUTES).findFirst().getAsInt(); + int minutesOffset = 5; ZonedDateTime now = ZonedDateTime.now(Clock.systemDefaultZone()); - ZonedDateTime next = now.plusDays(1).toLocalDate().atStartOfDay(now.getZone()).plusMinutes(30).plusMinutes(minutesOffset); - return TimeValue.timeValueMillis(next.toInstant().toEpochMilli() - now.toInstant().toEpochMilli()); + ZonedDateTime next = now.plusMinutes(minutesOffset); + var ret = TimeValue.timeValueMillis(next.toInstant().toEpochMilli() - now.toInstant().toEpochMilli()); + logger.warn("Delay until next time [{}] is [{}]", next, ret); + return ret; } public synchronized void start() { - logger.debug("Starting ML daily maintenance service"); + logger.info("Starting ML daily maintenance service"); scheduleNext(); } @@ -214,33 +245,61 @@ private void triggerTasks() { } private void triggerAnomalyDetectionMaintenance() { - // Step 4: Log any error that could have happened + // Step 5: Log any error that could have happened ActionListener finalListener = ActionListener.wrap( - unused -> {}, - e -> logger.warn("An error occurred during [ML] maintenance tasks execution", e) + response -> { + if (response.isAcknowledged() == false) { + logger.warn("[ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask failed"); + } else { + logger.info("[ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask succeeded"); + } + }, + e -> logger.warn("An error occurred during [ML] maintenance tasks execution ", e) ); + + // Step 4: Roll over results indices if necessary + ActionListener rollResultsIndicesIfNecessaryListener = ActionListener.wrap( + unused -> { + logger.warn("1. About to call [triggerRollResultsIndicesIfNecessaryTask]"); + + triggerRollResultsIndicesIfNecessaryTask(finalListener);}, + e -> { + logger.warn("[ML] maintenance task: triggerDeleteExpiredDataTask failed ", e); + logger.warn("2. About to call [triggerRollResultsIndicesIfNecessaryTask]"); + + + // Note: Steps 1-4 are independent, so continue upon errors. + triggerRollResultsIndicesIfNecessaryTask(finalListener); + } ); // Step 3: Delete expired data ActionListener deleteJobsListener = ActionListener.wrap( - unused -> triggerDeleteExpiredDataTask(finalListener), + unused -> { + logger.warn("About to call [triggerDeleteExpiredDataTask]"); + triggerDeleteExpiredDataTask(rollResultsIndicesIfNecessaryListener);}, e -> { logger.warn("[ML] maintenance task: triggerResetJobsInStateResetWithoutResetTask failed", e); - // Note: Steps 1-3 are independent, so continue upon errors. - triggerDeleteExpiredDataTask(finalListener); + logger.warn("About to call [triggerDeleteExpiredDataTask]"); + // Note: Steps 1-4 are independent, so continue upon errors. + triggerDeleteExpiredDataTask(rollResultsIndicesIfNecessaryListener); } ); // Step 2: Reset jobs that are in resetting state without task ActionListener resetJobsListener = ActionListener.wrap( - unused -> triggerResetJobsInStateResetWithoutResetTask(deleteJobsListener), + unused -> { + logger.warn("About to call [triggerResetJobsInStateResetWithoutResetTask]"); + triggerResetJobsInStateResetWithoutResetTask(deleteJobsListener);}, e -> { logger.warn("[ML] maintenance task: triggerDeleteJobsInStateDeletingWithoutDeletionTask failed", e); - // Note: Steps 1-3 are independent, so continue upon errors. + logger.warn("About to call [triggerResetJobsInStateResetWithoutResetTask]"); + // Note: Steps 1-4 are independent, so continue upon errors. triggerResetJobsInStateResetWithoutResetTask(deleteJobsListener); } ); // Step 1: Delete jobs that are in deleting state without task + logger.warn("About to call [triggerDeleteJobsInStateDeletingWithoutDeletionTask]"); triggerDeleteJobsInStateDeletingWithoutDeletionTask(resetJobsListener); } @@ -252,6 +311,180 @@ private void triggerNlpMaintenance() { // Currently a NOOP } + void removeRolloverAlias( + String index, + String alias, + IndicesAliasesRequestBuilder aliasRequestBuilder, + ActionListener listener + ) { + aliasRequestBuilder.removeAlias(index, alias); + MlAnomaliesIndexUtils.updateAliases(aliasRequestBuilder, listener); + } + + private void rollAndUpdateAliases(ClusterState clusterState, String index, ActionListener listener) { + // Create an alias specifically for rolling over. + // The ml-anomalies index has aliases for each job, any + // of which could be used but that means one alias is + // treated differently. + // Using a `.` in the alias name avoids any conflicts + // as AD job Ids cannot start with `.` + String rolloverAlias = index + ".rollover_alias"; + + // If the index does not end in a digit then rollover does not know + // what to name the new index so it must be specified in the request. + // Otherwise leave null and rollover will calculate the new name + String newIndexName = MlIndexAndAlias.has6DigitSuffix(index) ? null : index + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; + IndicesAliasesRequestBuilder aliasRequestBuilder = client.admin() + .indices() + .prepareAliases( + MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT, + MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT + ); + + // 3 Clean up any dangling aliases + ActionListener aliasListener = ActionListener.wrap(r -> { + logger.warn("[ML] Update of aliases succeeded.", rolloverAlias); + listener.onResponse(r); + }, e -> { + if (e instanceof IndexNotFoundException) { + logger.warn("[ML] Update of aliases failed: ", e); + // Removal of the rollover alias may have failed in the case of rollover not occurring, e.g. when the rollover conditions + // were not satisfied. + // We must still clean up the temporary alias from the original index. + // The index name is either the original one provided or the original with a suffix appended. + var indexName = MlIndexAndAlias.has6DigitSuffix(index) ? index : index + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; + logger.warn( + "[ML] Removing dangling rollover alias [{}] from index [{}].", + rolloverAlias, + indexName + ); + + // Make sure we use a fresh IndicesAliasesRequestBuilder, the original one may have changed internal state. + IndicesAliasesRequestBuilder localAliasRequestBuilder = client.admin() + .indices() + .prepareAliases( + MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT, + MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT + ); + + // Execute the cleanup, no need to propagate the original failure. + removeRolloverAlias(indexName, rolloverAlias, localAliasRequestBuilder, listener); + } else { + listener.onFailure(e); + } + }); + + // 3 Update aliases + ActionListener rolloverListener = ActionListener.wrap(newIndexNameResponse -> { + logger.warn( + "[ML] maintenance task: rollAndUpdateAliases for index [{}] succeeded. Cleaning up dangling alias [{}].", + newIndexNameResponse, + rolloverAlias + ); + MlAnomaliesIndexUtils.addIndexAliasesRequests(aliasRequestBuilder, index, newIndexNameResponse, clusterState); + // On success, the rollover alias may have been moved to the new index, so we attempt to remove it from there. + // Note that the rollover request is considered "successful" even if it didn't occur due to a condition not being met + // (no exception will be thrown). In which case the attempt to remove the alias here will fail with an + // IndexNotFoundException. We handle this case with a secondary listener. + removeRolloverAlias(newIndexNameResponse, rolloverAlias, aliasRequestBuilder, aliasListener); + }, e -> { + // If rollover fails, we must still clean up the temporary alias from the original index. + // The index name is either the original one provided or the original with a suffix appended. + var indexName = MlIndexAndAlias.has6DigitSuffix(index) ? index : index + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; + logger.warn( + "[ML] maintenance task: rollAndUpdateAliases for index [{}] failed with exception [{}]. Cleaning up dangling alias [{}]", + indexName, + e, + rolloverAlias + ); + // Execute the cleanup, no need to propagate the original failure. + removeRolloverAlias(indexName, rolloverAlias, aliasRequestBuilder, aliasListener); + }); + + // 2 rollover the index alias to the new index name + ActionListener getIndicesAliasesListener = ActionListener.wrap(getIndicesAliasesResponse -> { + logger.info( + "[ML] getIndicesAliasesResponse: [{}] about to execute rollover request of alias [{}] to new concrete index name [{}]", + getIndicesAliasesResponse, + rolloverAlias, + newIndexName + ); + MlAnomaliesIndexUtils.rollover( + client, + new RolloverRequestBuilder(client).setRolloverTarget(rolloverAlias) + .setNewIndexName(newIndexName) + // .setConditions(RolloverConditions.newBuilder().addMaxIndexSizeCondition(ByteSizeValue.of(50, + // ByteSizeUnit.GB)).build()) // TODO Make these settings? + .setConditions( + RolloverConditions.newBuilder().addMaxIndexSizeCondition(ByteSizeValue.of(2, ByteSizeUnit.MB)).build() + ) // TODO + // Make + // these + // changeable + // settings? + .request(), + rolloverListener + ); + }, (e) -> { + logger.warn("XXX [ML] getIndicesAliasesResponse: [{}] rollover request failed ", e); + rolloverListener.onFailure(e); + }); + + // 1. Create necessary aliases + logger.warn("Creating rollover alias [{}] for index [{}]", rolloverAlias, index); + MlAnomaliesIndexUtils.createAliasForRollover(logger, client, index, rolloverAlias, getIndicesAliasesListener); + } + + // TODO make public for testing? + private void triggerRollResultsIndicesIfNecessaryTask(ActionListener finalListener) { + + List failures = new ArrayList<>(); + + ClusterState clusterState = clusterService.state(); + // list all indices starting .ml-anomalies- + // this includes the shared index and all custom results indices + String[] indices = expressionResolver.concreteIndexNames( + clusterState, + IndicesOptions.lenientExpandOpenHidden(), + AnomalyDetectorsIndex.jobResultsIndexPattern() + ); + + logger.info("[ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask"); + logger.warn("AD results indices [{}]", (Object) indices); + + + for (String index : indices) { + logger.warn("Processing index [{}]", index); + // Check if this index has already been rolled over + String latestIndex = MlIndexAndAlias.latestIndexMatchingBaseName(index, expressionResolver, clusterState); + + if (index.equals(latestIndex) == false) { + logger.warn("index [{}] will not be rolled over as there is a later index [{}]", index, latestIndex); + continue; + } + + ActionListener rollAndUpdateAliasesResponseListener = finalListener.delegateFailureAndWrap( + (l, rolledAndUpdatedAliasesResponse) -> { + if (rolledAndUpdatedAliasesResponse) { + logger.warn( + "2: Successfully completed [ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask for index [{}]", + index + ); + } else { + logger.warn( + "2: Unsuccessful run of [ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask for index [{}]", + index + ); + } + l.onResponse(AcknowledgedResponse.TRUE); // TODO return false if operation failed for any index? + } + ); + + logger.warn("Executing [rollAndUpdateAliases]"); + rollAndUpdateAliases(clusterState, index, rollAndUpdateAliasesResponseListener); + } + } + private void triggerDeleteExpiredDataTask(ActionListener finalListener) { ActionListener deleteExpiredDataActionListener = finalListener.delegateFailureAndWrap( (l, deleteExpiredDataResponse) -> { @@ -329,6 +562,7 @@ private void triggerJobsInStateWithoutMatchingTask( ) { SetOnce> jobsInStateHolder = new SetOnce<>(); + // 3. Filter job responses by those that were not acknowledged (failed) and log an appropriate message ActionListener>> jobsActionListener = finalListener.delegateFailureAndWrap( (delegate, jobsResponses) -> { List jobIds = jobsResponses.stream().filter(t -> t.v2().isAcknowledged() == false).map(Tuple::v1).collect(toList()); @@ -337,23 +571,31 @@ private void triggerJobsInStateWithoutMatchingTask( } else { logger.info("[ML] maintenance task {} failed for jobs: {}", maintenanceTaskName, jobIds); } - delegate.onResponse(AcknowledgedResponse.TRUE); + delegate.onResponse(AcknowledgedResponse.TRUE); // The overall return value is always true } ); + // 2. Get all ML tasks ActionListener listTasksActionListener = ActionListener.wrap(listTasksResponse -> { + // 2a work out all jobs in the specified state that *don't* have an associated task Set jobsInState = jobsInStateHolder.get(); Set jobsWithTask = listTasksResponse.getTasks().stream().map(jobIdExtractor).filter(Objects::nonNull).collect(toSet()); Set jobsInStateWithoutTask = Sets.difference(jobsInState, jobsWithTask); if (jobsInStateWithoutTask.isEmpty()) { - finalListener.onResponse(AcknowledgedResponse.TRUE); + finalListener.onResponse(AcknowledgedResponse.TRUE); // If nothing to do set true in finalListener and return, performing no + // further operations return; } + + // 2b Create a chained task executor whose associated responses will have return type Tuple TypedChainTaskExecutor> chainTaskExecutor = new TypedChainTaskExecutor<>( EsExecutors.DIRECT_EXECUTOR_SERVICE, Predicates.always(), Predicates.always() ); + + // 2c for each job in the specified state without an associated persistent task, add a supplied request to the list of chained + // tasks to execute for (String jobId : jobsInStateWithoutTask) { chainTaskExecutor.add( listener -> executeAsyncWithOrigin( @@ -365,16 +607,25 @@ private void triggerJobsInStateWithoutMatchingTask( ) ); } + + // 2d Execute the list of chained requests chainTaskExecutor.execute(jobsActionListener); }, finalListener::onFailure); + // 1. Get all jobs ActionListener getJobsActionListener = ActionListener.wrap(getJobsResponse -> { + // 1a Filter jobs by specified particular state Set jobsInState = getJobsResponse.getResponse().results().stream().filter(jobFilter).map(Job::getId).collect(toSet()); if (jobsInState.isEmpty()) { - finalListener.onResponse(AcknowledgedResponse.TRUE); + logger.warn("[{}]: no jobs in state [{}]", maintenanceTaskName, jobsInState); + finalListener.onResponse(AcknowledgedResponse.TRUE); // If nothing to do return true in the final listener, do not perform + // any more operations return; } + // 1b Stash the filtered jobs in a set for further operations, do this once and only once jobsInStateHolder.set(jobsInState); + + // 1c Execute another operation to list all permanent ML tasks executeAsyncWithOrigin( client, ML_ORIGIN, @@ -384,6 +635,7 @@ private void triggerJobsInStateWithoutMatchingTask( ); }, finalListener::onFailure); + logger.warn("Executing GetJobsAction"); executeAsyncWithOrigin(client, ML_ORIGIN, GetJobsAction.INSTANCE, new GetJobsAction.Request("*"), getJobsActionListener); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java index 5b37b7e75b737..6fedcb0da068c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.component.LifecycleListener; import org.elasticsearch.common.settings.Settings; @@ -67,6 +68,7 @@ public final class MlInitializationService implements ClusterStateListener { Client client, AdaptiveAllocationsScalerService adaptiveAllocationsScalerService, MlAssignmentNotifier mlAssignmentNotifier, + IndexNameExpressionResolver indexNameExpressionResolver, boolean isAnomalyDetectionEnabled, boolean isDataFrameAnalyticsEnabled, boolean isNlpEnabled @@ -81,6 +83,7 @@ public final class MlInitializationService implements ClusterStateListener { client, clusterService, mlAssignmentNotifier, + indexNameExpressionResolver, isAnomalyDetectionEnabled, isDataFrameAnalyticsEnabled, isNlpEnabled diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index 164a6ea8ad560..fb9885d08a7c3 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -204,23 +204,32 @@ static void validateCategorizationAnalyzerOrSetDefault( AnalysisRegistry analysisRegistry, MlConfigVersion minNodeVersion ) throws IOException { + LogManager.getLogger(JobManager.class).warn("XXX: 1. Validating categorization analyzer"); AnalysisConfig analysisConfig = jobBuilder.getAnalysisConfig(); + LogManager.getLogger(JobManager.class).warn("XXX: 1a. Validating categorization analyzer"); + CategorizationAnalyzerConfig categorizationAnalyzerConfig = analysisConfig.getCategorizationAnalyzerConfig(); + LogManager.getLogger(JobManager.class).warn("XXX: 1b. Validating categorization analyzer"); + if (categorizationAnalyzerConfig != null) { + LogManager.getLogger(JobManager.class).warn("XXX: 2. Validating categorization analyzer"); CategorizationAnalyzer.verifyConfigBuilder( new CategorizationAnalyzerConfig.Builder(categorizationAnalyzerConfig), analysisRegistry ); } else if (analysisConfig.getCategorizationFieldName() != null && minNodeVersion.onOrAfter(MIN_ML_CONFIG_VERSION_FOR_STANDARD_CATEGORIZATION_ANALYZER)) { + LogManager.getLogger(JobManager.class).warn("XXX: 3. Setting standard categorization analyzer"); // Any supplied categorization filters are transferred into the new categorization analyzer. // The user supplied categorization filters will already have been validated when the put job // request was built, so we know they're valid. AnalysisConfig.Builder analysisConfigBuilder = new AnalysisConfig.Builder(analysisConfig).setCategorizationAnalyzerConfig( CategorizationAnalyzerConfig.buildStandardCategorizationAnalyzer(analysisConfig.getCategorizationFilters()) ).setCategorizationFilters(null); + LogManager.getLogger(JobManager.class).warn("XXX: 4. Validating categorization analyzer"); jobBuilder.setAnalysisConfig(analysisConfigBuilder); } + LogManager.getLogger(JobManager.class).warn("XXX: 5. Done validating categorization analyzer"); } /** @@ -234,13 +243,21 @@ public void putJob( ) throws IOException { MlConfigVersion minNodeVersion = MlConfigVersion.getMinMlConfigVersion(state.getNodes()); - Job.Builder jobBuilder = request.getJobBuilder(); + logger.warn("[ML]: 1. Putting job [{}]", jobBuilder.getId()); + jobBuilder.validateAnalysisLimitsAndSetDefaults(maxModelMemoryLimitSupplier.get()); + logger.warn("[ML]: 2. Putting job [{}]", jobBuilder.getId()); + jobBuilder.validateModelSnapshotRetentionSettingsAndSetDefaults(); + logger.warn("[ML]: 3. Putting job [{}]", jobBuilder.getId()); + validateCategorizationAnalyzerOrSetDefault(jobBuilder, analysisRegistry, minNodeVersion); - Job job = jobBuilder.build(new Date()); + logger.warn("[ML]: Building job [{}]", jobBuilder.getId()); + Job job = jobBuilder.build(new Date(), state, indexNameExpressionResolver); + logger.warn("[ML]: Created job [{}]", jobBuilder.getId()); + ActionListener putJobListener = new ActionListener<>() { @Override @@ -428,6 +445,10 @@ public void deleteJob( ); } + public IndexNameExpressionResolver indexNameExpressionResolver() { + return indexNameExpressionResolver; + } + private void postJobUpdate(UpdateJobAction.Request request, Job updatedJob, ActionListener actionListener) { // Autodetect must be updated if the fields that the C++ uses are changed JobUpdate jobUpdate = request.getJobUpdate(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java index a1140aff0bdee..9fbb7f08b1e20 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.core.ml.utils.MlAnomaliesIndexUtils; import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias; import java.util.List; @@ -52,15 +53,15 @@ public class MlAnomaliesIndexUpdateTests extends ESTestCase { public void testIsAnomaliesWriteAlias() { - assertTrue(MlAnomaliesIndexUpdate.isAnomaliesWriteAlias(AnomalyDetectorsIndex.resultsWriteAlias("foo"))); - assertFalse(MlAnomaliesIndexUpdate.isAnomaliesWriteAlias(AnomalyDetectorsIndex.jobResultsAliasedName("foo"))); - assertFalse(MlAnomaliesIndexUpdate.isAnomaliesWriteAlias("some-index")); + assertTrue(MlAnomaliesIndexUtils.isAnomaliesWriteAlias(AnomalyDetectorsIndex.resultsWriteAlias("foo"))); + assertFalse(MlAnomaliesIndexUtils.isAnomaliesWriteAlias(AnomalyDetectorsIndex.jobResultsAliasedName("foo"))); + assertFalse(MlAnomaliesIndexUtils.isAnomaliesWriteAlias("some-index")); } public void testIsAnomaliesAlias() { - assertTrue(MlAnomaliesIndexUpdate.isAnomaliesReadAlias(AnomalyDetectorsIndex.jobResultsAliasedName("foo"))); - assertFalse(MlAnomaliesIndexUpdate.isAnomaliesReadAlias(AnomalyDetectorsIndex.resultsWriteAlias("foo"))); - assertFalse(MlAnomaliesIndexUpdate.isAnomaliesReadAlias("some-index")); + assertTrue(MlAnomaliesIndexUtils.isAnomaliesReadAlias(AnomalyDetectorsIndex.jobResultsAliasedName("foo"))); + assertFalse(MlAnomaliesIndexUtils.isAnomaliesReadAlias(AnomalyDetectorsIndex.resultsWriteAlias("foo"))); + assertFalse(MlAnomaliesIndexUtils.isAnomaliesReadAlias("some-index")); } public void testIsAbleToRun_IndicesDoNotExist() { @@ -114,7 +115,7 @@ public void testBuildIndexAliasesRequest() { ); var newIndex = anomaliesIndex + "-000001"; - var request = updater.addIndexAliasesRequests(aliasRequestBuilder, anomaliesIndex, newIndex, csBuilder.build()); + var request = MlAnomaliesIndexUtils.addIndexAliasesRequests(aliasRequestBuilder, anomaliesIndex, newIndex, csBuilder.build()); var actions = request.request().getAliasActions(); assertThat(actions, hasSize(6)); @@ -194,7 +195,7 @@ public void testLatestIndexMatchingBaseName_isLatest() { ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); csBuilder.metadata(metadata); - var latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName( + var latest = MlIndexAndAlias.latestIndexMatchingBaseName( ".ml-anomalies-custom-foo", TestIndexNameExpressionResolver.newInstance(), csBuilder.build() @@ -217,14 +218,14 @@ public void testLatestIndexMatchingBaseName_hasLater() { assertTrue(MlIndexAndAlias.has6DigitSuffix(".ml-anomalies-custom-foo-000002")); - var latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName( + var latest = MlIndexAndAlias.latestIndexMatchingBaseName( ".ml-anomalies-custom-foo", TestIndexNameExpressionResolver.newInstance(), state ); assertEquals(".ml-anomalies-custom-foo-000002", latest); - latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName( + latest = MlIndexAndAlias.latestIndexMatchingBaseName( ".ml-anomalies-custom-baz-000001", TestIndexNameExpressionResolver.newInstance(), state @@ -243,14 +244,14 @@ public void testLatestIndexMatchingBaseName_CollidingIndexNames() { csBuilder.metadata(metadata); var state = csBuilder.build(); - var latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName( + var latest = MlIndexAndAlias.latestIndexMatchingBaseName( ".ml-anomalies-custom-foo", TestIndexNameExpressionResolver.newInstance(), state ); assertEquals(".ml-anomalies-custom-foo", latest); - latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName( + latest = MlIndexAndAlias.latestIndexMatchingBaseName( ".ml-anomalies-custom-foo-notthisone-000001", TestIndexNameExpressionResolver.newInstance(), state diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceServiceTests.java index 43f1bf5d1710b..28b499ecbccd5 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceServiceTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskInfo; @@ -147,6 +148,7 @@ public void testNoAnomalyDetectionTasksWhenDisabled() throws InterruptedExceptio verify(mlAssignmentNotifier, Mockito.atLeast(1)).auditUnassignedMlTasks(eq(Metadata.DEFAULT_PROJECT_ID), any(), any()); } + // XXX private void assertThatBothTasksAreTriggered(Answer deleteExpiredDataAnswer, Answer getJobsAnswer) throws InterruptedException { when(clusterService.state()).thenReturn(createClusterState(false)); doAnswer(deleteExpiredDataAnswer).when(client).execute(same(DeleteExpiredDataAction.INSTANCE), any(), any()); @@ -319,6 +321,7 @@ private void executeMaintenanceTriggers( clusterService, mlAssignmentNotifier, scheduleProvider, + TestIndexNameExpressionResolver.newInstance(), isAnomalyDetectionEnabled, isDataFrameAnalyticsEnabled, isNlpEnabled diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java index 80c957ecb7a09..1ee26d244679a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue; +import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.ml.inference.adaptiveallocations.AdaptiveAllocationsScalerService; @@ -75,6 +76,7 @@ public void testInitialize() { client, adaptiveAllocationsScalerService, mlAssignmentNotifier, + TestIndexNameExpressionResolver.newInstance(), true, true, true @@ -91,6 +93,7 @@ public void testInitialize_noMasterNode() { client, adaptiveAllocationsScalerService, mlAssignmentNotifier, + TestIndexNameExpressionResolver.newInstance(), true, true, true diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/custom_all_field.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/custom_all_field.yml index eefd9b937cbec..f072e67e266fb 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/custom_all_field.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/custom_all_field.yml @@ -75,7 +75,7 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.refresh: - index: [.ml-anomalies-shared] + index: [.ml-anomalies-shared-000001] --- "Test querying custom all field": @@ -148,7 +148,7 @@ setup: - do: search: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 expand_wildcards: all rest_total_hits_as_int: true body: { query: { bool: { must: [ diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/get_datafeed_stats.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/get_datafeed_stats.yml index afc0ee9db16bd..ab24331c2f65d 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/get_datafeed_stats.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/get_datafeed_stats.yml @@ -275,7 +275,7 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.delete: - index: ".ml-anomalies-shared" + index: ".ml-anomalies-shared-000001" - do: headers: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml index 52f70efcf4b04..ddbd4eeeb339a 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml @@ -69,7 +69,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-60_1" body: { @@ -85,7 +85,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-60_2" body: { @@ -101,7 +101,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-60_3" body: { @@ -116,7 +116,7 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-30_1" body: { @@ -132,7 +132,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-30_2" body: { @@ -148,7 +148,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-30_3" body: { @@ -164,7 +164,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-17_1" body: { @@ -180,7 +180,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-17_2" body: { @@ -196,7 +196,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-17_3" body: { @@ -212,7 +212,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-17_4" body: { @@ -228,7 +228,7 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.refresh: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 --- "Test overall buckets given missing job": diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_stats.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_stats.yml index ab4c7311d8302..b0383b52f47ab 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_stats.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_stats.yml @@ -279,7 +279,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: job-stats-v54-bwc-test-data-counts body: { @@ -303,7 +303,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: job-stats-v54-bwc-test-model_size_stats body: { @@ -329,7 +329,7 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.refresh: - index: [.ml-anomalies-shared] + index: [.ml-anomalies-shared-000001] - do: ml.get_job_stats: @@ -366,7 +366,7 @@ setup: - do: indices.delete: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 - do: ml.get_job_stats: {} @@ -440,11 +440,11 @@ setup: - do: indices.close: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 # With the index closed the low level ML API reports a problem - do: - catch: /type=cluster_block_exception, reason=index \[.ml-anomalies-shared\] blocked by. \[FORBIDDEN\/.\/index closed\]/ + catch: /type=cluster_block_exception, reason=index \[.ml-anomalies-shared-000001\] blocked by. \[FORBIDDEN\/.\/index closed\]/ ml.get_job_stats: {} # But the high level X-Pack API returns what it can - we do this diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml index d157cc0531b65..ac2d374f3922b 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml @@ -24,7 +24,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "new_doc" body: > { @@ -35,15 +35,15 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.refresh: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.get_field_mapping: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 fields: new_field - - match: {\.ml-anomalies-shared.mappings.new_field.mapping.new_field.type: keyword} + - match: {\.ml-anomalies-shared-000001.mappings.new_field.mapping.new_field.type: keyword} --- "Test _meta exists when two jobs share an index": @@ -67,14 +67,14 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.refresh: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.get_mapping: - index: .ml-anomalies-shared - - is_true: \.ml-anomalies-shared.mappings._meta.version + index: .ml-anomalies-shared-000001 + - is_true: \.ml-anomalies-shared-000001.mappings._meta.version - do: ml.put_job: @@ -95,11 +95,11 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.refresh: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.get_mapping: - index: .ml-anomalies-shared - - is_true: \.ml-anomalies-shared.mappings._meta.version + index: .ml-anomalies-shared-000001 + - is_true: \.ml-anomalies-shared-000001.mappings._meta.version diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/upgrade_job_snapshot.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/upgrade_job_snapshot.yml index e0281880f0f95..6e8495cda21dc 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/upgrade_job_snapshot.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/upgrade_job_snapshot.yml @@ -30,7 +30,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: upgrade-model-snapshot_model_snapshot_1234567890 body: > { @@ -60,7 +60,7 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.refresh: - index: [.ml-anomalies-shared,.ml-state-000001] + index: [.ml-anomalies-shared-000001,.ml-state-000001] --- "Test with unknown job id": diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml index 3738bd6657d07..e8f65cea773d4 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml @@ -141,7 +141,7 @@ # killing the node - do: cluster.health: - index: [".ml-state", ".ml-anomalies-shared"] + index: [".ml-state-000001", ".ml-anomalies-shared-000001"] wait_for_status: green --- diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml index 0c0575e51db91..3027fc07e718a 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml @@ -1,7 +1,7 @@ setup: - do: index: - index: .ml-state + index: .ml-state-000001 id: "dummy-document-to-make-index-creation-idempotent" body: > { @@ -9,7 +9,7 @@ setup: - do: cluster.health: - index: [".ml-state"] + index: [".ml-state-000001"] wait_for_status: green --- @@ -72,7 +72,7 @@ setup: # killing the node - do: cluster.health: - index: [".ml-state", ".ml-anomalies-shared"] + index: [".ml-state-000001", ".ml-anomalies-shared-000001"] wait_for_status: green --- @@ -136,7 +136,7 @@ setup: # killing the node - do: cluster.health: - index: [".ml-state", ".ml-anomalies-shared"] + index: [".ml-state-000001", ".ml-anomalies-shared-000001"] wait_for_status: green --- @@ -196,7 +196,7 @@ setup: # killing the node - do: cluster.health: - index: [".ml-state", ".ml-anomalies-shared"] + index: [".ml-state-000001", ".ml-anomalies-shared-000001"] wait_for_status: green --- @@ -259,7 +259,7 @@ setup: # killing the node - do: cluster.health: - index: [".ml-state", ".ml-anomalies-shared"] + index: [".ml-state-000001", ".ml-anomalies-shared-000001"] wait_for_status: green --- From 0ec99b6e218dc0a322f13273632dcbdff6be89b5 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Mon, 6 Oct 2025 15:29:21 +1300 Subject: [PATCH 02/42] Spotless Apply --- .../xpack/core/ml/job/config/Job.java | 6 +- .../core/ml/utils/MlAnomaliesIndexUtils.java | 27 ++-- .../xpack/ml/MlAnomaliesIndexUpdate.java | 3 +- .../xpack/ml/MlDailyMaintenanceService.java | 129 ++++++++---------- .../xpack/ml/job/JobManager.java | 5 +- 5 files changed, 71 insertions(+), 99 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index bda3a53abc5cd..45b081483ca21 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -22,7 +22,6 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.logging.LogManager; -import org.elasticsearch.logging.Logger; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ObjectParser.ValueType; import org.elasticsearch.xcontent.ParseField; @@ -30,7 +29,6 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.common.time.TimeUtils; import org.elasticsearch.xpack.core.ml.MlConfigVersion; -import org.elasticsearch.xpack.core.ml.utils.MlAnomaliesIndexUtils; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; @@ -1327,8 +1325,8 @@ public Job build( ClusterState state, IndexNameExpressionResolver indexNameExpressionResolver ) { -// setCreateTime(createTime); -// setJobVersion(MlConfigVersion.CURRENT); + // setCreateTime(createTime); + // setJobVersion(MlConfigVersion.CURRENT); setClusterState(state); setIndexNameExpressionResolver(indexNameExpressionResolver); return build(createTime); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlAnomaliesIndexUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlAnomaliesIndexUtils.java index 1fbdae0201d82..e68efd3028bf5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlAnomaliesIndexUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlAnomaliesIndexUtils.java @@ -25,18 +25,15 @@ public class MlAnomaliesIndexUtils { public static void rollover(Client client, RolloverRequest rolloverRequest, ActionListener listener) { client.admin() - .indices() - .rolloverIndex( - rolloverRequest, - ActionListener.wrap(response -> listener.onResponse(response.getNewIndex()), e -> { - if (e instanceof ResourceAlreadyExistsException alreadyExistsException) { - // The destination index already exists possibly because it has been rolled over already. - listener.onResponse(alreadyExistsException.getIndex().getName()); - } else { - listener.onFailure(e); - } - }) - ); + .indices() + .rolloverIndex(rolloverRequest, ActionListener.wrap(response -> listener.onResponse(response.getNewIndex()), e -> { + if (e instanceof ResourceAlreadyExistsException alreadyExistsException) { + // The destination index already exists possibly because it has been rolled over already. + listener.onResponse(alreadyExistsException.getIndex().getName()); + } else { + listener.onFailure(e); + } + })); } public static void createAliasForRollover( @@ -49,10 +46,7 @@ public static void createAliasForRollover( logger.warn("creating rollover [{}] alias for [{}]", aliasName, indexName); client.admin() .indices() - .prepareAliases( - TimeValue.THIRTY_SECONDS, - TimeValue.THIRTY_SECONDS - ) + .prepareAliases(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS) .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(indexName).alias(aliasName).isHidden(true)) .execute(listener); } @@ -113,5 +107,4 @@ static boolean isAnomaliesReadAlias(String aliasName) { return MlStrings.isValidId(jobIdPart); } - } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java index bc18f92769f18..a6487b8e6a8ee 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java @@ -11,7 +11,6 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionListener; - import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; @@ -176,7 +175,7 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio } private void rollover(String alias, @Nullable String newIndexName, ActionListener listener) { - MlAnomaliesIndexUtils.rollover(client, new RolloverRequest(alias, newIndexName), listener); + MlAnomaliesIndexUtils.rollover(client, new RolloverRequest(alias, newIndexName), listener); } private void createAliasForRollover(String indexName, String aliasName, ActionListener listener) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index 2bef624a91853..6fe90b001f7cb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -6,13 +6,6 @@ */ package org.elasticsearch.xpack.ml; -import org.elasticsearch.action.admin.indices.rollover.RolloverConditions; -import org.elasticsearch.action.admin.indices.rollover.RolloverRequestBuilder; -import org.elasticsearch.client.internal.OriginSettingClient; -import org.elasticsearch.common.unit.ByteSizeUnit; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.logging.LogManager; import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionType; @@ -21,16 +14,21 @@ import org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; +import org.elasticsearch.action.admin.indices.rollover.RolloverConditions; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.set.Sets; @@ -38,6 +36,8 @@ import org.elasticsearch.core.Releasable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.logging.LogManager; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.threadpool.Scheduler; @@ -92,7 +92,6 @@ public class MlDailyMaintenanceService implements Releasable { private final IndexNameExpressionResolver expressionResolver; - private final boolean isAnomalyDetectionEnabled; private final boolean isDataFrameAnalyticsEnabled; private final boolean isNlpEnabled; @@ -164,14 +163,14 @@ void setDeleteExpiredDataRequestsPerSecond(float value) { * @param clusterName the cluster name is used to seed the random offset * @return the delay to the next time the maintenance should be triggered */ -// private static TimeValue delayToNextTime(ClusterName clusterName) { -// Random random = new Random(clusterName.hashCode()); -// int minutesOffset = random.ints(0, MAX_TIME_OFFSET_MINUTES).findFirst().getAsInt(); -// -// ZonedDateTime now = ZonedDateTime.now(Clock.systemDefaultZone()); -// ZonedDateTime next = now.plusDays(1).toLocalDate().atStartOfDay(now.getZone()).plusMinutes(30).plusMinutes(minutesOffset); -// return TimeValue.timeValueMillis(next.toInstant().toEpochMilli() - now.toInstant().toEpochMilli()); -// } + // private static TimeValue delayToNextTime(ClusterName clusterName) { + // Random random = new Random(clusterName.hashCode()); + // int minutesOffset = random.ints(0, MAX_TIME_OFFSET_MINUTES).findFirst().getAsInt(); + // + // ZonedDateTime now = ZonedDateTime.now(Clock.systemDefaultZone()); + // ZonedDateTime next = now.plusDays(1).toLocalDate().atStartOfDay(now.getZone()).plusMinutes(30).plusMinutes(minutesOffset); + // return TimeValue.timeValueMillis(next.toInstant().toEpochMilli() - now.toInstant().toEpochMilli()); + // } private static TimeValue delayToNextTime(ClusterName clusterName) { Random random = new Random(clusterName.hashCode()); @@ -179,7 +178,7 @@ private static TimeValue delayToNextTime(ClusterName clusterName) { ZonedDateTime now = ZonedDateTime.now(Clock.systemDefaultZone()); ZonedDateTime next = now.plusMinutes(minutesOffset); - var ret = TimeValue.timeValueMillis(next.toInstant().toEpochMilli() - now.toInstant().toEpochMilli()); + var ret = TimeValue.timeValueMillis(next.toInstant().toEpochMilli() - now.toInstant().toEpochMilli()); logger.warn("Delay until next time [{}] is [{}]", next, ret); return ret; } @@ -246,57 +245,48 @@ private void triggerTasks() { private void triggerAnomalyDetectionMaintenance() { // Step 5: Log any error that could have happened - ActionListener finalListener = ActionListener.wrap( - response -> { - if (response.isAcknowledged() == false) { - logger.warn("[ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask failed"); - } else { - logger.info("[ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask succeeded"); - } - }, - e -> logger.warn("An error occurred during [ML] maintenance tasks execution ", e) ); + ActionListener finalListener = ActionListener.wrap(response -> { + if (response.isAcknowledged() == false) { + logger.warn("[ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask failed"); + } else { + logger.info("[ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask succeeded"); + } + }, e -> logger.warn("An error occurred during [ML] maintenance tasks execution ", e)); // Step 4: Roll over results indices if necessary - ActionListener rollResultsIndicesIfNecessaryListener = ActionListener.wrap( - unused -> { - logger.warn("1. About to call [triggerRollResultsIndicesIfNecessaryTask]"); - - triggerRollResultsIndicesIfNecessaryTask(finalListener);}, - e -> { - logger.warn("[ML] maintenance task: triggerDeleteExpiredDataTask failed ", e); - logger.warn("2. About to call [triggerRollResultsIndicesIfNecessaryTask]"); + ActionListener rollResultsIndicesIfNecessaryListener = ActionListener.wrap(unused -> { + logger.warn("1. About to call [triggerRollResultsIndicesIfNecessaryTask]"); + triggerRollResultsIndicesIfNecessaryTask(finalListener); + }, e -> { + logger.warn("[ML] maintenance task: triggerDeleteExpiredDataTask failed ", e); + logger.warn("2. About to call [triggerRollResultsIndicesIfNecessaryTask]"); - // Note: Steps 1-4 are independent, so continue upon errors. - triggerRollResultsIndicesIfNecessaryTask(finalListener); - } - ); + // Note: Steps 1-4 are independent, so continue upon errors. + triggerRollResultsIndicesIfNecessaryTask(finalListener); + }); // Step 3: Delete expired data - ActionListener deleteJobsListener = ActionListener.wrap( - unused -> { - logger.warn("About to call [triggerDeleteExpiredDataTask]"); - triggerDeleteExpiredDataTask(rollResultsIndicesIfNecessaryListener);}, - e -> { - logger.warn("[ML] maintenance task: triggerResetJobsInStateResetWithoutResetTask failed", e); - logger.warn("About to call [triggerDeleteExpiredDataTask]"); - // Note: Steps 1-4 are independent, so continue upon errors. - triggerDeleteExpiredDataTask(rollResultsIndicesIfNecessaryListener); - } - ); + ActionListener deleteJobsListener = ActionListener.wrap(unused -> { + logger.warn("About to call [triggerDeleteExpiredDataTask]"); + triggerDeleteExpiredDataTask(rollResultsIndicesIfNecessaryListener); + }, e -> { + logger.warn("[ML] maintenance task: triggerResetJobsInStateResetWithoutResetTask failed", e); + logger.warn("About to call [triggerDeleteExpiredDataTask]"); + // Note: Steps 1-4 are independent, so continue upon errors. + triggerDeleteExpiredDataTask(rollResultsIndicesIfNecessaryListener); + }); // Step 2: Reset jobs that are in resetting state without task - ActionListener resetJobsListener = ActionListener.wrap( - unused -> { - logger.warn("About to call [triggerResetJobsInStateResetWithoutResetTask]"); - triggerResetJobsInStateResetWithoutResetTask(deleteJobsListener);}, - e -> { - logger.warn("[ML] maintenance task: triggerDeleteJobsInStateDeletingWithoutDeletionTask failed", e); - logger.warn("About to call [triggerResetJobsInStateResetWithoutResetTask]"); - // Note: Steps 1-4 are independent, so continue upon errors. - triggerResetJobsInStateResetWithoutResetTask(deleteJobsListener); - } - ); + ActionListener resetJobsListener = ActionListener.wrap(unused -> { + logger.warn("About to call [triggerResetJobsInStateResetWithoutResetTask]"); + triggerResetJobsInStateResetWithoutResetTask(deleteJobsListener); + }, e -> { + logger.warn("[ML] maintenance task: triggerDeleteJobsInStateDeletingWithoutDeletionTask failed", e); + logger.warn("About to call [triggerResetJobsInStateResetWithoutResetTask]"); + // Note: Steps 1-4 are independent, so continue upon errors. + triggerResetJobsInStateResetWithoutResetTask(deleteJobsListener); + }); // Step 1: Delete jobs that are in deleting state without task logger.warn("About to call [triggerDeleteJobsInStateDeletingWithoutDeletionTask]"); @@ -353,11 +343,7 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio // We must still clean up the temporary alias from the original index. // The index name is either the original one provided or the original with a suffix appended. var indexName = MlIndexAndAlias.has6DigitSuffix(index) ? index : index + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; - logger.warn( - "[ML] Removing dangling rollover alias [{}] from index [{}].", - rolloverAlias, - indexName - ); + logger.warn("[ML] Removing dangling rollover alias [{}] from index [{}].", rolloverAlias, indexName); // Make sure we use a fresh IndicesAliasesRequestBuilder, the original one may have changed internal state. IndicesAliasesRequestBuilder localAliasRequestBuilder = client.admin() @@ -415,13 +401,11 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio .setNewIndexName(newIndexName) // .setConditions(RolloverConditions.newBuilder().addMaxIndexSizeCondition(ByteSizeValue.of(50, // ByteSizeUnit.GB)).build()) // TODO Make these settings? - .setConditions( - RolloverConditions.newBuilder().addMaxIndexSizeCondition(ByteSizeValue.of(2, ByteSizeUnit.MB)).build() - ) // TODO - // Make - // these - // changeable - // settings? + .setConditions(RolloverConditions.newBuilder().addMaxIndexSizeCondition(ByteSizeValue.of(2, ByteSizeUnit.MB)).build()) // TODO + // Make + // these + // changeable + // settings? .request(), rolloverListener ); @@ -452,7 +436,6 @@ private void triggerRollResultsIndicesIfNecessaryTask(ActionListener putJobListener = new ActionListener<>() { @Override public void onResponse(Boolean mappingsUpdated) { From 8405ff313ec2abf4231da6ef7f330240c7ada2b6 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Mon, 6 Oct 2025 15:35:22 +1300 Subject: [PATCH 03/42] Spotless Apply --- .../xpack/ml/MlDailyMaintenanceService.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index 6fe90b001f7cb..baea6fd7b9976 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -399,13 +399,10 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio client, new RolloverRequestBuilder(client).setRolloverTarget(rolloverAlias) .setNewIndexName(newIndexName) + // TODO Make these conditions configurable settings? // .setConditions(RolloverConditions.newBuilder().addMaxIndexSizeCondition(ByteSizeValue.of(50, - // ByteSizeUnit.GB)).build()) // TODO Make these settings? - .setConditions(RolloverConditions.newBuilder().addMaxIndexSizeCondition(ByteSizeValue.of(2, ByteSizeUnit.MB)).build()) // TODO - // Make - // these - // changeable - // settings? + // ByteSizeUnit.GB)).build()) + .setConditions(RolloverConditions.newBuilder().addMaxIndexSizeCondition(ByteSizeValue.of(2, ByteSizeUnit.MB)).build()) .request(), rolloverListener ); From d10d09db5836d58373af6878bfed492ccdabae37 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Mon, 6 Oct 2025 16:29:50 +1300 Subject: [PATCH 04/42] Bit of a tidy up --- .../alias/IndicesAliasesRequestBuilder.java | 2 - .../alias/TransportIndicesAliasesAction.java | 1 - .../xpack/core/ml/job/config/Job.java | 14 --- .../xpack/core/ml/utils/MlIndexAndAlias.java | 4 +- .../xpack/ml/MlDailyMaintenanceService.java | 103 ++++-------------- .../xpack/ml/job/JobManager.java | 17 --- .../ml/MlDailyMaintenanceServiceTests.java | 1 - 7 files changed, 20 insertions(+), 122 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequestBuilder.java index 62649fbded846..3b700384b85a6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequestBuilder.java @@ -14,7 +14,6 @@ import org.elasticsearch.client.internal.ElasticsearchClient; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.logging.LogManager; import java.util.Map; @@ -143,7 +142,6 @@ public IndicesAliasesRequestBuilder addAlias(String index, String alias, boolean * @param alias The alias */ public IndicesAliasesRequestBuilder removeAlias(String index, String alias) { - LogManager.getLogger(IndicesAliasesRequestBuilder.class).info("removing alias [{}] from index [{}]", alias, index); request.addAliasAction(AliasActions.remove().index(index).alias(alias)); return this; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java index d41bad1d2e234..cbb0392d4c89a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java @@ -247,7 +247,6 @@ protected void masterOperation( break; case REMOVE: for (String alias : concreteAliases(action, projectMetadata, index.getName())) { - logger.warn("Adding alias [{}] for index [{}] to remove list", alias, index.getName()); finalActions.add(new AliasAction.Remove(index.getName(), alias, action.mustExist())); numAliasesRemoved++; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index 45b081483ca21..9c3bb9c061178 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -1325,8 +1325,6 @@ public Job build( ClusterState state, IndexNameExpressionResolver indexNameExpressionResolver ) { - // setCreateTime(createTime); - // setJobVersion(MlConfigVersion.CURRENT); setClusterState(state); setIndexNameExpressionResolver(indexNameExpressionResolver); return build(createTime); @@ -1370,27 +1368,19 @@ public Job build() { // Creation time is NOT required in user input, hence validated only on build ExceptionsHelper.requireNonNull(createTime, CREATE_TIME.getPreferredName()); - LogManager.getLogger(Job.class).warn("resultsIndexName: [{}]: ", resultsIndexName); - if (Strings.isNullOrEmpty(resultsIndexName)) { resultsIndexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; - LogManager.getLogger(Job.class).warn("Using default resultsIndexName: [{}]: ", resultsIndexName); - } else if (resultsIndexName.equals(AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT) == false) { // User-defined names are prepended with "custom" and end with a 6 digit suffix // Conditional guards against multiple prepending due to updates instead of first creation resultsIndexName = resultsIndexName.startsWith("custom-") ? resultsIndexName : "custom-" + resultsIndexName; } - LogManager.getLogger(Job.class).warn("Before: [{}]: ", resultsIndexName); - resultsIndexName = MlIndexAndAlias.indexNameHasSixDigitSuffix(resultsIndexName) ? resultsIndexName : resultsIndexName + "-000001"; if (indexNameExpressionResolver.get() != null && clusterState.get() != null) { - LogManager.getLogger(Job.class).warn("Getting latest index matching base name: [{}]: ", resultsIndexName); - String tmpResultsIndexName = MlIndexAndAlias.latestIndexMatchingBaseName( AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + resultsIndexName, indexNameExpressionResolver.get(), @@ -1398,12 +1388,8 @@ public Job build() { ); resultsIndexName = tmpResultsIndexName.substring(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX.length()); - - LogManager.getLogger(Job.class).warn("OBTAINED latest index matching base name: [{}]: ", resultsIndexName); } - LogManager.getLogger(Job.class).warn("After: [{}]: ", resultsIndexName); - if (datafeedConfig != null) { if (datafeedConfig.getId() == null) { datafeedConfig.setId(id); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java index 79420f030dfa1..c4c303dde8c52 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java @@ -461,9 +461,7 @@ public static boolean indexIsReadWriteCompatibleInV9(IndexVersion version) { * True if the index name ends with a 6 digit suffix, e.g. 000001 */ public static boolean indexNameHasSixDigitSuffix(String indexName) { - boolean ret = HAS_SIX_DIGIT_SUFFIX.test(indexName); - logger.warn("indexNameHasSixDigitSuffix [{}] returning [{}]", indexName, ret); - return ret; + return HAS_SIX_DIGIT_SUFFIX.test(indexName); } /** diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index baea6fd7b9976..c67cfaa00b77a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -163,25 +163,14 @@ void setDeleteExpiredDataRequestsPerSecond(float value) { * @param clusterName the cluster name is used to seed the random offset * @return the delay to the next time the maintenance should be triggered */ - // private static TimeValue delayToNextTime(ClusterName clusterName) { - // Random random = new Random(clusterName.hashCode()); - // int minutesOffset = random.ints(0, MAX_TIME_OFFSET_MINUTES).findFirst().getAsInt(); - // - // ZonedDateTime now = ZonedDateTime.now(Clock.systemDefaultZone()); - // ZonedDateTime next = now.plusDays(1).toLocalDate().atStartOfDay(now.getZone()).plusMinutes(30).plusMinutes(minutesOffset); - // return TimeValue.timeValueMillis(next.toInstant().toEpochMilli() - now.toInstant().toEpochMilli()); - // } - - private static TimeValue delayToNextTime(ClusterName clusterName) { - Random random = new Random(clusterName.hashCode()); - int minutesOffset = 5; - - ZonedDateTime now = ZonedDateTime.now(Clock.systemDefaultZone()); - ZonedDateTime next = now.plusMinutes(minutesOffset); - var ret = TimeValue.timeValueMillis(next.toInstant().toEpochMilli() - now.toInstant().toEpochMilli()); - logger.warn("Delay until next time [{}] is [{}]", next, ret); - return ret; - } + private static TimeValue delayToNextTime(ClusterName clusterName) { + Random random = new Random(clusterName.hashCode()); + int minutesOffset = random.ints(0, MAX_TIME_OFFSET_MINUTES).findFirst().getAsInt(); + + ZonedDateTime now = ZonedDateTime.now(Clock.systemDefaultZone()); + ZonedDateTime next = now.plusDays(1).toLocalDate().atStartOfDay(now.getZone()).plusMinutes(30).plusMinutes(minutesOffset); + return TimeValue.timeValueMillis(next.toInstant().toEpochMilli() - now.toInstant().toEpochMilli()); + } public synchronized void start() { logger.info("Starting ML daily maintenance service"); @@ -255,41 +244,29 @@ private void triggerAnomalyDetectionMaintenance() { // Step 4: Roll over results indices if necessary ActionListener rollResultsIndicesIfNecessaryListener = ActionListener.wrap(unused -> { - logger.warn("1. About to call [triggerRollResultsIndicesIfNecessaryTask]"); - triggerRollResultsIndicesIfNecessaryTask(finalListener); }, e -> { - logger.warn("[ML] maintenance task: triggerDeleteExpiredDataTask failed ", e); - logger.warn("2. About to call [triggerRollResultsIndicesIfNecessaryTask]"); - // Note: Steps 1-4 are independent, so continue upon errors. triggerRollResultsIndicesIfNecessaryTask(finalListener); }); // Step 3: Delete expired data ActionListener deleteJobsListener = ActionListener.wrap(unused -> { - logger.warn("About to call [triggerDeleteExpiredDataTask]"); triggerDeleteExpiredDataTask(rollResultsIndicesIfNecessaryListener); }, e -> { - logger.warn("[ML] maintenance task: triggerResetJobsInStateResetWithoutResetTask failed", e); - logger.warn("About to call [triggerDeleteExpiredDataTask]"); // Note: Steps 1-4 are independent, so continue upon errors. triggerDeleteExpiredDataTask(rollResultsIndicesIfNecessaryListener); }); // Step 2: Reset jobs that are in resetting state without task ActionListener resetJobsListener = ActionListener.wrap(unused -> { - logger.warn("About to call [triggerResetJobsInStateResetWithoutResetTask]"); triggerResetJobsInStateResetWithoutResetTask(deleteJobsListener); }, e -> { - logger.warn("[ML] maintenance task: triggerDeleteJobsInStateDeletingWithoutDeletionTask failed", e); - logger.warn("About to call [triggerResetJobsInStateResetWithoutResetTask]"); // Note: Steps 1-4 are independent, so continue upon errors. triggerResetJobsInStateResetWithoutResetTask(deleteJobsListener); }); // Step 1: Delete jobs that are in deleting state without task - logger.warn("About to call [triggerDeleteJobsInStateDeletingWithoutDeletionTask]"); triggerDeleteJobsInStateDeletingWithoutDeletionTask(resetJobsListener); } @@ -331,19 +308,16 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT ); - // 3 Clean up any dangling aliases + // 4 Clean up any dangling aliases ActionListener aliasListener = ActionListener.wrap(r -> { - logger.warn("[ML] Update of aliases succeeded.", rolloverAlias); listener.onResponse(r); }, e -> { if (e instanceof IndexNotFoundException) { - logger.warn("[ML] Update of aliases failed: ", e); // Removal of the rollover alias may have failed in the case of rollover not occurring, e.g. when the rollover conditions // were not satisfied. // We must still clean up the temporary alias from the original index. // The index name is either the original one provided or the original with a suffix appended. var indexName = MlIndexAndAlias.has6DigitSuffix(index) ? index : index + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; - logger.warn("[ML] Removing dangling rollover alias [{}] from index [{}].", rolloverAlias, indexName); // Make sure we use a fresh IndicesAliasesRequestBuilder, the original one may have changed internal state. IndicesAliasesRequestBuilder localAliasRequestBuilder = client.admin() @@ -362,11 +336,6 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio // 3 Update aliases ActionListener rolloverListener = ActionListener.wrap(newIndexNameResponse -> { - logger.warn( - "[ML] maintenance task: rollAndUpdateAliases for index [{}] succeeded. Cleaning up dangling alias [{}].", - newIndexNameResponse, - rolloverAlias - ); MlAnomaliesIndexUtils.addIndexAliasesRequests(aliasRequestBuilder, index, newIndexNameResponse, clusterState); // On success, the rollover alias may have been moved to the new index, so we attempt to remove it from there. // Note that the rollover request is considered "successful" even if it didn't occur due to a condition not being met @@ -377,42 +346,24 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio // If rollover fails, we must still clean up the temporary alias from the original index. // The index name is either the original one provided or the original with a suffix appended. var indexName = MlIndexAndAlias.has6DigitSuffix(index) ? index : index + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; - logger.warn( - "[ML] maintenance task: rollAndUpdateAliases for index [{}] failed with exception [{}]. Cleaning up dangling alias [{}]", - indexName, - e, - rolloverAlias - ); // Execute the cleanup, no need to propagate the original failure. removeRolloverAlias(indexName, rolloverAlias, aliasRequestBuilder, aliasListener); }); // 2 rollover the index alias to the new index name ActionListener getIndicesAliasesListener = ActionListener.wrap(getIndicesAliasesResponse -> { - logger.info( - "[ML] getIndicesAliasesResponse: [{}] about to execute rollover request of alias [{}] to new concrete index name [{}]", - getIndicesAliasesResponse, - rolloverAlias, - newIndexName - ); MlAnomaliesIndexUtils.rollover( client, new RolloverRequestBuilder(client).setRolloverTarget(rolloverAlias) .setNewIndexName(newIndexName) // TODO Make these conditions configurable settings? - // .setConditions(RolloverConditions.newBuilder().addMaxIndexSizeCondition(ByteSizeValue.of(50, - // ByteSizeUnit.GB)).build()) - .setConditions(RolloverConditions.newBuilder().addMaxIndexSizeCondition(ByteSizeValue.of(2, ByteSizeUnit.MB)).build()) + .setConditions(RolloverConditions.newBuilder().addMaxIndexSizeCondition(ByteSizeValue.of(50, ByteSizeUnit.GB)).build()) .request(), rolloverListener ); - }, (e) -> { - logger.warn("XXX [ML] getIndicesAliasesResponse: [{}] rollover request failed ", e); - rolloverListener.onFailure(e); - }); + }, rolloverListener::onFailure); // 1. Create necessary aliases - logger.warn("Creating rollover alias [{}] for index [{}]", rolloverAlias, index); MlAnomaliesIndexUtils.createAliasForRollover(logger, client, index, rolloverAlias, getIndicesAliasesListener); } @@ -431,28 +382,25 @@ private void triggerRollResultsIndicesIfNecessaryTask(ActionListener rollAndUpdateAliasesResponseListener = finalListener.delegateFailureAndWrap( (l, rolledAndUpdatedAliasesResponse) -> { if (rolledAndUpdatedAliasesResponse) { - logger.warn( - "2: Successfully completed [ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask for index [{}]", + logger.info( + "Successfully completed [ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask for index [{}]", index ); } else { logger.warn( - "2: Unsuccessful run of [ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask for index [{}]", + "Unsuccessful run of [ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask for index [{}]", index ); } @@ -460,7 +408,6 @@ private void triggerRollResultsIndicesIfNecessaryTask(ActionListener> jobsInStateHolder = new SetOnce<>(); - // 3. Filter job responses by those that were not acknowledged (failed) and log an appropriate message ActionListener>> jobsActionListener = finalListener.delegateFailureAndWrap( (delegate, jobsResponses) -> { List jobIds = jobsResponses.stream().filter(t -> t.v2().isAcknowledged() == false).map(Tuple::v1).collect(toList()); @@ -551,31 +497,26 @@ private void triggerJobsInStateWithoutMatchingTask( } else { logger.info("[ML] maintenance task {} failed for jobs: {}", maintenanceTaskName, jobIds); } - delegate.onResponse(AcknowledgedResponse.TRUE); // The overall return value is always true + delegate.onResponse(AcknowledgedResponse.TRUE); } ); - // 2. Get all ML tasks ActionListener listTasksActionListener = ActionListener.wrap(listTasksResponse -> { - // 2a work out all jobs in the specified state that *don't* have an associated task Set jobsInState = jobsInStateHolder.get(); Set jobsWithTask = listTasksResponse.getTasks().stream().map(jobIdExtractor).filter(Objects::nonNull).collect(toSet()); Set jobsInStateWithoutTask = Sets.difference(jobsInState, jobsWithTask); if (jobsInStateWithoutTask.isEmpty()) { - finalListener.onResponse(AcknowledgedResponse.TRUE); // If nothing to do set true in finalListener and return, performing no - // further operations + finalListener.onResponse(AcknowledgedResponse.TRUE); + return; } - // 2b Create a chained task executor whose associated responses will have return type Tuple TypedChainTaskExecutor> chainTaskExecutor = new TypedChainTaskExecutor<>( EsExecutors.DIRECT_EXECUTOR_SERVICE, Predicates.always(), Predicates.always() ); - // 2c for each job in the specified state without an associated persistent task, add a supplied request to the list of chained - // tasks to execute for (String jobId : jobsInStateWithoutTask) { chainTaskExecutor.add( listener -> executeAsyncWithOrigin( @@ -588,24 +529,19 @@ private void triggerJobsInStateWithoutMatchingTask( ); } - // 2d Execute the list of chained requests chainTaskExecutor.execute(jobsActionListener); }, finalListener::onFailure); - // 1. Get all jobs ActionListener getJobsActionListener = ActionListener.wrap(getJobsResponse -> { - // 1a Filter jobs by specified particular state Set jobsInState = getJobsResponse.getResponse().results().stream().filter(jobFilter).map(Job::getId).collect(toSet()); if (jobsInState.isEmpty()) { logger.warn("[{}]: no jobs in state [{}]", maintenanceTaskName, jobsInState); - finalListener.onResponse(AcknowledgedResponse.TRUE); // If nothing to do return true in the final listener, do not perform - // any more operations + finalListener.onResponse(AcknowledgedResponse.TRUE); + return; } - // 1b Stash the filtered jobs in a set for further operations, do this once and only once jobsInStateHolder.set(jobsInState); - // 1c Execute another operation to list all permanent ML tasks executeAsyncWithOrigin( client, ML_ORIGIN, @@ -615,7 +551,6 @@ private void triggerJobsInStateWithoutMatchingTask( ); }, finalListener::onFailure); - logger.warn("Executing GetJobsAction"); executeAsyncWithOrigin(client, ML_ORIGIN, GetJobsAction.INSTANCE, new GetJobsAction.Request("*"), getJobsActionListener); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index bad14685afb19..1d5fa1b26a1ce 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -204,32 +204,23 @@ static void validateCategorizationAnalyzerOrSetDefault( AnalysisRegistry analysisRegistry, MlConfigVersion minNodeVersion ) throws IOException { - LogManager.getLogger(JobManager.class).warn("XXX: 1. Validating categorization analyzer"); AnalysisConfig analysisConfig = jobBuilder.getAnalysisConfig(); - LogManager.getLogger(JobManager.class).warn("XXX: 1a. Validating categorization analyzer"); - CategorizationAnalyzerConfig categorizationAnalyzerConfig = analysisConfig.getCategorizationAnalyzerConfig(); - LogManager.getLogger(JobManager.class).warn("XXX: 1b. Validating categorization analyzer"); - if (categorizationAnalyzerConfig != null) { - LogManager.getLogger(JobManager.class).warn("XXX: 2. Validating categorization analyzer"); CategorizationAnalyzer.verifyConfigBuilder( new CategorizationAnalyzerConfig.Builder(categorizationAnalyzerConfig), analysisRegistry ); } else if (analysisConfig.getCategorizationFieldName() != null && minNodeVersion.onOrAfter(MIN_ML_CONFIG_VERSION_FOR_STANDARD_CATEGORIZATION_ANALYZER)) { - LogManager.getLogger(JobManager.class).warn("XXX: 3. Setting standard categorization analyzer"); // Any supplied categorization filters are transferred into the new categorization analyzer. // The user supplied categorization filters will already have been validated when the put job // request was built, so we know they're valid. AnalysisConfig.Builder analysisConfigBuilder = new AnalysisConfig.Builder(analysisConfig).setCategorizationAnalyzerConfig( CategorizationAnalyzerConfig.buildStandardCategorizationAnalyzer(analysisConfig.getCategorizationFilters()) ).setCategorizationFilters(null); - LogManager.getLogger(JobManager.class).warn("XXX: 4. Validating categorization analyzer"); jobBuilder.setAnalysisConfig(analysisConfigBuilder); } - LogManager.getLogger(JobManager.class).warn("XXX: 5. Done validating categorization analyzer"); } /** @@ -244,19 +235,11 @@ public void putJob( MlConfigVersion minNodeVersion = MlConfigVersion.getMinMlConfigVersion(state.getNodes()); Job.Builder jobBuilder = request.getJobBuilder(); - logger.warn("[ML]: 1. Putting job [{}]", jobBuilder.getId()); - jobBuilder.validateAnalysisLimitsAndSetDefaults(maxModelMemoryLimitSupplier.get()); - logger.warn("[ML]: 2. Putting job [{}]", jobBuilder.getId()); - jobBuilder.validateModelSnapshotRetentionSettingsAndSetDefaults(); - logger.warn("[ML]: 3. Putting job [{}]", jobBuilder.getId()); - validateCategorizationAnalyzerOrSetDefault(jobBuilder, analysisRegistry, minNodeVersion); - logger.warn("[ML]: Building job [{}]", jobBuilder.getId()); Job job = jobBuilder.build(new Date(), state, indexNameExpressionResolver); - logger.warn("[ML]: Created job [{}]", jobBuilder.getId()); ActionListener putJobListener = new ActionListener<>() { @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceServiceTests.java index 28b499ecbccd5..1982ef8d45571 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceServiceTests.java @@ -148,7 +148,6 @@ public void testNoAnomalyDetectionTasksWhenDisabled() throws InterruptedExceptio verify(mlAssignmentNotifier, Mockito.atLeast(1)).auditUnassignedMlTasks(eq(Metadata.DEFAULT_PROJECT_ID), any(), any()); } - // XXX private void assertThatBothTasksAreTriggered(Answer deleteExpiredDataAnswer, Answer getJobsAnswer) throws InterruptedException { when(clusterService.state()).thenReturn(createClusterState(false)); doAnswer(deleteExpiredDataAnswer).when(client).execute(same(DeleteExpiredDataAction.INSTANCE), any(), any()); From dfb19391e5dedbf439bb548a3cdaec6a9a7a5eb2 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Mon, 6 Oct 2025 16:46:32 +1300 Subject: [PATCH 05/42] Slight refactor --- .../core/ml/utils/MlAnomaliesIndexUtils.java | 110 ------------------ .../xpack/core/ml/utils/MlIndexAndAlias.java | 89 ++++++++++++++ .../xpack/ml/MlAnomaliesIndexUpdate.java | 9 +- .../xpack/ml/MlDailyMaintenanceService.java | 9 +- .../xpack/ml/MlAnomaliesIndexUpdateTests.java | 15 ++- 5 files changed, 104 insertions(+), 128 deletions(-) delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlAnomaliesIndexUtils.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlAnomaliesIndexUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlAnomaliesIndexUtils.java deleted file mode 100644 index e68efd3028bf5..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlAnomaliesIndexUtils.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.core.ml.utils; - -import org.elasticsearch.ResourceAlreadyExistsException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; -import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.logging.Logger; -import org.elasticsearch.xpack.core.ml.job.config.Job; -import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; -import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; - -public class MlAnomaliesIndexUtils { - public static void rollover(Client client, RolloverRequest rolloverRequest, ActionListener listener) { - client.admin() - .indices() - .rolloverIndex(rolloverRequest, ActionListener.wrap(response -> listener.onResponse(response.getNewIndex()), e -> { - if (e instanceof ResourceAlreadyExistsException alreadyExistsException) { - // The destination index already exists possibly because it has been rolled over already. - listener.onResponse(alreadyExistsException.getIndex().getName()); - } else { - listener.onFailure(e); - } - })); - } - - public static void createAliasForRollover( - Logger logger, - Client client, - String indexName, - String aliasName, - ActionListener listener - ) { - logger.warn("creating rollover [{}] alias for [{}]", aliasName, indexName); - client.admin() - .indices() - .prepareAliases(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS) - .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(indexName).alias(aliasName).isHidden(true)) - .execute(listener); - } - - public static void updateAliases(IndicesAliasesRequestBuilder request, ActionListener listener) { - request.execute(listener.delegateFailure((l, response) -> l.onResponse(Boolean.TRUE))); - } - - public static IndicesAliasesRequestBuilder addIndexAliasesRequests( - IndicesAliasesRequestBuilder aliasRequestBuilder, - String oldIndex, - String newIndex, - ClusterState clusterState - ) { - // Multiple jobs can share the same index each job - // has a read and write alias that needs updating - // after the rollover - var meta = clusterState.metadata().getProject().index(oldIndex); - assert meta != null; - if (meta == null) { - return aliasRequestBuilder; - } - - for (var alias : meta.getAliases().values()) { - if (isAnomaliesWriteAlias(alias.alias())) { - aliasRequestBuilder.addAliasAction( - IndicesAliasesRequest.AliasActions.add().index(newIndex).alias(alias.alias()).isHidden(true).writeIndex(true) - ); - aliasRequestBuilder.addAliasAction(IndicesAliasesRequest.AliasActions.remove().index(oldIndex).alias(alias.alias())); - } else if (isAnomaliesReadAlias(alias.alias())) { - String jobId = AnomalyDetectorsIndex.jobIdFromAlias(alias.alias()); - aliasRequestBuilder.addAliasAction( - IndicesAliasesRequest.AliasActions.add() - .index(newIndex) - .alias(alias.alias()) - .isHidden(true) - .filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), jobId)) - ); - } - } - - return aliasRequestBuilder; - } - - public static boolean isAnomaliesWriteAlias(String aliasName) { - return aliasName.startsWith(AnomalyDetectorsIndexFields.RESULTS_INDEX_WRITE_PREFIX); - } - - static boolean isAnomaliesReadAlias(String aliasName) { - if (aliasName.startsWith(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX) == false) { - return false; - } - - // See {@link AnomalyDetectorsIndex#jobResultsAliasedName} - String jobIdPart = aliasName.substring(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX.length()); - // If this is a write alias it will start with a `.` character - // which is not a valid job id. - return MlStrings.isValidId(jobIdPart); - } - -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java index c4c303dde8c52..9597bd38d402d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java @@ -20,6 +20,7 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; @@ -33,9 +34,13 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; import org.elasticsearch.xpack.core.template.IndexTemplateConfig; import java.io.IOException; @@ -504,4 +509,88 @@ public static String latestIndexMatchingBaseName( return MlIndexAndAlias.latestIndex(filtered); } + + public static void rollover(Client client, RolloverRequest rolloverRequest, ActionListener listener) { + client.admin() + .indices() + .rolloverIndex(rolloverRequest, ActionListener.wrap(response -> listener.onResponse(response.getNewIndex()), e -> { + if (e instanceof ResourceAlreadyExistsException alreadyExistsException) { + // The destination index already exists possibly because it has been rolled over already. + listener.onResponse(alreadyExistsException.getIndex().getName()); + } else { + listener.onFailure(e); + } + })); + } + + public static void createAliasForRollover( + org.elasticsearch.logging.Logger logger, + Client client, + String indexName, + String aliasName, + ActionListener listener + ) { + logger.warn("creating rollover [{}] alias for [{}]", aliasName, indexName); + client.admin() + .indices() + .prepareAliases(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS) + .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(indexName).alias(aliasName).isHidden(true)) + .execute(listener); + } + + public static void updateAliases(IndicesAliasesRequestBuilder request, ActionListener listener) { + request.execute(listener.delegateFailure((l, response) -> l.onResponse(Boolean.TRUE))); + } + + public static IndicesAliasesRequestBuilder addIndexAliasesRequests( + IndicesAliasesRequestBuilder aliasRequestBuilder, + String oldIndex, + String newIndex, + ClusterState clusterState + ) { + // Multiple jobs can share the same index each job + // has a read and write alias that needs updating + // after the rollover + var meta = clusterState.metadata().getProject().index(oldIndex); + assert meta != null; + if (meta == null) { + return aliasRequestBuilder; + } + + for (var alias : meta.getAliases().values()) { + if (isAnomaliesWriteAlias(alias.alias())) { + aliasRequestBuilder.addAliasAction( + IndicesAliasesRequest.AliasActions.add().index(newIndex).alias(alias.alias()).isHidden(true).writeIndex(true) + ); + aliasRequestBuilder.addAliasAction(IndicesAliasesRequest.AliasActions.remove().index(oldIndex).alias(alias.alias())); + } else if (isAnomaliesReadAlias(alias.alias())) { + String jobId = AnomalyDetectorsIndex.jobIdFromAlias(alias.alias()); + aliasRequestBuilder.addAliasAction( + IndicesAliasesRequest.AliasActions.add() + .index(newIndex) + .alias(alias.alias()) + .isHidden(true) + .filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), jobId)) + ); + } + } + + return aliasRequestBuilder; + } + + public static boolean isAnomaliesWriteAlias(String aliasName) { + return aliasName.startsWith(AnomalyDetectorsIndexFields.RESULTS_INDEX_WRITE_PREFIX); + } + + public static boolean isAnomaliesReadAlias(String aliasName) { + if (aliasName.startsWith(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX) == false) { + return false; + } + + // See {@link AnomalyDetectorsIndex#jobResultsAliasedName} + String jobIdPart = aliasName.substring(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX.length()); + // If this is a write alias it will start with a `.` character + // which is not a valid job id. + return MlStrings.isValidId(jobIdPart); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java index a6487b8e6a8ee..a1363a5664001 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java @@ -28,7 +28,6 @@ import org.elasticsearch.logging.Logger; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; -import org.elasticsearch.xpack.core.ml.utils.MlAnomaliesIndexUtils; import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias; import java.util.ArrayList; @@ -167,18 +166,18 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio ).andThen((l, success) -> { rollover(rolloverAlias, newIndexName, l); }).andThen((l, newIndexNameResponse) -> { - MlAnomaliesIndexUtils.addIndexAliasesRequests(aliasRequestBuilder, index, newIndexNameResponse, clusterState); + MlIndexAndAlias.addIndexAliasesRequests(aliasRequestBuilder, index, newIndexNameResponse, clusterState); // Delete the new alias created for the rollover action aliasRequestBuilder.removeAlias(newIndexNameResponse, rolloverAlias); - MlAnomaliesIndexUtils.updateAliases(aliasRequestBuilder, l); + MlIndexAndAlias.updateAliases(aliasRequestBuilder, l); }).addListener(listener); } private void rollover(String alias, @Nullable String newIndexName, ActionListener listener) { - MlAnomaliesIndexUtils.rollover(client, new RolloverRequest(alias, newIndexName), listener); + MlIndexAndAlias.rollover(client, new RolloverRequest(alias, newIndexName), listener); } private void createAliasForRollover(String indexName, String aliasName, ActionListener listener) { - MlAnomaliesIndexUtils.createAliasForRollover(logger, client, indexName, aliasName, listener); + MlIndexAndAlias.createAliasForRollover(logger, client, indexName, aliasName, listener); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index c67cfaa00b77a..d207c354d246c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -50,7 +50,6 @@ import org.elasticsearch.xpack.core.ml.action.ResetJobAction; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; -import org.elasticsearch.xpack.core.ml.utils.MlAnomaliesIndexUtils; import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias; import org.elasticsearch.xpack.ml.utils.TypedChainTaskExecutor; @@ -285,7 +284,7 @@ void removeRolloverAlias( ActionListener listener ) { aliasRequestBuilder.removeAlias(index, alias); - MlAnomaliesIndexUtils.updateAliases(aliasRequestBuilder, listener); + MlIndexAndAlias.updateAliases(aliasRequestBuilder, listener); } private void rollAndUpdateAliases(ClusterState clusterState, String index, ActionListener listener) { @@ -336,7 +335,7 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio // 3 Update aliases ActionListener rolloverListener = ActionListener.wrap(newIndexNameResponse -> { - MlAnomaliesIndexUtils.addIndexAliasesRequests(aliasRequestBuilder, index, newIndexNameResponse, clusterState); + MlIndexAndAlias.addIndexAliasesRequests(aliasRequestBuilder, index, newIndexNameResponse, clusterState); // On success, the rollover alias may have been moved to the new index, so we attempt to remove it from there. // Note that the rollover request is considered "successful" even if it didn't occur due to a condition not being met // (no exception will be thrown). In which case the attempt to remove the alias here will fail with an @@ -352,7 +351,7 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio // 2 rollover the index alias to the new index name ActionListener getIndicesAliasesListener = ActionListener.wrap(getIndicesAliasesResponse -> { - MlAnomaliesIndexUtils.rollover( + MlIndexAndAlias.rollover( client, new RolloverRequestBuilder(client).setRolloverTarget(rolloverAlias) .setNewIndexName(newIndexName) @@ -364,7 +363,7 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio }, rolloverListener::onFailure); // 1. Create necessary aliases - MlAnomaliesIndexUtils.createAliasForRollover(logger, client, index, rolloverAlias, getIndicesAliasesListener); + MlIndexAndAlias.createAliasForRollover(logger, client, index, rolloverAlias, getIndicesAliasesListener); } // TODO make public for testing? diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java index 9fbb7f08b1e20..749e791772b98 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; -import org.elasticsearch.xpack.core.ml.utils.MlAnomaliesIndexUtils; import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias; import java.util.List; @@ -53,15 +52,15 @@ public class MlAnomaliesIndexUpdateTests extends ESTestCase { public void testIsAnomaliesWriteAlias() { - assertTrue(MlAnomaliesIndexUtils.isAnomaliesWriteAlias(AnomalyDetectorsIndex.resultsWriteAlias("foo"))); - assertFalse(MlAnomaliesIndexUtils.isAnomaliesWriteAlias(AnomalyDetectorsIndex.jobResultsAliasedName("foo"))); - assertFalse(MlAnomaliesIndexUtils.isAnomaliesWriteAlias("some-index")); + assertTrue(MlIndexAndAlias.isAnomaliesWriteAlias(AnomalyDetectorsIndex.resultsWriteAlias("foo"))); + assertFalse(MlIndexAndAlias.isAnomaliesWriteAlias(AnomalyDetectorsIndex.jobResultsAliasedName("foo"))); + assertFalse(MlIndexAndAlias.isAnomaliesWriteAlias("some-index")); } public void testIsAnomaliesAlias() { - assertTrue(MlAnomaliesIndexUtils.isAnomaliesReadAlias(AnomalyDetectorsIndex.jobResultsAliasedName("foo"))); - assertFalse(MlAnomaliesIndexUtils.isAnomaliesReadAlias(AnomalyDetectorsIndex.resultsWriteAlias("foo"))); - assertFalse(MlAnomaliesIndexUtils.isAnomaliesReadAlias("some-index")); + assertTrue(MlIndexAndAlias.isAnomaliesReadAlias(AnomalyDetectorsIndex.jobResultsAliasedName("foo"))); + assertFalse(MlIndexAndAlias.isAnomaliesReadAlias(AnomalyDetectorsIndex.resultsWriteAlias("foo"))); + assertFalse(MlIndexAndAlias.isAnomaliesReadAlias("some-index")); } public void testIsAbleToRun_IndicesDoNotExist() { @@ -115,7 +114,7 @@ public void testBuildIndexAliasesRequest() { ); var newIndex = anomaliesIndex + "-000001"; - var request = MlAnomaliesIndexUtils.addIndexAliasesRequests(aliasRequestBuilder, anomaliesIndex, newIndex, csBuilder.build()); + var request = MlIndexAndAlias.addIndexAliasesRequests(aliasRequestBuilder, anomaliesIndex, newIndex, csBuilder.build()); var actions = request.request().getAliasActions(); assertThat(actions, hasSize(6)); From ffc2842c0027bfe1cb07a58888c37759efa9fac2 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Mon, 6 Oct 2025 17:11:35 +1300 Subject: [PATCH 06/42] Another tidy --- .../xpack/core/ml/job/config/Job.java | 2 -- .../xpack/ml/MlDailyMaintenanceService.java | 26 ++++++------------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index 9c3bb9c061178..401683da6462f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.logging.LogManager; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ObjectParser.ValueType; import org.elasticsearch.xcontent.ParseField; @@ -1338,7 +1337,6 @@ public Job build( * @return The job */ public Job build(@SuppressWarnings("HiddenField") Date createTime) { - LogManager.getLogger(Job.class).debug("[ML] building job withe create time: [{}]", createTime); setCreateTime(createTime); setJobVersion(MlConfigVersion.CURRENT); return build(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index d207c354d246c..5778cec3aa2d2 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -162,14 +162,14 @@ void setDeleteExpiredDataRequestsPerSecond(float value) { * @param clusterName the cluster name is used to seed the random offset * @return the delay to the next time the maintenance should be triggered */ - private static TimeValue delayToNextTime(ClusterName clusterName) { - Random random = new Random(clusterName.hashCode()); - int minutesOffset = random.ints(0, MAX_TIME_OFFSET_MINUTES).findFirst().getAsInt(); + private static TimeValue delayToNextTime(ClusterName clusterName) { + Random random = new Random(clusterName.hashCode()); + int minutesOffset = random.ints(0, MAX_TIME_OFFSET_MINUTES).findFirst().getAsInt(); - ZonedDateTime now = ZonedDateTime.now(Clock.systemDefaultZone()); - ZonedDateTime next = now.plusDays(1).toLocalDate().atStartOfDay(now.getZone()).plusMinutes(30).plusMinutes(minutesOffset); - return TimeValue.timeValueMillis(next.toInstant().toEpochMilli() - now.toInstant().toEpochMilli()); - } + ZonedDateTime now = ZonedDateTime.now(Clock.systemDefaultZone()); + ZonedDateTime next = now.plusDays(1).toLocalDate().atStartOfDay(now.getZone()).plusMinutes(30).plusMinutes(minutesOffset); + return TimeValue.timeValueMillis(next.toInstant().toEpochMilli() - now.toInstant().toEpochMilli()); + } public synchronized void start() { logger.info("Starting ML daily maintenance service"); @@ -308,9 +308,7 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio ); // 4 Clean up any dangling aliases - ActionListener aliasListener = ActionListener.wrap(r -> { - listener.onResponse(r); - }, e -> { + ActionListener aliasListener = ActionListener.wrap(r -> { listener.onResponse(r); }, e -> { if (e instanceof IndexNotFoundException) { // Removal of the rollover alias may have failed in the case of rollover not occurring, e.g. when the rollover conditions // were not satisfied. @@ -506,16 +504,13 @@ private void triggerJobsInStateWithoutMatchingTask( Set jobsInStateWithoutTask = Sets.difference(jobsInState, jobsWithTask); if (jobsInStateWithoutTask.isEmpty()) { finalListener.onResponse(AcknowledgedResponse.TRUE); - return; } - TypedChainTaskExecutor> chainTaskExecutor = new TypedChainTaskExecutor<>( EsExecutors.DIRECT_EXECUTOR_SERVICE, Predicates.always(), Predicates.always() ); - for (String jobId : jobsInStateWithoutTask) { chainTaskExecutor.add( listener -> executeAsyncWithOrigin( @@ -527,16 +522,12 @@ private void triggerJobsInStateWithoutMatchingTask( ) ); } - chainTaskExecutor.execute(jobsActionListener); }, finalListener::onFailure); - ActionListener getJobsActionListener = ActionListener.wrap(getJobsResponse -> { Set jobsInState = getJobsResponse.getResponse().results().stream().filter(jobFilter).map(Job::getId).collect(toSet()); if (jobsInState.isEmpty()) { - logger.warn("[{}]: no jobs in state [{}]", maintenanceTaskName, jobsInState); finalListener.onResponse(AcknowledgedResponse.TRUE); - return; } jobsInStateHolder.set(jobsInState); @@ -549,7 +540,6 @@ private void triggerJobsInStateWithoutMatchingTask( listTasksActionListener ); }, finalListener::onFailure); - executeAsyncWithOrigin(client, ML_ORIGIN, GetJobsAction.INSTANCE, new GetJobsAction.Request("*"), getJobsActionListener); } From 8f17540fb75762a8d3a392a160ef12d07c962e4f Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Tue, 7 Oct 2025 14:30:49 +1300 Subject: [PATCH 07/42] Remove unused accessor --- .../java/org/elasticsearch/xpack/ml/job/JobManager.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index 1d5fa1b26a1ce..ae56f00903fc2 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -234,12 +234,14 @@ public void putJob( ) throws IOException { MlConfigVersion minNodeVersion = MlConfigVersion.getMinMlConfigVersion(state.getNodes()); + Job.Builder jobBuilder = request.getJobBuilder(); jobBuilder.validateAnalysisLimitsAndSetDefaults(maxModelMemoryLimitSupplier.get()); jobBuilder.validateModelSnapshotRetentionSettingsAndSetDefaults(); validateCategorizationAnalyzerOrSetDefault(jobBuilder, analysisRegistry, minNodeVersion); - Job job = jobBuilder.build(new Date(), state, indexNameExpressionResolver); +// Job job = jobBuilder.build(new Date(), state, indexNameExpressionResolver); + Job job = jobBuilder.build(new Date()); ActionListener putJobListener = new ActionListener<>() { @Override @@ -427,10 +429,6 @@ public void deleteJob( ); } - public IndexNameExpressionResolver indexNameExpressionResolver() { - return indexNameExpressionResolver; - } - private void postJobUpdate(UpdateJobAction.Request request, Job updatedJob, ActionListener actionListener) { // Autodetect must be updated if the fields that the C++ uses are changed JobUpdate jobUpdate = request.getJobUpdate(); From d60a811fdb7986b6e91e2afcad662de6da10cc28 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Tue, 7 Oct 2025 14:38:22 +1300 Subject: [PATCH 08/42] Update docs/changelog/136065.yaml --- docs/changelog/136065.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/136065.yaml diff --git a/docs/changelog/136065.yaml b/docs/changelog/136065.yaml new file mode 100644 index 0000000000000..45f073600e61e --- /dev/null +++ b/docs/changelog/136065.yaml @@ -0,0 +1,5 @@ +pr: 136065 +summary: Manage ad results indices +area: Machine Learning +type: enhancement +issues: [] From c5f58e1cc434aa93d5c27c3acc9c851f4ec7e095 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 7 Oct 2025 01:45:54 +0000 Subject: [PATCH 09/42] [CI] Auto commit changes from spotless --- .../main/java/org/elasticsearch/xpack/ml/job/JobManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index ae56f00903fc2..cd2cc501db4f5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -240,7 +240,7 @@ public void putJob( jobBuilder.validateModelSnapshotRetentionSettingsAndSetDefaults(); validateCategorizationAnalyzerOrSetDefault(jobBuilder, analysisRegistry, minNodeVersion); -// Job job = jobBuilder.build(new Date(), state, indexNameExpressionResolver); + // Job job = jobBuilder.build(new Date(), state, indexNameExpressionResolver); Job job = jobBuilder.build(new Date()); ActionListener putJobListener = new ActionListener<>() { From 7b6caf0bd32eba136834859703a71b6f788a586a Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Wed, 8 Oct 2025 13:47:24 +1300 Subject: [PATCH 10/42] Address some test failures --- .../xpack/core/ml/job/config/Job.java | 2 +- .../xpack/core/ml/utils/MlIndexAndAlias.java | 9 +-------- .../xpack/core/ml/job/config/JobTests.java | 2 +- .../core/ml/utils/MlIndexAndAliasTests.java | 13 +++++++++++++ .../xpack/ml/MlAnomaliesIndexUpdate.java | 2 +- .../xpack/ml/MlDailyMaintenanceService.java | 17 ++++++++++------- .../xpack/ml/MlAnomaliesIndexUpdateTests.java | 14 +------------- .../test/ml/upgrade_job_snapshot.yml | 6 +++--- 8 files changed, 31 insertions(+), 34 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index 401683da6462f..ea4578bc8bca6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -1374,7 +1374,7 @@ public Job build() { resultsIndexName = resultsIndexName.startsWith("custom-") ? resultsIndexName : "custom-" + resultsIndexName; } - resultsIndexName = MlIndexAndAlias.indexNameHasSixDigitSuffix(resultsIndexName) + resultsIndexName = MlIndexAndAlias.has6DigitSuffix(resultsIndexName) ? resultsIndexName : resultsIndexName + "-000001"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java index 9597bd38d402d..5bacc59d39a00 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java @@ -74,7 +74,7 @@ public final class MlIndexAndAlias { public static final String FIRST_INDEX_SIX_DIGIT_SUFFIX = "-000001"; private static final Logger logger = LogManager.getLogger(MlIndexAndAlias.class); - private static final Predicate HAS_SIX_DIGIT_SUFFIX = Pattern.compile("^.*\\d{6}$").asMatchPredicate(); + private static final Predicate HAS_SIX_DIGIT_SUFFIX = Pattern.compile("\\d{6}").asMatchPredicate(); static final Comparator INDEX_NAME_COMPARATOR = (index1, index2) -> { String[] index1Parts = index1.split("-"); @@ -462,13 +462,6 @@ public static boolean indexIsReadWriteCompatibleInV9(IndexVersion version) { return version.onOrAfter(IndexVersions.V_8_0_0); } - /** - * True if the index name ends with a 6 digit suffix, e.g. 000001 - */ - public static boolean indexNameHasSixDigitSuffix(String indexName) { - return HAS_SIX_DIGIT_SUFFIX.test(indexName); - } - /** * Strip any suffix from the index name and find any other indices * that match the base name. Then return the latest index from the diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java index 4f4c90aacbf35..82f139d49ef8f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java @@ -500,7 +500,7 @@ public void testBuilder_setsIndexName() { Job.Builder builder = buildJobBuilder("foo"); builder.setResultsIndexName("carol"); Job job = builder.build(); - assertEquals(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-carol", job.getInitialResultsIndexName()); + assertEquals(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-carol-000001", job.getInitialResultsIndexName()); } public void testBuilder_withInvalidIndexNameThrows() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java index b4bdb45bfb7b4..d46c999850bea 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java @@ -38,6 +38,7 @@ import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; import org.elasticsearch.xpack.core.template.IndexTemplateConfig; import org.junit.After; @@ -357,6 +358,18 @@ public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButLeg assertThat(createRequest.aliases(), equalTo(Collections.singleton(new Alias(TEST_INDEX_ALIAS).isHidden(true)))); } + public void testIsAnomaliesWriteAlias() { + assertTrue(MlIndexAndAlias.isAnomaliesWriteAlias(AnomalyDetectorsIndex.resultsWriteAlias("foo"))); + assertFalse(MlIndexAndAlias.isAnomaliesWriteAlias(AnomalyDetectorsIndex.jobResultsAliasedName("foo"))); + assertFalse(MlIndexAndAlias.isAnomaliesWriteAlias("some-index")); + } + + public void testIsAnomaliesAlias() { + assertTrue(MlIndexAndAlias.isAnomaliesReadAlias(AnomalyDetectorsIndex.jobResultsAliasedName("foo"))); + assertFalse(MlIndexAndAlias.isAnomaliesReadAlias(AnomalyDetectorsIndex.resultsWriteAlias("foo"))); + assertFalse(MlIndexAndAlias.isAnomaliesReadAlias("some-index")); + } + public void testIndexNameComparator() { Comparator comparator = MlIndexAndAlias.INDEX_NAME_COMPARATOR; assertThat(Stream.of("test-000001").max(comparator).get(), equalTo("test-000001")); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java index a1363a5664001..dd61f427ac861 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java @@ -100,7 +100,7 @@ public void runUpdate(ClusterState latestState) { ); // Ensure the index name is of a format amenable to simplifying maintenance - boolean isCompatibleIndexFormat = MlIndexAndAlias.indexNameHasSixDigitSuffix(index); + boolean isCompatibleIndexFormat = MlIndexAndAlias.has6DigitSuffix(index); if (isCompatibleIndexVersion && isCompatibleIndexFormat) { continue; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index 5778cec3aa2d2..a2ffc64d82f6b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -79,7 +79,7 @@ public class MlDailyMaintenanceService implements Releasable { private static final int MAX_TIME_OFFSET_MINUTES = 120; private final ThreadPool threadPool; - private final OriginSettingClient client; + private final Client client; private final ClusterService clusterService; private final MlAssignmentNotifier mlAssignmentNotifier; @@ -111,7 +111,7 @@ public class MlDailyMaintenanceService implements Releasable { boolean isNlpEnabled ) { this.threadPool = Objects.requireNonNull(threadPool); - this.client = new OriginSettingClient(client, ML_ORIGIN); + this.client = Objects.requireNonNull(client); this.clusterService = Objects.requireNonNull(clusterService); this.mlAssignmentNotifier = Objects.requireNonNull(mlAssignmentNotifier); this.schedulerProvider = Objects.requireNonNull(schedulerProvider); @@ -296,11 +296,14 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio // as AD job Ids cannot start with `.` String rolloverAlias = index + ".rollover_alias"; + OriginSettingClient originSettingClient = new OriginSettingClient(client, ML_ORIGIN); + + // If the index does not end in a digit then rollover does not know // what to name the new index so it must be specified in the request. // Otherwise leave null and rollover will calculate the new name String newIndexName = MlIndexAndAlias.has6DigitSuffix(index) ? null : index + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; - IndicesAliasesRequestBuilder aliasRequestBuilder = client.admin() + IndicesAliasesRequestBuilder aliasRequestBuilder = originSettingClient.admin() .indices() .prepareAliases( MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT, @@ -317,7 +320,7 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio var indexName = MlIndexAndAlias.has6DigitSuffix(index) ? index : index + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; // Make sure we use a fresh IndicesAliasesRequestBuilder, the original one may have changed internal state. - IndicesAliasesRequestBuilder localAliasRequestBuilder = client.admin() + IndicesAliasesRequestBuilder localAliasRequestBuilder = originSettingClient.admin() .indices() .prepareAliases( MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT, @@ -350,8 +353,8 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio // 2 rollover the index alias to the new index name ActionListener getIndicesAliasesListener = ActionListener.wrap(getIndicesAliasesResponse -> { MlIndexAndAlias.rollover( - client, - new RolloverRequestBuilder(client).setRolloverTarget(rolloverAlias) + originSettingClient, + new RolloverRequestBuilder(originSettingClient).setRolloverTarget(rolloverAlias) .setNewIndexName(newIndexName) // TODO Make these conditions configurable settings? .setConditions(RolloverConditions.newBuilder().addMaxIndexSizeCondition(ByteSizeValue.of(50, ByteSizeUnit.GB)).build()) @@ -361,7 +364,7 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio }, rolloverListener::onFailure); // 1. Create necessary aliases - MlIndexAndAlias.createAliasForRollover(logger, client, index, rolloverAlias, getIndicesAliasesListener); + MlIndexAndAlias.createAliasForRollover(logger, originSettingClient, index, rolloverAlias, getIndicesAliasesListener); } // TODO make public for testing? diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java index 749e791772b98..1ecf91e2a7ea9 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java @@ -51,18 +51,6 @@ public class MlAnomaliesIndexUpdateTests extends ESTestCase { - public void testIsAnomaliesWriteAlias() { - assertTrue(MlIndexAndAlias.isAnomaliesWriteAlias(AnomalyDetectorsIndex.resultsWriteAlias("foo"))); - assertFalse(MlIndexAndAlias.isAnomaliesWriteAlias(AnomalyDetectorsIndex.jobResultsAliasedName("foo"))); - assertFalse(MlIndexAndAlias.isAnomaliesWriteAlias("some-index")); - } - - public void testIsAnomaliesAlias() { - assertTrue(MlIndexAndAlias.isAnomaliesReadAlias(AnomalyDetectorsIndex.jobResultsAliasedName("foo"))); - assertFalse(MlIndexAndAlias.isAnomaliesReadAlias(AnomalyDetectorsIndex.resultsWriteAlias("foo"))); - assertFalse(MlIndexAndAlias.isAnomaliesReadAlias("some-index")); - } - public void testIsAbleToRun_IndicesDoNotExist() { RoutingTable.Builder routingTable = RoutingTable.builder(); var updater = new MlAnomaliesIndexUpdate(TestIndexNameExpressionResolver.newInstance(), mock(Client.class)); @@ -145,7 +133,7 @@ public void testBuildIndexAliasesRequest() { } public void testRunUpdate_UpToDateIndices() { - String indexName = ".ml-anomalies-sharedindex"; + String indexName = ".ml-anomalies-sharedindex-000001"; var jobs = List.of("job1", "job2"); IndexMetadata.Builder indexMetadata = createSharedResultsIndex(indexName, IndexVersion.current(), jobs); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/upgrade_job_snapshot.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/upgrade_job_snapshot.yml index 6e8495cda21dc..42857b494abee 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/upgrade_job_snapshot.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/upgrade_job_snapshot.yml @@ -98,15 +98,15 @@ setup: "Test existing but corrupt snapshot": - do: ml.upgrade_job_snapshot: - job_id: "upgrade-model-snapshot" + job_id: "upgrade-model-snapshotXXX" snapshot_id: "1234567890" wait_for_completion: false - do: ml.get_model_snapshot_upgrade_stats: - job_id: "upgrade-model-snapshot" + job_id: "upgrade-model-snapshotYYY" snapshot_id: "1234567890" - match: { count: 1 } - - match: { model_snapshot_upgrades.0.job_id: "upgrade-model-snapshot" } + - match: { model_snapshot_upgrades.0.job_id: "upgrade-model-snapshotZZZ" } - match: { model_snapshot_upgrades.0.snapshot_id: "1234567890" } - match: { model_snapshot_upgrades.0.state: /failed|loading_old_state/ } From 66d726833805c803c326cade85e06662c9dcd251 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Wed, 8 Oct 2025 13:52:46 +1300 Subject: [PATCH 11/42] Typos --- .../rest-api-spec/test/ml/upgrade_job_snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/upgrade_job_snapshot.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/upgrade_job_snapshot.yml index 42857b494abee..6e8495cda21dc 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/upgrade_job_snapshot.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/upgrade_job_snapshot.yml @@ -98,15 +98,15 @@ setup: "Test existing but corrupt snapshot": - do: ml.upgrade_job_snapshot: - job_id: "upgrade-model-snapshotXXX" + job_id: "upgrade-model-snapshot" snapshot_id: "1234567890" wait_for_completion: false - do: ml.get_model_snapshot_upgrade_stats: - job_id: "upgrade-model-snapshotYYY" + job_id: "upgrade-model-snapshot" snapshot_id: "1234567890" - match: { count: 1 } - - match: { model_snapshot_upgrades.0.job_id: "upgrade-model-snapshotZZZ" } + - match: { model_snapshot_upgrades.0.job_id: "upgrade-model-snapshot" } - match: { model_snapshot_upgrades.0.snapshot_id: "1234567890" } - match: { model_snapshot_upgrades.0.state: /failed|loading_old_state/ } From f9296fdd1a90ea66e18c312dcf8ba6fa8f609250 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 8 Oct 2025 01:03:15 +0000 Subject: [PATCH 12/42] [CI] Auto commit changes from spotless --- .../java/org/elasticsearch/xpack/core/ml/job/config/Job.java | 4 +--- .../org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index ea4578bc8bca6..e98c551a536de 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -1374,9 +1374,7 @@ public Job build() { resultsIndexName = resultsIndexName.startsWith("custom-") ? resultsIndexName : "custom-" + resultsIndexName; } - resultsIndexName = MlIndexAndAlias.has6DigitSuffix(resultsIndexName) - ? resultsIndexName - : resultsIndexName + "-000001"; + resultsIndexName = MlIndexAndAlias.has6DigitSuffix(resultsIndexName) ? resultsIndexName : resultsIndexName + "-000001"; if (indexNameExpressionResolver.get() != null && clusterState.get() != null) { String tmpResultsIndexName = MlIndexAndAlias.latestIndexMatchingBaseName( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index a2ffc64d82f6b..bfaa4b6304338 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -298,7 +298,6 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio OriginSettingClient originSettingClient = new OriginSettingClient(client, ML_ORIGIN); - // If the index does not end in a digit then rollover does not know // what to name the new index so it must be specified in the request. // Otherwise leave null and rollover will calculate the new name From 07ddbaa39f793133dbeb24fd977fe998c214e5e6 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Wed, 8 Oct 2025 16:10:08 +1300 Subject: [PATCH 13/42] Make the max results index size for rollover user configurable. --- .../configuration-reference/machine-learning-settings.md | 3 +++ .../java/org/elasticsearch/xpack/ml/MachineLearning.java | 9 +++++++++ .../elasticsearch/xpack/ml/MlInitializationService.java | 6 ++++++ .../java/org/elasticsearch/xpack/ml/job/JobManager.java | 3 +-- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md b/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md index 118fb5a480381..0d04af8d1f2be 100644 --- a/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md +++ b/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md @@ -86,6 +86,9 @@ $$$xpack.ml.max_open_jobs$$$ `xpack.ml.nightly_maintenance_requests_per_second` : ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The rate at which the nightly maintenance task deletes expired model snapshots and results. The setting is a proxy to the [`requests_per_second`](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-delete-by-query) parameter used in the delete by query requests and controls throttling. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Valid values must be greater than `0.0` or equal to `-1.0`, where `-1.0` means a default value is used. Defaults to `-1.0` +`xpack.ml.nightly_maintenance_rollover_max_size` +: ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The maximum size the anomaly detection results indices can reach before being rolled over by the nightly maintenance task. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Valid values must be greater than `0` or equal to `-1 Byte. Defaults to `50GB` + `xpack.ml.node_concurrent_job_allocations` : ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The maximum number of jobs that can concurrently be in the `opening` state on each node. Typically, jobs spend a small amount of time in this state before they move to `open` state. Jobs that must restore large models when they are opening spend more time in the `opening` state. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Defaults to `2`. diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 204dc5c57e4c6..03b34b43d8faa 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; +import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.Processors; import org.elasticsearch.common.util.FeatureFlag; @@ -717,6 +718,13 @@ public void loadExtensions(ExtensionLoader loader) { Property.NodeScope ); + public static final Setting NIGHTLY_MAINTENANCE_ROLLOVER_MAX_SIZE = Setting.byteSizeSetting( + "xpack.ml.nightly_maintenance_rollover_max_size", + ByteSizeValue.of(50, ByteSizeUnit.GB), + Property.OperatorDynamic, + Property.NodeScope + ); + /** * This is the maximum possible node size for a machine learning node. It is useful when determining if a job could ever be opened * on the cluster. @@ -841,6 +849,7 @@ public List> getSettings() { ModelLoadingService.INFERENCE_MODEL_CACHE_TTL, ResultsPersisterService.PERSIST_RESULTS_MAX_RETRIES, NIGHTLY_MAINTENANCE_REQUESTS_PER_SECOND, + NIGHTLY_MAINTENANCE_ROLLOVER_MAX_SIZE, MachineLearningField.USE_AUTO_MACHINE_MEMORY_PERCENT, MAX_ML_NODE_SIZE, DELAYED_DATA_CHECK_FREQ, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java index 6fedcb0da068c..87efa40fb57dc 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java @@ -114,6 +114,12 @@ public void afterStart() { MachineLearning.NIGHTLY_MAINTENANCE_REQUESTS_PER_SECOND, mlDailyMaintenanceService::setDeleteExpiredDataRequestsPerSecond ); + clusterService.getClusterSettings() + .addSettingsUpdateConsumer( + MachineLearning.NIGHTLY_MAINTENANCE_ROLLOVER_MAX_SIZE, + mlDailyMaintenanceService::setRolloverMaxSize + ); + } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index cd2cc501db4f5..39c3fa8190459 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -240,8 +240,7 @@ public void putJob( jobBuilder.validateModelSnapshotRetentionSettingsAndSetDefaults(); validateCategorizationAnalyzerOrSetDefault(jobBuilder, analysisRegistry, minNodeVersion); - // Job job = jobBuilder.build(new Date(), state, indexNameExpressionResolver); - Job job = jobBuilder.build(new Date()); + Job job = jobBuilder.build(new Date(), state, indexNameExpressionResolver); ActionListener putJobListener = new ActionListener<>() { @Override From cb85a496803213dbe741ff5428b4185a11148239 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Wed, 8 Oct 2025 16:20:42 +1300 Subject: [PATCH 14/42] Fix bad merge --- .../xpack/ml/MlDailyMaintenanceService.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index bfaa4b6304338..d20556dc05b1c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -97,6 +97,7 @@ public class MlDailyMaintenanceService implements Releasable { private volatile Scheduler.Cancellable cancellable; private volatile float deleteExpiredDataRequestsPerSecond; + private volatile ByteSizeValue rolloverMaxSize; MlDailyMaintenanceService( Settings settings, @@ -117,6 +118,7 @@ public class MlDailyMaintenanceService implements Releasable { this.schedulerProvider = Objects.requireNonNull(schedulerProvider); this.expressionResolver = Objects.requireNonNull(expressionResolver); this.deleteExpiredDataRequestsPerSecond = MachineLearning.NIGHTLY_MAINTENANCE_REQUESTS_PER_SECOND.get(settings); + this.rolloverMaxSize = MachineLearning.NIGHTLY_MAINTENANCE_ROLLOVER_MAX_SIZE.get(settings); this.isAnomalyDetectionEnabled = isAnomalyDetectionEnabled; this.isDataFrameAnalyticsEnabled = isDataFrameAnalyticsEnabled; this.isNlpEnabled = isNlpEnabled; @@ -152,6 +154,10 @@ void setDeleteExpiredDataRequestsPerSecond(float value) { this.deleteExpiredDataRequestsPerSecond = value; } + void setRolloverMaxSize(ByteSizeValue value) { + this.rolloverMaxSize = value; + } + /** * Calculates the delay until the next time the maintenance should be triggered. * The next time is 30 minutes past midnight of the following day plus a random @@ -355,8 +361,9 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio originSettingClient, new RolloverRequestBuilder(originSettingClient).setRolloverTarget(rolloverAlias) .setNewIndexName(newIndexName) - // TODO Make these conditions configurable settings? - .setConditions(RolloverConditions.newBuilder().addMaxIndexSizeCondition(ByteSizeValue.of(50, ByteSizeUnit.GB)).build()) + .setConditions( + RolloverConditions.newBuilder().addMaxIndexSizeCondition(rolloverMaxSize).build() + ) .request(), rolloverListener ); From 123c8cb484088c5cf009645f49a464ada29d1a5e Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 8 Oct 2025 03:29:56 +0000 Subject: [PATCH 15/42] [CI] Auto commit changes from spotless --- .../elasticsearch/xpack/ml/MlDailyMaintenanceService.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index d20556dc05b1c..25a9109539f45 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -27,7 +27,6 @@ import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; @@ -361,9 +360,7 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio originSettingClient, new RolloverRequestBuilder(originSettingClient).setRolloverTarget(rolloverAlias) .setNewIndexName(newIndexName) - .setConditions( - RolloverConditions.newBuilder().addMaxIndexSizeCondition(rolloverMaxSize).build() - ) + .setConditions(RolloverConditions.newBuilder().addMaxIndexSizeCondition(rolloverMaxSize).build()) .request(), rolloverListener ); From 80b1b7186ba7222a5a3881ef6df664a0c1f39c1f Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 9 Oct 2025 11:31:11 +1300 Subject: [PATCH 16/42] Test fixes --- .../rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml index 3027fc07e718a..370639afb2520 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml @@ -72,7 +72,7 @@ setup: # killing the node - do: cluster.health: - index: [".ml-state-000001", ".ml-anomalies-shared-000001"] + index: [".ml-state-000001", ".ml-anomalies-shared"] wait_for_status: green --- @@ -136,7 +136,7 @@ setup: # killing the node - do: cluster.health: - index: [".ml-state-000001", ".ml-anomalies-shared-000001"] + index: [".ml-state-000001", ".ml-anomalies-shared"] wait_for_status: green --- @@ -196,7 +196,7 @@ setup: # killing the node - do: cluster.health: - index: [".ml-state-000001", ".ml-anomalies-shared-000001"] + index: [".ml-state-000001", ".ml-anomalies-shared"] wait_for_status: green --- @@ -259,7 +259,7 @@ setup: # killing the node - do: cluster.health: - index: [".ml-state-000001", ".ml-anomalies-shared-000001"] + index: [".ml-state-000001", ".ml-anomalies-shared"] wait_for_status: green --- From 2a66d405b743a1f8a7e50c23889c95ed00081524 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 9 Oct 2025 12:47:41 +1300 Subject: [PATCH 17/42] Remove assertion for condition that is no longer entirely true --- .../elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java index 5bacc59d39a00..d9f2b5181a1d7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java @@ -487,8 +487,8 @@ public static String latestIndexMatchingBaseName( baseIndexName + "*" ); - // This should never happen - assert matching.length > 0 : "No indices matching [" + baseIndexName + "*]"; + // We used to assert here if no matching indices could be found. However, when called _before_ a job is created it may be the case + // that no .ml-anomalies-shared* indices yet exist if (matching.length == 0) { return index; } From 83e7e2bf6a6828ddbc9a27b3735b93494db8b639 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 9 Oct 2025 16:45:26 +1300 Subject: [PATCH 18/42] A few more test fixes --- .../xpack/ml/integration/AnomalyJobCRUDIT.java | 8 ++++---- .../xpack/ml/integration/BasicDistributedJobsIT.java | 4 ++-- .../xpack/ml/integration/JobResultsProviderIT.java | 2 +- .../xpack/ml/integration/JobStorageDeletionTaskIT.java | 4 ++-- .../xpack/ml/MlAnomaliesIndexUpdateTests.java | 2 +- .../ml/task/AbstractJobPersistentTasksExecutorTests.java | 6 +++--- .../restart/MlHiddenIndicesFullClusterRestartIT.java | 4 ++-- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnomalyJobCRUDIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnomalyJobCRUDIT.java index dcc16c0bea23b..ee8f3c24e90bd 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnomalyJobCRUDIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnomalyJobCRUDIT.java @@ -139,7 +139,7 @@ public void testCreateWithExistingQuantilesDocs() { public void testCreateWithExistingResultsDocs() { String jobId = "job-id-with-existing-docs"; testCreateWithExistingDocs( - prepareIndex(".ml-anomalies-shared").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + prepareIndex(".ml-anomalies-shared-000001").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .setId(jobId + "_1464739200000_1") .setSource("{\"job_id\": \"" + jobId + "\"}", XContentType.JSON) .request(), @@ -149,14 +149,14 @@ public void testCreateWithExistingResultsDocs() { public void testPutJobWithClosedResultsIndex() { String jobId = "job-with-closed-results-index"; - client().admin().indices().prepareCreate(".ml-anomalies-shared").get(); - client().admin().indices().prepareClose(".ml-anomalies-shared").get(); + client().admin().indices().prepareCreate(".ml-anomalies-shared-000001").get(); + client().admin().indices().prepareClose(".ml-anomalies-shared-000001").get(); ElasticsearchStatusException ex = expectThrows(ElasticsearchStatusException.class, () -> createJob(jobId)); assertThat( ex.getMessage(), containsString("Cannot create job [job-with-closed-results-index] as it requires closed index [.ml-anomalies-*]") ); - client().admin().indices().prepareDelete(".ml-anomalies-shared").get(); + client().admin().indices().prepareDelete(".ml-anomalies-shared-000001").get(); } public void testPutJobWithClosedStateIndex() { diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java index 842ed7a4a2a2e..194747e44cffe 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java @@ -369,7 +369,7 @@ public void testMlStateAndResultsIndicesNotAvailable() throws Exception { // Create the indices (using installed templates) and set the routing to specific nodes // State and results go on the state-and-results node, config goes on the config node - indicesAdmin().prepareCreate(".ml-anomalies-shared") + indicesAdmin().prepareCreate(".ml-anomalies-shared-000001") .setSettings( Settings.builder() .put("index.routing.allocation.include.ml-indices", "state-and-results") @@ -435,7 +435,7 @@ public void testMlStateAndResultsIndicesNotAvailable() throws Exception { ); assertThat(detailedMessage, containsString("because not all primary shards are active for the following indices")); assertThat(detailedMessage, containsString(".ml-state")); - assertThat(detailedMessage, containsString(".ml-anomalies-shared")); + assertThat(detailedMessage, containsString(".ml-anomalies-shared-000001")); logger.info("Start data node"); String nonMlNode = internalCluster().startNode( diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java index 9a513f2690917..d676e2405ace3 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java @@ -208,7 +208,7 @@ public void testPutJob_WithCustomResultsIndex() { client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(job)).actionGet(); - String customIndex = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-bar"; + String customIndex = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-bar-000001"; Map mappingProperties = getIndexMappingProperties(customIndex); // Assert mappings have a few fields from the template diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobStorageDeletionTaskIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobStorageDeletionTaskIT.java index 7e46f260ea475..61acf64fa5efe 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobStorageDeletionTaskIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobStorageDeletionTaskIT.java @@ -138,14 +138,14 @@ public void testDeleteDedicatedJobWithDataInShared() throws Exception { IndicesAliasesRequest.AliasActions.add() .alias(AnomalyDetectorsIndex.jobResultsAliasedName(jobIdDedicated)) .isHidden(true) - .index(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared") + .index(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared-000001") .writeIndex(false) .filter(QueryBuilders.boolQuery().filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), jobIdDedicated))) ) .addAliasAction( IndicesAliasesRequest.AliasActions.add() .alias(AnomalyDetectorsIndex.resultsWriteAlias(jobIdDedicated)) - .index(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared") + .index(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared-000001") .isHidden(true) .writeIndex(true) ) diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java index 1ecf91e2a7ea9..d77e60bfdd4af 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java @@ -61,7 +61,7 @@ public void testIsAbleToRun_IndicesDoNotExist() { } public void testIsAbleToRun_IndicesHaveNoRouting() { - IndexMetadata.Builder indexMetadata = IndexMetadata.builder(".ml-anomalies-shared"); + IndexMetadata.Builder indexMetadata = IndexMetadata.builder(".ml-anomalies-shared-000001"); indexMetadata.settings( Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutorTests.java index 98169d1aa6f5b..97ceb5365c03d 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutorTests.java @@ -56,7 +56,7 @@ public void testVerifyIndicesPrimaryShardsAreActive() { cs, resolver, true, - ".ml-anomalies-shared", + ".ml-anomalies-shared-000001", AnomalyDetectorsIndex.jobStateIndexPattern(), MlMetaIndex.indexName(), MlConfigIndex.indexName() @@ -69,7 +69,7 @@ public void testVerifyIndicesPrimaryShardsAreActive() { resolver.concreteIndexNames( cs, IndicesOptions.lenientExpandOpen(), - ".ml-anomalies-shared", + ".ml-anomalies-shared-000001", AnomalyDetectorsIndex.jobStateIndexPattern(), MlMetaIndex.indexName(), MlConfigIndex.indexName() @@ -100,7 +100,7 @@ public void testVerifyIndicesPrimaryShardsAreActive() { csBuilder.build(), resolver, true, - ".ml-anomalies-shared", + ".ml-anomalies-shared-000001", AnomalyDetectorsIndex.jobStateIndexPattern(), MlMetaIndex.indexName(), MlConfigIndex.indexName() diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index a83ad5b4f8da4..0797f375a5955 100644 --- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -45,8 +45,8 @@ public class MlHiddenIndicesFullClusterRestartIT extends AbstractXpackFullCluste Tuple.tuple(List.of(".ml-annotations-000001"), ".ml-annotations-read"), Tuple.tuple(List.of(".ml-annotations-000001"), ".ml-annotations-write"), Tuple.tuple(List.of(".ml-state", ".ml-state-000001"), ".ml-state-write"), - Tuple.tuple(List.of(".ml-anomalies-shared"), ".ml-anomalies-" + JOB_ID), - Tuple.tuple(List.of(".ml-anomalies-shared"), ".ml-anomalies-.write-" + JOB_ID) + Tuple.tuple(List.of(".ml-anomalies-shared-000001"), ".ml-anomalies-" + JOB_ID), + Tuple.tuple(List.of(".ml-anomalies-shared-000001"), ".ml-anomalies-.write-" + JOB_ID) ); public MlHiddenIndicesFullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) { From e9c0c2c12704a2e8ed4653ad11d6d6289759760e Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 9 Oct 2025 16:56:02 +1300 Subject: [PATCH 19/42] Fixed typo in docs --- .../configuration-reference/machine-learning-settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md b/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md index 0d04af8d1f2be..5eff8c71b3baa 100644 --- a/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md +++ b/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md @@ -87,7 +87,7 @@ $$$xpack.ml.max_open_jobs$$$ : ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The rate at which the nightly maintenance task deletes expired model snapshots and results. The setting is a proxy to the [`requests_per_second`](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-delete-by-query) parameter used in the delete by query requests and controls throttling. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Valid values must be greater than `0.0` or equal to `-1.0`, where `-1.0` means a default value is used. Defaults to `-1.0` `xpack.ml.nightly_maintenance_rollover_max_size` -: ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The maximum size the anomaly detection results indices can reach before being rolled over by the nightly maintenance task. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Valid values must be greater than `0` or equal to `-1 Byte. Defaults to `50GB` +: ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The maximum size the anomaly detection results indices can reach before being rolled over by the nightly maintenance task. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Valid values must be greater than `0B` or equal to `-1B`. Defaults to `50GB`. `xpack.ml.node_concurrent_job_allocations` : ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The maximum number of jobs that can concurrently be in the `opening` state on each node. Typically, jobs spend a small amount of time in this state before they move to `open` state. Jobs that must restore large models when they are opening spend more time in the `opening` state. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Defaults to `2`. From 0b6b79aa5c471ef37cb5459ada99f5448172cb4e Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 10 Oct 2025 13:27:13 +1300 Subject: [PATCH 20/42] Tweaks to yamlRestCompatTests --- x-pack/plugin/build.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index ea715b0d5c921..92982352254da 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -85,6 +85,9 @@ tasks.named("precommit").configure { } tasks.named("yamlRestCompatTestTransform").configure({ task -> + task.replaceValueTextByKeyValue("index", ".ml-anomalies-shared", ".ml-anomalies-shared-000001") + task.replaceValueTextByKeyValue("index", ".ml-anomalies-custom-all-test-1", ".ml-anomalies-custom-all-test-1-000001") + task.replaceValueTextByKeyValue("index", ".ml-anomalies-custom-all-test-2", ".ml-anomalies-custom-all-test-2-000001") task.skipTest("esql/60_usage/Basic ESQL usage output (telemetry)", "The telemetry output changed. We dropped a column. That's safe.") task.skipTest("inference/inference_crud/Test get all", "Assertions on number of inference models break due to default configs") task.skipTest("esql/60_usage/Basic ESQL usage output (telemetry) snapshot version", "The number of functions is constantly increasing") From 97106a1e2edc976642fb5479797ea8da2f72975d Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 10 Oct 2025 17:20:22 +1300 Subject: [PATCH 21/42] A few more test fixes --- .../rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml | 2 +- .../rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml index e8f65cea773d4..1a51e5a4d9e1c 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml @@ -141,7 +141,7 @@ # killing the node - do: cluster.health: - index: [".ml-state-000001", ".ml-anomalies-shared-000001"] + index: [".ml-state-000001", ".ml-anomalies-shared*"] wait_for_status: green --- diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml index 370639afb2520..fb9608c9cb3a8 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml @@ -72,7 +72,7 @@ setup: # killing the node - do: cluster.health: - index: [".ml-state-000001", ".ml-anomalies-shared"] + index: [".ml-state-000001", ".ml-anomalies*"] wait_for_status: green --- @@ -136,7 +136,7 @@ setup: # killing the node - do: cluster.health: - index: [".ml-state-000001", ".ml-anomalies-shared"] + index: [".ml-state-000001", ".ml-anomalies*"] wait_for_status: green --- @@ -196,7 +196,7 @@ setup: # killing the node - do: cluster.health: - index: [".ml-state-000001", ".ml-anomalies-shared"] + index: [".ml-state-000001", ".ml-anomalies*"] wait_for_status: green --- @@ -259,7 +259,7 @@ setup: # killing the node - do: cluster.health: - index: [".ml-state-000001", ".ml-anomalies-shared"] + index: [".ml-state-000001", ".ml-anomalies*"] wait_for_status: green --- From 468f7124516e363681481f0c5255624bf550b33f Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Mon, 13 Oct 2025 14:33:53 +1300 Subject: [PATCH 22/42] Another test fix through rewrite rules --- x-pack/plugin/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 92982352254da..85830b6006cec 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -85,6 +85,8 @@ tasks.named("precommit").configure { } tasks.named("yamlRestCompatTestTransform").configure({ task -> + task.replaceIsTrue("\\.ml-anomalies-shared.mappings._meta.version", "\\.ml-anomalies-shared-000001.mappings._meta.version") + task.replaceKeyInMatch("\\.ml-anomalies-shared.mappings.new_field.mapping.new_field.type", "\\.ml-anomalies-shared-000001.mappings.new_field.mapping.new_field.type") task.replaceValueTextByKeyValue("index", ".ml-anomalies-shared", ".ml-anomalies-shared-000001") task.replaceValueTextByKeyValue("index", ".ml-anomalies-custom-all-test-1", ".ml-anomalies-custom-all-test-1-000001") task.replaceValueTextByKeyValue("index", ".ml-anomalies-custom-all-test-2", ".ml-anomalies-custom-all-test-2-000001") From 63f97c59a17182e960dec0fc77d57adae1fa2d44 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Mon, 13 Oct 2025 16:58:00 +1300 Subject: [PATCH 23/42] Another REST compatibility test transformation --- x-pack/plugin/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 85830b6006cec..6bde49708693c 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -88,6 +88,7 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.replaceIsTrue("\\.ml-anomalies-shared.mappings._meta.version", "\\.ml-anomalies-shared-000001.mappings._meta.version") task.replaceKeyInMatch("\\.ml-anomalies-shared.mappings.new_field.mapping.new_field.type", "\\.ml-anomalies-shared-000001.mappings.new_field.mapping.new_field.type") task.replaceValueTextByKeyValue("index", ".ml-anomalies-shared", ".ml-anomalies-shared-000001") + task.replaceValueTextByKeyValue("index", ".ml-anomalies-custom-all-test-1,.ml-anomalies-custom-all-test-2", ".ml-anomalies-custom-all-test-1-000001,.ml-anomalies-custom-all-test-2-000001") task.replaceValueTextByKeyValue("index", ".ml-anomalies-custom-all-test-1", ".ml-anomalies-custom-all-test-1-000001") task.replaceValueTextByKeyValue("index", ".ml-anomalies-custom-all-test-2", ".ml-anomalies-custom-all-test-2-000001") task.skipTest("esql/60_usage/Basic ESQL usage output (telemetry)", "The telemetry output changed. We dropped a column. That's safe.") From c320cccc06e68dffe169a6cf6660c1196e5fe272 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 16 Oct 2025 14:12:02 +1300 Subject: [PATCH 24/42] First draft of integration tests --- .../core/src/main/java/module-info.java | 1 - .../xpack/core/ml/job/config/Job.java | 26 +- .../AnomalyDetectorsIndexFields.java | 1 + .../xpack/core/ml/utils/MlIndexAndAlias.java | 3 +- .../xpack/ml/integration/MlJobIT.java | 196 +++++++++- ...enanceServiceRolloverResultsIndicesIT.java | 337 ++++++++++++++++++ .../xpack/ml/MlAnomaliesIndexUpdate.java | 2 +- .../xpack/ml/MlDailyMaintenanceService.java | 22 +- 8 files changed, 555 insertions(+), 33 deletions(-) create mode 100644 x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java diff --git a/x-pack/plugin/core/src/main/java/module-info.java b/x-pack/plugin/core/src/main/java/module-info.java index 42c4910d82782..bb183f8d3845e 100644 --- a/x-pack/plugin/core/src/main/java/module-info.java +++ b/x-pack/plugin/core/src/main/java/module-info.java @@ -26,7 +26,6 @@ requires org.apache.httpcomponents.client5.httpclient5; requires org.apache.httpcomponents.core5.httpcore5; requires org.slf4j; - requires org.elasticsearch.logging; exports org.elasticsearch.index.engine.frozen; exports org.elasticsearch.license; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index e98c551a536de..fc9cfdd2fb946 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -809,8 +809,8 @@ public static class Builder implements Writeable { private boolean allowLazyOpen; private Blocked blocked = Blocked.none(); private DatafeedConfig.Builder datafeedConfig; - private SetOnce clusterState = new SetOnce<>(); - private SetOnce indexNameExpressionResolver = new SetOnce<>(); + private ClusterState clusterState; + private IndexNameExpressionResolver indexNameExpressionResolver; public Builder() {} @@ -886,11 +886,11 @@ public String getId() { } private void setClusterState(ClusterState state) { - this.clusterState.set(state); + this.clusterState = state; } private void setIndexNameExpressionResolver(IndexNameExpressionResolver indexNameExpressionResolver) { - this.indexNameExpressionResolver.set(indexNameExpressionResolver); + this.indexNameExpressionResolver = indexNameExpressionResolver; } public void setJobVersion(MlConfigVersion jobVersion) { @@ -1368,19 +1368,21 @@ public Job build() { if (Strings.isNullOrEmpty(resultsIndexName)) { resultsIndexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; - } else if (resultsIndexName.equals(AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT) == false) { - // User-defined names are prepended with "custom" and end with a 6 digit suffix - // Conditional guards against multiple prepending due to updates instead of first creation - resultsIndexName = resultsIndexName.startsWith("custom-") ? resultsIndexName : "custom-" + resultsIndexName; - } + } else if ((resultsIndexName.startsWith(AnomalyDetectorsIndexFields.RESULTS_INDEX_SHARED) + && MlIndexAndAlias.has6DigitSuffix(resultsIndexName) + && resultsIndexName.length() == AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT.length()) == false) { + // User-defined names are prepended with "custom" and end with a 6 digit suffix + // Conditional guards against multiple prepending due to updates instead of first creation + resultsIndexName = resultsIndexName.startsWith("custom-") ? resultsIndexName : "custom-" + resultsIndexName; + } resultsIndexName = MlIndexAndAlias.has6DigitSuffix(resultsIndexName) ? resultsIndexName : resultsIndexName + "-000001"; - if (indexNameExpressionResolver.get() != null && clusterState.get() != null) { + if (indexNameExpressionResolver != null && clusterState != null) { String tmpResultsIndexName = MlIndexAndAlias.latestIndexMatchingBaseName( AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + resultsIndexName, - indexNameExpressionResolver.get(), - clusterState.get() + indexNameExpressionResolver, + clusterState ); resultsIndexName = tmpResultsIndexName.substring(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX.length()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java index d36d031a6a4c3..3c4a65155af7f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java @@ -14,6 +14,7 @@ public final class AnomalyDetectorsIndexFields { // ".write" rather than simply "write" to avoid the danger of clashing // with the read alias of a job whose name begins with "write-" public static final String RESULTS_INDEX_WRITE_PREFIX = RESULTS_INDEX_PREFIX + ".write-"; + public static final String RESULTS_INDEX_SHARED = "shared"; public static final String RESULTS_INDEX_DEFAULT = "shared-000001"; private AnomalyDetectorsIndexFields() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java index d9f2b5181a1d7..58b3c51894a06 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java @@ -517,13 +517,12 @@ public static void rollover(Client client, RolloverRequest rolloverRequest, Acti } public static void createAliasForRollover( - org.elasticsearch.logging.Logger logger, Client client, String indexName, String aliasName, ActionListener listener ) { - logger.warn("creating rollover [{}] alias for [{}]", aliasName, indexName); + logger.info("creating rollover [{}] alias for [{}]", aliasName, indexName); client.admin() .indices() .prepareAliases(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS) diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java index 1c1acc99f02ed..1b1dc44eea72e 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.logging.LogManager; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.SecuritySettingsSourceField; import org.elasticsearch.test.rest.ESRestTestCase; @@ -81,6 +82,20 @@ public void testPutJob_GivenFarequoteConfig() throws Exception { Response response = createFarequoteJob("given-farequote-config-job"); String responseAsString = EntityUtils.toString(response.getEntity()); assertThat(responseAsString, containsString("\"job_id\":\"given-farequote-config-job\"")); + assertThat(responseAsString, containsString("\"results_index_name\":\"shared-000001\"")); + + String mlIndicesResponseAsString = getMlResultsIndices(); + assertThat(mlIndicesResponseAsString, containsString("green open .ml-anomalies-shared-000001")); + + String aliasesResponseAsString = getAliases(); + LogManager.getLogger(MlRestTestStateCleaner.class).warn(aliasesResponseAsString); + assertThat( + aliasesResponseAsString, + containsString( + "\".ml-anomalies-shared-000001\":{\"aliases\":" + + "{\".ml-anomalies-.write-given-farequote-config-job\":{\"is_hidden\":true},\".ml-anomalies-given-farequote-config-job\"" + ) + ); } public void testGetJob_GivenNoSuchJob() { @@ -257,11 +272,7 @@ public void testCreateJobsWithIndexNameOption() throws Exception { } }); - // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652 - String responseAsString = EntityUtils.toString( - client().performRequest(new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")) - .getEntity() - ); + String responseAsString = getMlResultsIndices(); assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName)); assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1)))); assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2)))); @@ -326,10 +337,7 @@ public void testCreateJobsWithIndexNameOption() throws Exception { assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1)))); assertThat(responseAsString, containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))); // job2 still exists - responseAsString = EntityUtils.toString( - client().performRequest(new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")) - .getEntity() - ); + responseAsString = getMlResultsIndices(); assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName)); refreshAllIndices(); @@ -347,13 +355,161 @@ public void testCreateJobsWithIndexNameOption() throws Exception { assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2)))); refreshAllIndices(); + responseAsString = getMlResultsIndices(); + assertThat( + responseAsString, + not(containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName + "-000001")) + ); + } + + // The same as testCreateJobsWithIndexNameOption but we don't supply the "-000001" suffix to the index name supplied in the job config + // We test that the final index name does indeed have the suffix. + public void testCreateJobsWithIndexNameNo6DigitSuffixOption() throws Exception { + String jobTemplate = """ + { + "analysis_config" : { + "detectors" :[{"function":"metric","field_name":"responsetime"}] + }, + "data_description": {}, + "results_index_name" : "%s"}"""; + + String jobId1 = "create-jobs-with-index-name-option-job-1"; + String indexName = "non-default-index"; + putJob(jobId1, Strings.format(jobTemplate, indexName)); + + String jobId2 = "create-jobs-with-index-name-option-job-2"; + putJob(jobId2, Strings.format(jobTemplate, indexName)); + + // With security enabled GET _aliases throws an index_not_found_exception + // if no aliases have been created. In multi-node tests the alias may not + // appear immediately so wait here. + assertBusy(() -> { + try { + String aliasesResponse = getAliases(); + assertThat(aliasesResponse, containsString(Strings.format(""" + "%s":{"aliases":{""", AnomalyDetectorsIndex.jobResultsAliasedName("custom-" + indexName+"-000001")))); + assertThat( + aliasesResponse, + containsString( + Strings.format( + """ + "%s":{"filter":{"term":{"job_id":{"value":"%s"}}},"is_hidden":true}""", + AnomalyDetectorsIndex.jobResultsAliasedName(jobId1), + jobId1 + ) + ) + ); + assertThat(aliasesResponse, containsString(Strings.format(""" + "%s":{"is_hidden":true}""", AnomalyDetectorsIndex.resultsWriteAlias(jobId1)))); + assertThat( + aliasesResponse, + containsString( + Strings.format( + """ + "%s":{"filter":{"term":{"job_id":{"value":"%s"}}},"is_hidden":true}""", + AnomalyDetectorsIndex.jobResultsAliasedName(jobId2), + jobId2 + ) + ) + ); + assertThat(aliasesResponse, containsString(Strings.format(""" + "%s":{"is_hidden":true}""", AnomalyDetectorsIndex.resultsWriteAlias(jobId2)))); + } catch (ResponseException e) { + throw new AssertionError(e); + } + }); + + String responseAsString = getMlResultsIndices(); + assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName+"-000001")); + assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1)))); + assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2)))); + + { // create jobId1 docs + String id = Strings.format("%s_bucket_%s_%s", jobId1, "1234", 300); + Request createResultRequest = new Request("PUT", AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) + "/_doc/" + id); + createResultRequest.setJsonEntity(Strings.format(""" + {"job_id":"%s", "timestamp": "%s", "result_type":"bucket", "bucket_span": "%s"}""", jobId1, "1234", 1)); + client().performRequest(createResultRequest); + + id = Strings.format("%s_bucket_%s_%s", jobId1, "1236", 300); + createResultRequest = new Request("PUT", AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) + "/_doc/" + id); + createResultRequest.setJsonEntity(Strings.format(""" + {"job_id":"%s", "timestamp": "%s", "result_type":"bucket", "bucket_span": "%s"}""", jobId1, "1236", 1)); + client().performRequest(createResultRequest); + + refreshAllIndices(); + + responseAsString = EntityUtils.toString( + client().performRequest(new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId1 + "/results/buckets")) + .getEntity() + ); + assertThat(responseAsString, containsString("\"count\":2")); + + responseAsString = EntityUtils.toString( + client().performRequest(new Request("GET", AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) + "/_search")).getEntity() + ); + assertThat(responseAsString, containsString("\"value\":2")); + } + { // create jobId2 docs + String id = Strings.format("%s_bucket_%s_%s", jobId2, "1234", 300); + Request createResultRequest = new Request("PUT", AnomalyDetectorsIndex.jobResultsAliasedName(jobId2) + "/_doc/" + id); + createResultRequest.setJsonEntity(Strings.format(""" + {"job_id":"%s", "timestamp": "%s", "result_type":"bucket", "bucket_span": "%s"}""", jobId2, "1234", 1)); + client().performRequest(createResultRequest); + + id = Strings.format("%s_bucket_%s_%s", jobId2, "1236", 300); + createResultRequest = new Request("PUT", AnomalyDetectorsIndex.jobResultsAliasedName(jobId2) + "/_doc/" + id); + createResultRequest.setJsonEntity(Strings.format(""" + {"job_id":"%s", "timestamp": "%s", "result_type":"bucket", "bucket_span": "%s"}""", jobId2, "1236", 1)); + client().performRequest(createResultRequest); + + refreshAllIndices(); + + responseAsString = EntityUtils.toString( + client().performRequest(new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId2 + "/results/buckets")) + .getEntity() + ); + assertThat(responseAsString, containsString("\"count\":2")); + + responseAsString = EntityUtils.toString( + client().performRequest(new Request("GET", AnomalyDetectorsIndex.jobResultsAliasedName(jobId2) + "/_search")).getEntity() + ); + assertThat(responseAsString, containsString("\"value\":2")); + } + + client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId1)); + + // check that indices still exist, but no longer have job1 entries and aliases are gone + responseAsString = getAliases(); + assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1)))); + assertThat(responseAsString, containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))); // job2 still exists + + responseAsString = getMlResultsIndices(); + assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName+"-000001")); + + refreshAllIndices(); + responseAsString = EntityUtils.toString( - client().performRequest(new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")) - .getEntity() + client().performRequest( + new Request("GET", AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName+"-000001" + "/_count") + ).getEntity() + ); + assertThat(responseAsString, containsString("\"count\":2")); + + // Delete the second job and verify aliases are gone, and original concrete/custom index is gone + client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId2)); + responseAsString = getAliases(); + assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2)))); + + refreshAllIndices(); + responseAsString = getMlResultsIndices(); + assertThat( + responseAsString, + not(containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName + "-000001")) ); - assertThat(responseAsString, not(containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName))); } + public void testCreateJobInSharedIndexUpdatesMapping() throws Exception { String jobTemplate = """ { @@ -370,6 +526,14 @@ public void testCreateJobInSharedIndexUpdatesMapping() throws Exception { putJob(jobId1, Strings.format(jobTemplate, byFieldName1)); + String mlIndicesResponseAsString = getMlResultsIndices(); + assertThat( + mlIndicesResponseAsString, + containsString( + "green open " + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT + ) + ); + // Check the index mapping contains the first by_field_name Request getResultsMappingRequest = new Request( "GET", @@ -974,6 +1138,14 @@ public void testDelete_multipleRequest() throws Exception { assertEquals(numThreads, recreationGuard.get()); } + private String getMlResultsIndices() throws IOException { + // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652 + return EntityUtils.toString( + client().performRequest(new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")) + .getEntity() + ); + } + private String getAliases() throws IOException { final Request aliasesRequest = new Request("GET", "/_aliases"); // Allow system index deprecation warnings - this can be removed once system indices are omitted from responses rather than diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java new file mode 100644 index 0000000000000..465262c6eb00c --- /dev/null +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java @@ -0,0 +1,337 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.ml.integration; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.admin.indices.rollover.RolloverConditions; +import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.indices.TestIndexNameExpressionResolver; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ml.action.PutJobAction; +import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.ml.MlAssignmentNotifier; +import org.elasticsearch.xpack.ml.MlDailyMaintenanceService; + +import org.junit.Before; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; + +import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1, numClientNodes = 0, supportsDedicatedMasters = false) +public class MlDailyMaintenanceServiceRolloverResultsIndicesIT extends BaseMlIntegTestCase { + + private MlDailyMaintenanceService maintenanceService; + + @Before + public void createComponents() throws Exception { + Settings settings = nodeSettings(0, Settings.EMPTY); + ThreadPool threadPool = mockThreadPool(); + + ClusterService clusterService = internalCluster().clusterService(internalCluster().getMasterName()); + + initClusterAndJob(); + + maintenanceService = new MlDailyMaintenanceService( + settings(IndexVersion.current()).build(), + ClusterName.DEFAULT, + threadPool, + client(), + clusterService, + mock(MlAssignmentNotifier.class), + TestIndexNameExpressionResolver.newInstance(), + true, + true, + true + ); + + // replace the default set of conditions with an empty set so we can roll the index unconditionally + // It's not the conditions or even the rollover itself we are testing but the state of the indices and aliases afterwards. + maintenanceService.setRolloverConditions(RolloverConditions.newBuilder().build()); + } + + private void initClusterAndJob() { + internalCluster().ensureAtLeastNumDataNodes(1); + ensureStableCluster(1); + } + + public void testTriggerRollResultsIndicesIfNecessaryTask_givenNoIndices() throws Exception { + // The null case, nothing to do. + + { + GetIndexResponse getIndexResponse = client().admin() + .indices() + .prepareGetIndex(TEST_REQUEST_TIMEOUT) + .setIndices(".ml-anomalies*") + .get(); + logger.warn("get_index_response: {}", getIndexResponse.toString()); + assertThat(getIndexResponse.getIndices().length, is(0)); + var aliases = getIndexResponse.getAliases(); + assertThat(aliases.size(), is(0)); + } + + blockingCall(maintenanceService::triggerRollResultsIndicesIfNecessaryTask); + + { + GetIndexResponse getIndexResponse = client().admin() + .indices() + .prepareGetIndex(TEST_REQUEST_TIMEOUT) + .setIndices(".ml-anomalies*") + .get(); + logger.warn("get_index_response: {}", getIndexResponse.toString()); + assertThat(getIndexResponse.getIndices().length, is(0)); + var aliases = getIndexResponse.getAliases(); + assertThat(aliases.size(), is(0)); + } + } + + public void testTriggerRollResultsIndicesIfNecessaryTask() throws Exception { + + // Create jobs that will use the default results indices - ".ml-anomalies-shared-*" + Job.Builder[] jobs_with_default_index = { createJob("job_using_default_index"), createJob("another_job_using_default_index") }; + + // Create jobs that will use custom results indices - ".ml-anomalies-custom-fred-*" + Job.Builder[] jobs_with_custom_index = { + createJob("job_using_custom_index").setResultsIndexName("fred"), + createJob("another_job_using_custom_index").setResultsIndexName("fred") }; + + Job.Builder[][] job_lists = { jobs_with_default_index, jobs_with_custom_index }; + + for (Job.Builder[] job_list : job_lists) { + putJob(job_list[0]); + String jobId = job_list[0].getId(); + String index = (jobId.contains("job_using_custom_index")) ? "custom-fred" : "shared"; + { + GetIndexResponse getIndexResponse = client().admin() + .indices() + .prepareGetIndex(TEST_REQUEST_TIMEOUT) + .setIndices(".ml-anomalies-" + index + "*") + .get(); + logger.warn("get_index_response: {}", getIndexResponse.toString()); + assertThat(getIndexResponse.getIndices().length, is(1)); + var aliases = getIndexResponse.getAliases(); + assertThat(aliases.size(), is(1)); + + StringBuilder sb = new StringBuilder("Before Rollover. Aliases found:\n"); + + List aliasMetadata = aliases.get(".ml-anomalies-" + index + "-000001"); + + assertThat(aliasMetadata.size(), is(2)); + + List aliasesList = new ArrayList<>(aliasMetadata.stream().map(AliasMetadata::alias).toList()); + + assertThat(aliasesList, containsInAnyOrder(".ml-anomalies-.write-" + jobId, ".ml-anomalies-" + jobId)); + + sb.append(" Index [").append(".ml-anomalies-shared-000001").append("]: ").append(aliasesList).append("\n"); + + logger.warn(sb.toString().trim()); + } + + blockingCall(maintenanceService::triggerRollResultsIndicesIfNecessaryTask); + + // Check indices and aliases after rollover, there should be a read alias for ".ml-anomalies--000001" + // and read/write aliases for ".ml-anomalies--000002". There should be no other aliases pointing to these indices. + { + GetIndexResponse getIndexResponse = client().admin() + .indices() + .prepareGetIndex(TEST_REQUEST_TIMEOUT) + .setIndices(".ml-anomalies-" + index + "*") + .get(); + logger.warn("get_index_response: {}", getIndexResponse.toString()); + assertThat(getIndexResponse.getIndices().length, is(2)); + var aliases = getIndexResponse.getAliases(); + assertThat(aliases.size(), is(2)); + + StringBuilder sb = new StringBuilder("After Rollover. Aliases found:\n"); + List aliasMetadata1 = aliases.get(".ml-anomalies-" + index + "-000001"); + List aliasMetadata2 = aliases.get(".ml-anomalies-" + index + "-000002"); + + assertThat(aliasMetadata1.size(), is(1)); + assertThat(aliasMetadata2.size(), is(2)); + + List aliases1List = new ArrayList<>(aliasMetadata1.stream().map(AliasMetadata::alias).toList()); + List aliases2List = new ArrayList<>(aliasMetadata2.stream().map(AliasMetadata::alias).toList()); + + assertThat(aliases1List, containsInAnyOrder(".ml-anomalies-" + jobId)); + assertThat(aliases2List, containsInAnyOrder(".ml-anomalies-.write-" + jobId, ".ml-anomalies-" + jobId)); + + sb.append(" Index [") + .append(".ml-anomalies-") + .append(index) + .append("-000001") + .append("]: ") + .append(aliases1List) + .append("\n"); + sb.append(" Index [") + .append(".ml-anomalies-") + .append(index) + .append("-000002") + .append("]: ") + .append(aliases2List) + .append("\n"); + + logger.warn(sb.toString().trim()); + } + + // Now open another job. + putJob(job_list[1]); + + // Check indices and aliases, there should be new read/write aliases for the new job + // pointing to ".ml-anomalies--000002". + { + GetIndexResponse getIndexResponse = client().admin() + .indices() + .prepareGetIndex(TEST_REQUEST_TIMEOUT) + .setIndices(".ml-anomalies-" + index + "*") + .get(); + logger.warn("get_index_response: {}", getIndexResponse.toString()); + assertThat(getIndexResponse.getIndices().length, is(2)); + var aliases = getIndexResponse.getAliases(); + assertThat(aliases.size(), is(2)); + + StringBuilder sb = new StringBuilder("After 2nd Job creation. Aliases found:\n"); + List aliasMetadata1 = aliases.get(".ml-anomalies-" + index + "-000001"); + List aliasMetadata2 = aliases.get(".ml-anomalies-" + index + "-000002"); + + assertThat(aliasMetadata1.size(), is(1)); + assertThat(aliasMetadata2.size(), is(4)); + + List aliases1List = new ArrayList<>(aliasMetadata1.stream().map(AliasMetadata::alias).toList()); + List aliases2List = new ArrayList<>(aliasMetadata2.stream().map(AliasMetadata::alias).toList()); + + assertThat(aliases1List, containsInAnyOrder(".ml-anomalies-" + jobId)); + assertThat( + aliases2List, + containsInAnyOrder( + ".ml-anomalies-.write-" + jobId, + ".ml-anomalies-" + jobId, + ".ml-anomalies-.write-another_" + jobId, + ".ml-anomalies-another_" + jobId + ) + ); + + sb.append(" Index [") + .append(".ml-anomalies-") + .append(index) + .append("-000001") + .append("]: ") + .append(aliases1List) + .append("\n"); + sb.append(" Index [") + .append(".ml-anomalies-") + .append(index) + .append("-000002") + .append("]: ") + .append(aliases2List) + .append("\n"); + + logger.warn(sb.toString().trim()); + } + + // Now trigger another rollover event + blockingCall(maintenanceService::triggerRollResultsIndicesIfNecessaryTask); + + // Check indices and aliases, there should be a new index ".ml-anomalies--000003", + // with read/write aliases for both jobs pointing to it. There should be read aliases for + // both jobs pointing to ".ml-anomalies--000002" and a read alias for the initial job + // pointing to ".ml-anomalies--000001" and no other aliases referencing any of the 3 indices. + { + GetIndexResponse getIndexResponse = client().admin() + .indices() + .prepareGetIndex(TEST_REQUEST_TIMEOUT) + .setIndices(".ml-anomalies-" + index + "*") + .get(); + logger.warn("get_index_response: {}", getIndexResponse.toString()); + assertThat(getIndexResponse.getIndices().length, is(3)); + var aliases = getIndexResponse.getAliases(); + assertThat(aliases.size(), is(3)); + + StringBuilder sb = new StringBuilder("After 2nd Rollover. Aliases found:\n"); + List aliasMetadata1 = aliases.get(".ml-anomalies-" + index + "-000001"); + List aliasMetadata2 = aliases.get(".ml-anomalies-" + index + "-000002"); + List aliasMetadata3 = aliases.get(".ml-anomalies-" + index + "-000003"); + + assertThat(aliasMetadata1.size(), is(1)); + assertThat(aliasMetadata2.size(), is(2)); + assertThat(aliasMetadata3.size(), is(4)); + + List aliases1List = new ArrayList<>(aliasMetadata1.stream().map(AliasMetadata::alias).toList()); + List aliases2List = new ArrayList<>(aliasMetadata2.stream().map(AliasMetadata::alias).toList()); + List aliases3List = new ArrayList<>(aliasMetadata3.stream().map(AliasMetadata::alias).toList()); + + assertThat(aliases1List, containsInAnyOrder(".ml-anomalies-" + jobId)); + assertThat(aliases2List, containsInAnyOrder(".ml-anomalies-" + jobId, ".ml-anomalies-another_" + jobId)); + assertThat( + aliases3List, + containsInAnyOrder( + ".ml-anomalies-.write-" + jobId, + ".ml-anomalies-" + jobId, + ".ml-anomalies-.write-another_" + jobId, + ".ml-anomalies-another_" + jobId + ) + ); + + sb.append(" Index [") + .append(".ml-anomalies-") + .append(index) + .append("-000001") + .append("]: ") + .append(aliases1List) + .append("\n"); + sb.append(" Index [") + .append(".ml-anomalies-") + .append(index) + .append("-000002") + .append("]: ") + .append(aliases2List) + .append("\n"); + sb.append(" Index [") + .append(".ml-anomalies-") + .append(index) + .append("-000003") + .append("]: ") + .append(aliases3List) + .append("\n"); + + logger.warn(sb.toString().trim()); + } + } + } + + private void blockingCall(Consumer> function) throws InterruptedException { + AtomicReference exceptionHolder = new AtomicReference<>(); + CountDownLatch latch = new CountDownLatch(1); + ActionListener listener = ActionListener.wrap(r -> { latch.countDown(); }, e -> { + exceptionHolder.set(e); + latch.countDown(); + }); + function.accept(listener); + latch.await(); + if (exceptionHolder.get() != null) { + fail(exceptionHolder.get().getMessage()); + } + } + + private PutJobAction.Response putJob(Job.Builder job) { + PutJobAction.Request request = new PutJobAction.Request(job); + return client().execute(PutJobAction.INSTANCE, request).actionGet(); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java index 65d7fda17e17a..6f509138cb77a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java @@ -177,6 +177,6 @@ private void rollover(String alias, @Nullable String newIndexName, ActionListene } private void createAliasForRollover(String indexName, String aliasName, ActionListener listener) { - MlIndexAndAlias.createAliasForRollover(logger, client, indexName, aliasName, listener); + MlIndexAndAlias.createAliasForRollover(client, indexName, aliasName, listener); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index 25a9109539f45..c1e07effe139c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -94,6 +94,8 @@ public class MlDailyMaintenanceService implements Releasable { private final boolean isDataFrameAnalyticsEnabled; private final boolean isNlpEnabled; + private RolloverConditions rolloverConditions; + private volatile Scheduler.Cancellable cancellable; private volatile float deleteExpiredDataRequestsPerSecond; private volatile ByteSizeValue rolloverMaxSize; @@ -121,6 +123,8 @@ public class MlDailyMaintenanceService implements Releasable { this.isAnomalyDetectionEnabled = isAnomalyDetectionEnabled; this.isDataFrameAnalyticsEnabled = isDataFrameAnalyticsEnabled; this.isNlpEnabled = isNlpEnabled; + + this.rolloverConditions = RolloverConditions.newBuilder().addMaxIndexSizeCondition(rolloverMaxSize).build(); } public MlDailyMaintenanceService( @@ -360,20 +364,23 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio originSettingClient, new RolloverRequestBuilder(originSettingClient).setRolloverTarget(rolloverAlias) .setNewIndexName(newIndexName) - .setConditions(RolloverConditions.newBuilder().addMaxIndexSizeCondition(rolloverMaxSize).build()) + .setConditions(rolloverConditions) .request(), rolloverListener ); }, rolloverListener::onFailure); // 1. Create necessary aliases - MlIndexAndAlias.createAliasForRollover(logger, originSettingClient, index, rolloverAlias, getIndicesAliasesListener); + MlIndexAndAlias.createAliasForRollover(originSettingClient, index, rolloverAlias, getIndicesAliasesListener); } - // TODO make public for testing? - private void triggerRollResultsIndicesIfNecessaryTask(ActionListener finalListener) { + // public for testing + public void setRolloverConditions(RolloverConditions rolloverConditions) { + this.rolloverConditions = Objects.requireNonNull(rolloverConditions); + } - List failures = new ArrayList<>(); + // public for testing + public void triggerRollResultsIndicesIfNecessaryTask(ActionListener finalListener) { ClusterState clusterState = clusterService.state(); // list all indices starting .ml-anomalies- @@ -386,6 +393,11 @@ private void triggerRollResultsIndicesIfNecessaryTask(ActionListener Date: Thu, 16 Oct 2025 14:44:29 +1300 Subject: [PATCH 25/42] 2nd draft of integration test --- ...enanceServiceRolloverResultsIndicesIT.java | 378 ++++++++---------- 1 file changed, 172 insertions(+), 206 deletions(-) diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java index 465262c6eb00c..adcf20370e221 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java @@ -20,12 +20,13 @@ import org.elasticsearch.xpack.core.ml.action.PutJobAction; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.ml.MlAssignmentNotifier; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.ml.MlDailyMaintenanceService; import org.junit.Before; -import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -62,10 +63,6 @@ public void createComponents() throws Exception { true, true ); - - // replace the default set of conditions with an empty set so we can roll the index unconditionally - // It's not the conditions or even the rollover itself we are testing but the state of the indices and aliases afterwards. - maintenanceService.setRolloverConditions(RolloverConditions.newBuilder().build()); } private void initClusterAndJob() { @@ -76,6 +73,9 @@ private void initClusterAndJob() { public void testTriggerRollResultsIndicesIfNecessaryTask_givenNoIndices() throws Exception { // The null case, nothing to do. + // replace the default set of conditions with an empty set so we can roll the index unconditionally + // It's not the conditions or even the rollover itself we are testing but the state of the indices and aliases afterwards. + maintenanceService.setRolloverConditions(RolloverConditions.newBuilder().build()); { GetIndexResponse getIndexResponse = client().admin() .indices() @@ -103,7 +103,23 @@ public void testTriggerRollResultsIndicesIfNecessaryTask_givenNoIndices() throws } } + public void testTriggerRollResultsIndicesIfNecessaryTask_givenUnmetConditions() throws Exception { + // Create jobs that will use the default results indices - ".ml-anomalies-shared-*" + Job.Builder[] jobs_with_default_index = { createJob("job_using_default_index"), createJob("another_job_using_default_index") }; + + // Create jobs that will use custom results indices - ".ml-anomalies-custom-fred-*" + Job.Builder[] jobs_with_custom_index = { + createJob("job_using_custom_index").setResultsIndexName("fred"), + createJob("another_job_using_custom_index").setResultsIndexName("fred") }; + + runTestScenarioWithUnmetConditions(jobs_with_default_index, "shared"); + runTestScenarioWithUnmetConditions(jobs_with_custom_index, "custom-fred"); + } + public void testTriggerRollResultsIndicesIfNecessaryTask() throws Exception { + // replace the default set of conditions with an empty set so we can roll the index unconditionally + // It's not the conditions or even the rollover itself we are testing but the state of the indices and aliases afterwards. + maintenanceService.setRolloverConditions(RolloverConditions.newBuilder().build()); // Create jobs that will use the default results indices - ".ml-anomalies-shared-*" Job.Builder[] jobs_with_default_index = { createJob("job_using_default_index"), createJob("another_job_using_default_index") }; @@ -113,207 +129,156 @@ public void testTriggerRollResultsIndicesIfNecessaryTask() throws Exception { createJob("job_using_custom_index").setResultsIndexName("fred"), createJob("another_job_using_custom_index").setResultsIndexName("fred") }; - Job.Builder[][] job_lists = { jobs_with_default_index, jobs_with_custom_index }; - - for (Job.Builder[] job_list : job_lists) { - putJob(job_list[0]); - String jobId = job_list[0].getId(); - String index = (jobId.contains("job_using_custom_index")) ? "custom-fred" : "shared"; - { - GetIndexResponse getIndexResponse = client().admin() - .indices() - .prepareGetIndex(TEST_REQUEST_TIMEOUT) - .setIndices(".ml-anomalies-" + index + "*") - .get(); - logger.warn("get_index_response: {}", getIndexResponse.toString()); - assertThat(getIndexResponse.getIndices().length, is(1)); - var aliases = getIndexResponse.getAliases(); - assertThat(aliases.size(), is(1)); - - StringBuilder sb = new StringBuilder("Before Rollover. Aliases found:\n"); - - List aliasMetadata = aliases.get(".ml-anomalies-" + index + "-000001"); - - assertThat(aliasMetadata.size(), is(2)); - - List aliasesList = new ArrayList<>(aliasMetadata.stream().map(AliasMetadata::alias).toList()); - - assertThat(aliasesList, containsInAnyOrder(".ml-anomalies-.write-" + jobId, ".ml-anomalies-" + jobId)); - - sb.append(" Index [").append(".ml-anomalies-shared-000001").append("]: ").append(aliasesList).append("\n"); - - logger.warn(sb.toString().trim()); - } - - blockingCall(maintenanceService::triggerRollResultsIndicesIfNecessaryTask); - - // Check indices and aliases after rollover, there should be a read alias for ".ml-anomalies--000001" - // and read/write aliases for ".ml-anomalies--000002". There should be no other aliases pointing to these indices. - { - GetIndexResponse getIndexResponse = client().admin() - .indices() - .prepareGetIndex(TEST_REQUEST_TIMEOUT) - .setIndices(".ml-anomalies-" + index + "*") - .get(); - logger.warn("get_index_response: {}", getIndexResponse.toString()); - assertThat(getIndexResponse.getIndices().length, is(2)); - var aliases = getIndexResponse.getAliases(); - assertThat(aliases.size(), is(2)); - - StringBuilder sb = new StringBuilder("After Rollover. Aliases found:\n"); - List aliasMetadata1 = aliases.get(".ml-anomalies-" + index + "-000001"); - List aliasMetadata2 = aliases.get(".ml-anomalies-" + index + "-000002"); - - assertThat(aliasMetadata1.size(), is(1)); - assertThat(aliasMetadata2.size(), is(2)); - - List aliases1List = new ArrayList<>(aliasMetadata1.stream().map(AliasMetadata::alias).toList()); - List aliases2List = new ArrayList<>(aliasMetadata2.stream().map(AliasMetadata::alias).toList()); - - assertThat(aliases1List, containsInAnyOrder(".ml-anomalies-" + jobId)); - assertThat(aliases2List, containsInAnyOrder(".ml-anomalies-.write-" + jobId, ".ml-anomalies-" + jobId)); - - sb.append(" Index [") - .append(".ml-anomalies-") - .append(index) - .append("-000001") - .append("]: ") - .append(aliases1List) - .append("\n"); - sb.append(" Index [") - .append(".ml-anomalies-") - .append(index) - .append("-000002") - .append("]: ") - .append(aliases2List) - .append("\n"); - - logger.warn(sb.toString().trim()); - } - - // Now open another job. - putJob(job_list[1]); - - // Check indices and aliases, there should be new read/write aliases for the new job - // pointing to ".ml-anomalies--000002". - { - GetIndexResponse getIndexResponse = client().admin() - .indices() - .prepareGetIndex(TEST_REQUEST_TIMEOUT) - .setIndices(".ml-anomalies-" + index + "*") - .get(); - logger.warn("get_index_response: {}", getIndexResponse.toString()); - assertThat(getIndexResponse.getIndices().length, is(2)); - var aliases = getIndexResponse.getAliases(); - assertThat(aliases.size(), is(2)); - - StringBuilder sb = new StringBuilder("After 2nd Job creation. Aliases found:\n"); - List aliasMetadata1 = aliases.get(".ml-anomalies-" + index + "-000001"); - List aliasMetadata2 = aliases.get(".ml-anomalies-" + index + "-000002"); - - assertThat(aliasMetadata1.size(), is(1)); - assertThat(aliasMetadata2.size(), is(4)); - - List aliases1List = new ArrayList<>(aliasMetadata1.stream().map(AliasMetadata::alias).toList()); - List aliases2List = new ArrayList<>(aliasMetadata2.stream().map(AliasMetadata::alias).toList()); - - assertThat(aliases1List, containsInAnyOrder(".ml-anomalies-" + jobId)); - assertThat( - aliases2List, - containsInAnyOrder( - ".ml-anomalies-.write-" + jobId, - ".ml-anomalies-" + jobId, - ".ml-anomalies-.write-another_" + jobId, - ".ml-anomalies-another_" + jobId - ) - ); - - sb.append(" Index [") - .append(".ml-anomalies-") - .append(index) - .append("-000001") - .append("]: ") - .append(aliases1List) - .append("\n"); - sb.append(" Index [") - .append(".ml-anomalies-") - .append(index) - .append("-000002") - .append("]: ") - .append(aliases2List) - .append("\n"); - - logger.warn(sb.toString().trim()); - } - - // Now trigger another rollover event - blockingCall(maintenanceService::triggerRollResultsIndicesIfNecessaryTask); - - // Check indices and aliases, there should be a new index ".ml-anomalies--000003", - // with read/write aliases for both jobs pointing to it. There should be read aliases for - // both jobs pointing to ".ml-anomalies--000002" and a read alias for the initial job - // pointing to ".ml-anomalies--000001" and no other aliases referencing any of the 3 indices. - { - GetIndexResponse getIndexResponse = client().admin() - .indices() - .prepareGetIndex(TEST_REQUEST_TIMEOUT) - .setIndices(".ml-anomalies-" + index + "*") - .get(); - logger.warn("get_index_response: {}", getIndexResponse.toString()); - assertThat(getIndexResponse.getIndices().length, is(3)); - var aliases = getIndexResponse.getAliases(); - assertThat(aliases.size(), is(3)); - - StringBuilder sb = new StringBuilder("After 2nd Rollover. Aliases found:\n"); - List aliasMetadata1 = aliases.get(".ml-anomalies-" + index + "-000001"); - List aliasMetadata2 = aliases.get(".ml-anomalies-" + index + "-000002"); - List aliasMetadata3 = aliases.get(".ml-anomalies-" + index + "-000003"); - - assertThat(aliasMetadata1.size(), is(1)); - assertThat(aliasMetadata2.size(), is(2)); - assertThat(aliasMetadata3.size(), is(4)); - - List aliases1List = new ArrayList<>(aliasMetadata1.stream().map(AliasMetadata::alias).toList()); - List aliases2List = new ArrayList<>(aliasMetadata2.stream().map(AliasMetadata::alias).toList()); - List aliases3List = new ArrayList<>(aliasMetadata3.stream().map(AliasMetadata::alias).toList()); - - assertThat(aliases1List, containsInAnyOrder(".ml-anomalies-" + jobId)); - assertThat(aliases2List, containsInAnyOrder(".ml-anomalies-" + jobId, ".ml-anomalies-another_" + jobId)); - assertThat( - aliases3List, - containsInAnyOrder( - ".ml-anomalies-.write-" + jobId, - ".ml-anomalies-" + jobId, - ".ml-anomalies-.write-another_" + jobId, - ".ml-anomalies-another_" + jobId - ) - ); - - sb.append(" Index [") - .append(".ml-anomalies-") - .append(index) - .append("-000001") - .append("]: ") - .append(aliases1List) - .append("\n"); - sb.append(" Index [") - .append(".ml-anomalies-") - .append(index) - .append("-000002") - .append("]: ") - .append(aliases2List) - .append("\n"); - sb.append(" Index [") - .append(".ml-anomalies-") - .append(index) - .append("-000003") - .append("]: ") - .append(aliases3List) - .append("\n"); - - logger.warn(sb.toString().trim()); - } - } + runTestScenario(jobs_with_default_index, "shared"); + runTestScenario(jobs_with_custom_index, "custom-fred"); + } + + private void runTestScenarioWithUnmetConditions(Job.Builder[] jobs, String indexNamePart) throws Exception { + String firstJobId = jobs[0].getId(); + String secondJobId = jobs[1].getId(); + String indexWildcard = AnomalyDetectorsIndex.jobResultsIndexPrefix() + indexNamePart + "*"; + String firstIndexName = AnomalyDetectorsIndex.jobResultsIndexPrefix() + indexNamePart + "-000001"; + + // 1. Create the first job, which creates the first index and aliases + putJob(jobs[0]); + assertIndicesAndAliases( + "Before first rollover attempt", + indexWildcard, + Map.of(firstIndexName, List.of(writeAlias(firstJobId), readAlias(firstJobId))) + ); + + // 2. Trigger the first rollover attempt + blockingCall(maintenanceService::triggerRollResultsIndicesIfNecessaryTask); + assertIndicesAndAliases( + "After first rollover attempt", + indexWildcard, + Map.of(firstIndexName, List.of(writeAlias(firstJobId), readAlias(firstJobId))) + ); + + // 3. Create the second job, which adds its aliases to the current write index + putJob(jobs[1]); + assertIndicesAndAliases( + "After second job creation", + indexWildcard, + Map.of( + firstIndexName, + List.of( + writeAlias(firstJobId), + readAlias(firstJobId), + writeAlias(secondJobId), + readAlias(secondJobId) + ) + ) + ); + + // 4. Trigger the second rollover attempt + blockingCall(maintenanceService::triggerRollResultsIndicesIfNecessaryTask); + assertIndicesAndAliases( + "After second job creation", + indexWildcard, + Map.of( + firstIndexName, + List.of( + writeAlias(firstJobId), + readAlias(firstJobId), + writeAlias(secondJobId), + readAlias(secondJobId) + ) + ) + ); + } + + + private void runTestScenario(Job.Builder[] jobs, String indexNamePart) throws Exception { + String firstJobId = jobs[0].getId(); + String secondJobId = jobs[1].getId(); + String indexWildcard = AnomalyDetectorsIndex.jobResultsIndexPrefix() + indexNamePart + "*"; + String firstIndexName = AnomalyDetectorsIndex.jobResultsIndexPrefix() + indexNamePart + "-000001"; + String secondIndexName = AnomalyDetectorsIndex.jobResultsIndexPrefix() + indexNamePart + "-000002"; + String thirdIndexName = AnomalyDetectorsIndex.jobResultsIndexPrefix() + indexNamePart + "-000003"; + + // 1. Create the first job, which creates the first index and aliases + putJob(jobs[0]); + assertIndicesAndAliases( + "Before first rollover", + indexWildcard, + Map.of(firstIndexName, List.of(writeAlias(firstJobId), readAlias(firstJobId))) + ); + + // 2. Trigger the first rollover + blockingCall(maintenanceService::triggerRollResultsIndicesIfNecessaryTask); + assertIndicesAndAliases( + "After first rollover", + indexWildcard, + Map.of( + firstIndexName, + List.of(readAlias(firstJobId)), + secondIndexName, + List.of(writeAlias(firstJobId), readAlias(firstJobId)) + ) + ); + + // 3. Create the second job, which adds its aliases to the current write index + putJob(jobs[1]); + assertIndicesAndAliases( + "After second job creation", + indexWildcard, + Map.of( + firstIndexName, + List.of(readAlias(firstJobId)), + secondIndexName, + List.of(writeAlias(firstJobId), readAlias(firstJobId), writeAlias(secondJobId), readAlias(secondJobId)) + ) + ); + + // 4. Trigger the second rollover + blockingCall(maintenanceService::triggerRollResultsIndicesIfNecessaryTask); + assertIndicesAndAliases( + "After second rollover", + indexWildcard, + Map.of( + firstIndexName, + List.of(readAlias(firstJobId)), + secondIndexName, + List.of(readAlias(firstJobId), readAlias(secondJobId)), + thirdIndexName, + List.of(writeAlias(firstJobId), readAlias(firstJobId), writeAlias(secondJobId), readAlias(secondJobId)) + ) + ); + } + + private void assertIndicesAndAliases(String context, String indexWildcard, Map> expectedAliases) { + GetIndexResponse getIndexResponse = client().admin() + .indices() + .prepareGetIndex(TEST_REQUEST_TIMEOUT) + .setIndices(indexWildcard) + .get(); + + var aliases = getIndexResponse.getAliases(); + assertThat("Context: " + context, aliases.size(), is(expectedAliases.size())); + + StringBuilder sb = new StringBuilder(context).append(". Aliases found:\n"); + + expectedAliases.forEach((indexName, expectedAliasList) -> { + assertTrue("Expected index [" + indexName + "] was not found. Context: " + context, aliases.containsKey(indexName)); + List actualAliasMetadata = aliases.get(indexName); + List actualAliasList = actualAliasMetadata.stream().map(AliasMetadata::alias).toList(); + assertThat( + "Alias mismatch for index [" + indexName + "]. Context: " + context, + actualAliasList, + containsInAnyOrder(expectedAliasList.toArray(String[]::new)) + ); + sb.append(" Index [").append(indexName).append("]: ").append(actualAliasList).append("\n"); + }); + logger.warn(sb.toString().trim()); + } + + private String readAlias(String jobId) { + return AnomalyDetectorsIndex.jobResultsAliasedName(jobId); + } + + private String writeAlias(String jobId) { + return AnomalyDetectorsIndex.resultsWriteAlias(jobId); } private void blockingCall(Consumer> function) throws InterruptedException { @@ -335,3 +300,4 @@ private PutJobAction.Response putJob(Job.Builder job) { return client().execute(PutJobAction.INSTANCE, request).actionGet(); } } + From 644c57f7a2213f8aa46ba0758366e5d35fa96a64 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 16 Oct 2025 14:46:34 +1300 Subject: [PATCH 26/42] spotless fixes --- .../java/org/elasticsearch/xpack/core/ml/job/config/Job.java | 1 - .../org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java | 1 - 2 files changed, 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index fc9cfdd2fb946..6e2ee608f6576 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.core.ml.job.config; -import org.apache.lucene.util.SetOnce; import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.ClusterState; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index c1e07effe139c..c623eccca0a2c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -54,7 +54,6 @@ import java.time.Clock; import java.time.ZonedDateTime; -import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Random; From 5465b24b6d5498cd675ace65615ce1199c871428 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 16 Oct 2025 01:56:28 +0000 Subject: [PATCH 27/42] [CI] Auto commit changes from spotless --- .../xpack/ml/integration/MlJobIT.java | 15 ++++---- ...enanceServiceRolloverResultsIndicesIT.java | 37 +++---------------- 2 files changed, 13 insertions(+), 39 deletions(-) diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java index 1b1dc44eea72e..ce159748e4416 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java @@ -92,8 +92,8 @@ public void testPutJob_GivenFarequoteConfig() throws Exception { assertThat( aliasesResponseAsString, containsString( - "\".ml-anomalies-shared-000001\":{\"aliases\":" + - "{\".ml-anomalies-.write-given-farequote-config-job\":{\"is_hidden\":true},\".ml-anomalies-given-farequote-config-job\"" + "\".ml-anomalies-shared-000001\":{\"aliases\":" + + "{\".ml-anomalies-.write-given-farequote-config-job\":{\"is_hidden\":true},\".ml-anomalies-given-farequote-config-job\"" ) ); } @@ -362,7 +362,7 @@ public void testCreateJobsWithIndexNameOption() throws Exception { ); } - // The same as testCreateJobsWithIndexNameOption but we don't supply the "-000001" suffix to the index name supplied in the job config + // The same as testCreateJobsWithIndexNameOption but we don't supply the "-000001" suffix to the index name supplied in the job config // We test that the final index name does indeed have the suffix. public void testCreateJobsWithIndexNameNo6DigitSuffixOption() throws Exception { String jobTemplate = """ @@ -387,7 +387,7 @@ public void testCreateJobsWithIndexNameNo6DigitSuffixOption() throws Exception { try { String aliasesResponse = getAliases(); assertThat(aliasesResponse, containsString(Strings.format(""" - "%s":{"aliases":{""", AnomalyDetectorsIndex.jobResultsAliasedName("custom-" + indexName+"-000001")))); + "%s":{"aliases":{""", AnomalyDetectorsIndex.jobResultsAliasedName("custom-" + indexName + "-000001")))); assertThat( aliasesResponse, containsString( @@ -420,7 +420,7 @@ public void testCreateJobsWithIndexNameNo6DigitSuffixOption() throws Exception { }); String responseAsString = getMlResultsIndices(); - assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName+"-000001")); + assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName + "-000001")); assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1)))); assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2)))); @@ -485,13 +485,13 @@ public void testCreateJobsWithIndexNameNo6DigitSuffixOption() throws Exception { assertThat(responseAsString, containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))); // job2 still exists responseAsString = getMlResultsIndices(); - assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName+"-000001")); + assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName + "-000001")); refreshAllIndices(); responseAsString = EntityUtils.toString( client().performRequest( - new Request("GET", AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName+"-000001" + "/_count") + new Request("GET", AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName + "-000001" + "/_count") ).getEntity() ); assertThat(responseAsString, containsString("\"count\":2")); @@ -509,7 +509,6 @@ public void testCreateJobsWithIndexNameNo6DigitSuffixOption() throws Exception { ); } - public void testCreateJobInSharedIndexUpdatesMapping() throws Exception { String jobTemplate = """ { diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java index adcf20370e221..c1ba06be80070 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java @@ -9,8 +9,8 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.rollover.RolloverConditions; -import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; @@ -19,10 +19,10 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ml.action.PutJobAction; import org.elasticsearch.xpack.core.ml.job.config.Job; -import org.elasticsearch.xpack.ml.MlAssignmentNotifier; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.ml.MlAssignmentNotifier; import org.elasticsearch.xpack.ml.MlDailyMaintenanceService; - +import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; import org.junit.Before; import java.util.List; @@ -35,8 +35,6 @@ import static org.hamcrest.Matchers.is; import static org.mockito.Mockito.mock; -import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; - @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1, numClientNodes = 0, supportsDedicatedMasters = false) public class MlDailyMaintenanceServiceRolloverResultsIndicesIT extends BaseMlIntegTestCase { @@ -160,15 +158,7 @@ private void runTestScenarioWithUnmetConditions(Job.Builder[] jobs, String index assertIndicesAndAliases( "After second job creation", indexWildcard, - Map.of( - firstIndexName, - List.of( - writeAlias(firstJobId), - readAlias(firstJobId), - writeAlias(secondJobId), - readAlias(secondJobId) - ) - ) + Map.of(firstIndexName, List.of(writeAlias(firstJobId), readAlias(firstJobId), writeAlias(secondJobId), readAlias(secondJobId))) ); // 4. Trigger the second rollover attempt @@ -176,19 +166,10 @@ private void runTestScenarioWithUnmetConditions(Job.Builder[] jobs, String index assertIndicesAndAliases( "After second job creation", indexWildcard, - Map.of( - firstIndexName, - List.of( - writeAlias(firstJobId), - readAlias(firstJobId), - writeAlias(secondJobId), - readAlias(secondJobId) - ) - ) + Map.of(firstIndexName, List.of(writeAlias(firstJobId), readAlias(firstJobId), writeAlias(secondJobId), readAlias(secondJobId))) ); } - private void runTestScenario(Job.Builder[] jobs, String indexNamePart) throws Exception { String firstJobId = jobs[0].getId(); String secondJobId = jobs[1].getId(); @@ -210,12 +191,7 @@ private void runTestScenario(Job.Builder[] jobs, String indexNamePart) throws Ex assertIndicesAndAliases( "After first rollover", indexWildcard, - Map.of( - firstIndexName, - List.of(readAlias(firstJobId)), - secondIndexName, - List.of(writeAlias(firstJobId), readAlias(firstJobId)) - ) + Map.of(firstIndexName, List.of(readAlias(firstJobId)), secondIndexName, List.of(writeAlias(firstJobId), readAlias(firstJobId))) ); // 3. Create the second job, which adds its aliases to the current write index @@ -300,4 +276,3 @@ private PutJobAction.Response putJob(Job.Builder job) { return client().execute(PutJobAction.INSTANCE, request).actionGet(); } } - From 42e63c9f1d572ca902ee0846c61075b9794c44ab Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 16 Oct 2025 16:48:32 +1300 Subject: [PATCH 28/42] spotless fixes --- .../xpack/ml/integration/MlJobIT.java | 16 ++++---- ...enanceServiceRolloverResultsIndicesIT.java | 37 +++---------------- 2 files changed, 14 insertions(+), 39 deletions(-) diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java index 1b1dc44eea72e..56ef442ee6ac8 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java @@ -92,8 +92,9 @@ public void testPutJob_GivenFarequoteConfig() throws Exception { assertThat( aliasesResponseAsString, containsString( - "\".ml-anomalies-shared-000001\":{\"aliases\":" + - "{\".ml-anomalies-.write-given-farequote-config-job\":{\"is_hidden\":true},\".ml-anomalies-given-farequote-config-job\"" + "\".ml-anomalies-shared-000001\":{\"aliases\":" + + "{\".ml-anomalies-.write-given-farequote-config-job\":" + + "{\"is_hidden\":true},\".ml-anomalies-given-farequote-config-job\"" ) ); } @@ -362,7 +363,7 @@ public void testCreateJobsWithIndexNameOption() throws Exception { ); } - // The same as testCreateJobsWithIndexNameOption but we don't supply the "-000001" suffix to the index name supplied in the job config + // The same as testCreateJobsWithIndexNameOption but we don't supply the "-000001" suffix to the index name supplied in the job config // We test that the final index name does indeed have the suffix. public void testCreateJobsWithIndexNameNo6DigitSuffixOption() throws Exception { String jobTemplate = """ @@ -387,7 +388,7 @@ public void testCreateJobsWithIndexNameNo6DigitSuffixOption() throws Exception { try { String aliasesResponse = getAliases(); assertThat(aliasesResponse, containsString(Strings.format(""" - "%s":{"aliases":{""", AnomalyDetectorsIndex.jobResultsAliasedName("custom-" + indexName+"-000001")))); + "%s":{"aliases":{""", AnomalyDetectorsIndex.jobResultsAliasedName("custom-" + indexName + "-000001")))); assertThat( aliasesResponse, containsString( @@ -420,7 +421,7 @@ public void testCreateJobsWithIndexNameNo6DigitSuffixOption() throws Exception { }); String responseAsString = getMlResultsIndices(); - assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName+"-000001")); + assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName + "-000001")); assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1)))); assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2)))); @@ -485,13 +486,13 @@ public void testCreateJobsWithIndexNameNo6DigitSuffixOption() throws Exception { assertThat(responseAsString, containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))); // job2 still exists responseAsString = getMlResultsIndices(); - assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName+"-000001")); + assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName + "-000001")); refreshAllIndices(); responseAsString = EntityUtils.toString( client().performRequest( - new Request("GET", AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName+"-000001" + "/_count") + new Request("GET", AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName + "-000001" + "/_count") ).getEntity() ); assertThat(responseAsString, containsString("\"count\":2")); @@ -509,7 +510,6 @@ public void testCreateJobsWithIndexNameNo6DigitSuffixOption() throws Exception { ); } - public void testCreateJobInSharedIndexUpdatesMapping() throws Exception { String jobTemplate = """ { diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java index adcf20370e221..c1ba06be80070 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java @@ -9,8 +9,8 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.rollover.RolloverConditions; -import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; @@ -19,10 +19,10 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ml.action.PutJobAction; import org.elasticsearch.xpack.core.ml.job.config.Job; -import org.elasticsearch.xpack.ml.MlAssignmentNotifier; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.ml.MlAssignmentNotifier; import org.elasticsearch.xpack.ml.MlDailyMaintenanceService; - +import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; import org.junit.Before; import java.util.List; @@ -35,8 +35,6 @@ import static org.hamcrest.Matchers.is; import static org.mockito.Mockito.mock; -import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; - @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1, numClientNodes = 0, supportsDedicatedMasters = false) public class MlDailyMaintenanceServiceRolloverResultsIndicesIT extends BaseMlIntegTestCase { @@ -160,15 +158,7 @@ private void runTestScenarioWithUnmetConditions(Job.Builder[] jobs, String index assertIndicesAndAliases( "After second job creation", indexWildcard, - Map.of( - firstIndexName, - List.of( - writeAlias(firstJobId), - readAlias(firstJobId), - writeAlias(secondJobId), - readAlias(secondJobId) - ) - ) + Map.of(firstIndexName, List.of(writeAlias(firstJobId), readAlias(firstJobId), writeAlias(secondJobId), readAlias(secondJobId))) ); // 4. Trigger the second rollover attempt @@ -176,19 +166,10 @@ private void runTestScenarioWithUnmetConditions(Job.Builder[] jobs, String index assertIndicesAndAliases( "After second job creation", indexWildcard, - Map.of( - firstIndexName, - List.of( - writeAlias(firstJobId), - readAlias(firstJobId), - writeAlias(secondJobId), - readAlias(secondJobId) - ) - ) + Map.of(firstIndexName, List.of(writeAlias(firstJobId), readAlias(firstJobId), writeAlias(secondJobId), readAlias(secondJobId))) ); } - private void runTestScenario(Job.Builder[] jobs, String indexNamePart) throws Exception { String firstJobId = jobs[0].getId(); String secondJobId = jobs[1].getId(); @@ -210,12 +191,7 @@ private void runTestScenario(Job.Builder[] jobs, String indexNamePart) throws Ex assertIndicesAndAliases( "After first rollover", indexWildcard, - Map.of( - firstIndexName, - List.of(readAlias(firstJobId)), - secondIndexName, - List.of(writeAlias(firstJobId), readAlias(firstJobId)) - ) + Map.of(firstIndexName, List.of(readAlias(firstJobId)), secondIndexName, List.of(writeAlias(firstJobId), readAlias(firstJobId))) ); // 3. Create the second job, which adds its aliases to the current write index @@ -300,4 +276,3 @@ private PutJobAction.Response putJob(Job.Builder job) { return client().execute(PutJobAction.INSTANCE, request).actionGet(); } } - From 16cb2d1fb68293d3566d160f3b58b446a4a15bc2 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 17 Oct 2025 13:12:53 +1300 Subject: [PATCH 29/42] Fix broken test case --- .../xpack/ml/MlDailyMaintenanceService.java | 89 +++++++++++++------ 1 file changed, 62 insertions(+), 27 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index c623eccca0a2c..d16a4346fde82 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.ml; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; @@ -17,6 +19,7 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverConditions; import org.elasticsearch.action.admin.indices.rollover.RolloverRequestBuilder; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; @@ -31,6 +34,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Predicates; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.TimeValue; @@ -38,6 +42,7 @@ import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.logging.LogManager; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.threadpool.ThreadPool; @@ -54,6 +59,8 @@ import java.time.Clock; import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.Random; @@ -291,10 +298,22 @@ void removeRolloverAlias( IndicesAliasesRequestBuilder aliasRequestBuilder, ActionListener listener ) { + logger.trace("removeRolloverAlias: index: {}, alias: {}", index, alias); aliasRequestBuilder.removeAlias(index, alias); MlIndexAndAlias.updateAliases(aliasRequestBuilder, listener); } + private void rollover(Client client, String rolloverAlias, @Nullable String newIndexName, ActionListener listener) { + MlIndexAndAlias.rollover( + client, + new RolloverRequestBuilder(client).setRolloverTarget(rolloverAlias) + .setNewIndexName(newIndexName) + .setConditions(rolloverConditions) + .request(), + listener + ); + } + private void rollAndUpdateAliases(ClusterState clusterState, String index, ActionListener listener) { // Create an alias specifically for rolling over. // The ml-anomalies index has aliases for each job, any @@ -318,7 +337,7 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio ); // 4 Clean up any dangling aliases - ActionListener aliasListener = ActionListener.wrap(r -> { listener.onResponse(r); }, e -> { + ActionListener aliasListener = ActionListener.wrap(listener::onResponse, e -> { if (e instanceof IndexNotFoundException) { // Removal of the rollover alias may have failed in the case of rollover not occurring, e.g. when the rollover conditions // were not satisfied. @@ -334,8 +353,20 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT ); + ActionListener traceListener = ActionListener.wrap(r -> { + if (r) { + logger.trace("Successfully removed rollover alias {} from index {}", rolloverAlias, index); + } else { + logger.trace("Failed to remove rollover alias {} from index {}", rolloverAlias, index); + } + listener.onResponse(r); + }, x -> { + logger.trace("Failed to remove rollover alias {} from index {}, caught exception {}", rolloverAlias, index, x); + listener.onFailure(x); + }); + // Execute the cleanup, no need to propagate the original failure. - removeRolloverAlias(indexName, rolloverAlias, localAliasRequestBuilder, listener); + removeRolloverAlias(indexName, rolloverAlias, localAliasRequestBuilder, traceListener); } else { listener.onFailure(e); } @@ -359,15 +390,12 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio // 2 rollover the index alias to the new index name ActionListener getIndicesAliasesListener = ActionListener.wrap(getIndicesAliasesResponse -> { - MlIndexAndAlias.rollover( - originSettingClient, - new RolloverRequestBuilder(originSettingClient).setRolloverTarget(rolloverAlias) - .setNewIndexName(newIndexName) - .setConditions(rolloverConditions) - .request(), - rolloverListener - ); - }, rolloverListener::onFailure); + rollover(originSettingClient, rolloverAlias, newIndexName, rolloverListener); + }, e -> { + // If rollover fails, we must still clean up the temporary alias from the original index. + removeRolloverAlias(index, rolloverAlias, aliasRequestBuilder, aliasListener); + rolloverListener.onFailure(e); + }); // 1. Create necessary aliases MlIndexAndAlias.createAliasForRollover(originSettingClient, index, rolloverAlias, getIndicesAliasesListener); @@ -380,6 +408,7 @@ public void setRolloverConditions(RolloverConditions rolloverConditions) { // public for testing public void triggerRollResultsIndicesIfNecessaryTask(ActionListener finalListener) { + List failures = new ArrayList<>(); ClusterState clusterState = clusterService.state(); // list all indices starting .ml-anomalies- @@ -392,6 +421,8 @@ public void triggerRollResultsIndicesIfNecessaryTask(ActionListener rollAndUpdateAliasesResponseListener = finalListener.delegateFailureAndWrap( - (l, rolledAndUpdatedAliasesResponse) -> { - if (rolledAndUpdatedAliasesResponse) { - logger.info( - "Successfully completed [ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask for index [{}]", - index - ); - } else { - logger.warn( - "Unsuccessful run of [ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask for index [{}]", - index - ); - } - l.onResponse(AcknowledgedResponse.TRUE); // TODO return false if operation failed for any index? + PlainActionFuture updated = new PlainActionFuture<>(); + rollAndUpdateAliases(clusterState, index, updated); + try { + updated.actionGet(); + } catch (Exception ex) { + var message = "failed rolling over ml anomalies index [" + index + "]"; + logger.warn(message, ex); + if (ex instanceof ElasticsearchException elasticsearchException) { + failures.add(new ElasticsearchStatusException(message, elasticsearchException.status(), elasticsearchException)); + } else { + failures.add(new ElasticsearchStatusException(message, RestStatus.REQUEST_TIMEOUT, ex)); } - ); + } + } - rollAndUpdateAliases(clusterState, index, rollAndUpdateAliasesResponseListener); + if (failures.isEmpty()) { + logger.info("ml anomalies indices [{}] rolled over and aliases updated", String.join(",", indices)); + finalListener.onResponse(AcknowledgedResponse.TRUE); + return; } + + logger.warn("failed to roll over ml anomalies results indices: [{}]", failures); + finalListener.onResponse(AcknowledgedResponse.FALSE); } private void triggerDeleteExpiredDataTask(ActionListener finalListener) { From b5513a2b2c333ef16773dca919aa034eeaf2c768 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 17 Oct 2025 15:49:53 +1300 Subject: [PATCH 30/42] Tidy up --- .../xpack/ml/MlDailyMaintenanceService.java | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index d16a4346fde82..300eab06360af 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -353,20 +353,8 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT ); - ActionListener traceListener = ActionListener.wrap(r -> { - if (r) { - logger.trace("Successfully removed rollover alias {} from index {}", rolloverAlias, index); - } else { - logger.trace("Failed to remove rollover alias {} from index {}", rolloverAlias, index); - } - listener.onResponse(r); - }, x -> { - logger.trace("Failed to remove rollover alias {} from index {}, caught exception {}", rolloverAlias, index, x); - listener.onFailure(x); - }); - // Execute the cleanup, no need to propagate the original failure. - removeRolloverAlias(indexName, rolloverAlias, localAliasRequestBuilder, traceListener); + removeRolloverAlias(indexName, rolloverAlias, localAliasRequestBuilder, listener); } else { listener.onFailure(e); } @@ -391,11 +379,7 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio // 2 rollover the index alias to the new index name ActionListener getIndicesAliasesListener = ActionListener.wrap(getIndicesAliasesResponse -> { rollover(originSettingClient, rolloverAlias, newIndexName, rolloverListener); - }, e -> { - // If rollover fails, we must still clean up the temporary alias from the original index. - removeRolloverAlias(index, rolloverAlias, aliasRequestBuilder, aliasListener); - rolloverListener.onFailure(e); - }); + }, rolloverListener::onFailure); // 1. Create necessary aliases MlIndexAndAlias.createAliasForRollover(originSettingClient, index, rolloverAlias, getIndicesAliasesListener); From 29faf9edbb7c3ba1d7c9afc0c2a23f8578bd82a4 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 17 Oct 2025 16:45:32 +1300 Subject: [PATCH 31/42] Add more test scenarios --- ...enanceServiceRolloverResultsIndicesIT.java | 118 +++++++++++++++++- 1 file changed, 115 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java index c1ba06be80070..34f5ff21cd478 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java @@ -12,11 +12,11 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ml.action.DeleteJobAction; import org.elasticsearch.xpack.core.ml.action.PutJobAction; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; @@ -25,6 +25,7 @@ import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase; import org.junit.Before; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; @@ -42,7 +43,6 @@ public class MlDailyMaintenanceServiceRolloverResultsIndicesIT extends BaseMlInt @Before public void createComponents() throws Exception { - Settings settings = nodeSettings(0, Settings.EMPTY); ThreadPool threadPool = mockThreadPool(); ClusterService clusterService = internalCluster().clusterService(internalCluster().getMasterName()); @@ -114,6 +114,110 @@ public void testTriggerRollResultsIndicesIfNecessaryTask_givenUnmetConditions() runTestScenarioWithUnmetConditions(jobs_with_custom_index, "custom-fred"); } + public void testTriggerRollResultsIndicesIfNecessaryTask_withMixedIndexTypes() throws Exception { + maintenanceService.setRolloverConditions(RolloverConditions.newBuilder().build()); + + // 1. Create a job using the default shared index + Job.Builder sharedJob = createJob("shared-job"); + putJob(sharedJob); + assertIndicesAndAliases( + "After shared job creation", + AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared*", + Map.of( + AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared-000001", + List.of(writeAlias(sharedJob.getId()), readAlias(sharedJob.getId())) + ) + ); + + // 2. Create a job using a custom index + Job.Builder customJob = createJob("custom-job").setResultsIndexName("my-custom"); + putJob(customJob); + assertIndicesAndAliases( + "After custom job creation", + AnomalyDetectorsIndex.jobResultsIndexPrefix() + "custom-my-custom*", + Map.of( + AnomalyDetectorsIndex.jobResultsIndexPrefix() + "custom-my-custom-000001", + List.of(writeAlias(customJob.getId()), readAlias(customJob.getId())) + ) + ); + + // 3. Trigger a single maintenance run + blockingCall(maintenanceService::triggerRollResultsIndicesIfNecessaryTask); + + // 4. Verify BOTH indices were rolled over correctly + assertIndicesAndAliases( + "After rollover (shared)", + AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared*", + Map.of( + AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared-000001", + List.of(readAlias(sharedJob.getId())), + AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared-000002", + List.of(writeAlias(sharedJob.getId()), readAlias(sharedJob.getId())) + ) + ); + + assertIndicesAndAliases( + "After rollover (custom)", + AnomalyDetectorsIndex.jobResultsIndexPrefix() + "custom-my-custom*", + Map.of( + AnomalyDetectorsIndex.jobResultsIndexPrefix() + "custom-my-custom-000001", + List.of(readAlias(customJob.getId())), + AnomalyDetectorsIndex.jobResultsIndexPrefix() + "custom-my-custom-000002", + List.of(writeAlias(customJob.getId()), readAlias(customJob.getId())) + ) + ); + } + + public void testTriggerRollResultsIndicesIfNecessaryTask_givenNoJobAliases() throws Exception { + maintenanceService.setRolloverConditions(RolloverConditions.newBuilder().build()); + + String jobId = "job-to-be-deleted"; + Job.Builder job = createJob(jobId); + putJob(job); + + String indexName = AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared-000001"; + String rolledIndexName = AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared-000002"; + assertIndicesAndAliases( + "Before job deletion", + AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared*", + Map.of(indexName, List.of(writeAlias(jobId), readAlias(jobId))) + ); + + // Delete the job, which also removes its aliases + deleteJob(jobId); + + // Verify the index still exists but has no aliases + GetIndexResponse getIndexResponse = client().admin() + .indices() + .prepareGetIndex(TEST_REQUEST_TIMEOUT) + .setIndices(indexName) + .get(); + assertThat(getIndexResponse.getIndices().length, is(1)); + assertThat(getIndexResponse.getAliases().size(), is(0)); + + // Trigger maintenance + blockingCall(maintenanceService::triggerRollResultsIndicesIfNecessaryTask); + + // Verify that the index was rolled over, even though it had no ML aliases + GetIndexResponse finalIndexResponse = client().admin() + .indices() + .prepareGetIndex(TEST_REQUEST_TIMEOUT) + .setIndices(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared*") + .get(); + assertThat(finalIndexResponse.getIndices().length, is(2)); + List expectedIndexList = List.of(indexName, rolledIndexName); + List actualIndexList = Arrays.asList(finalIndexResponse.getIndices()); + + assertThat( + "Mismatch for indices", + actualIndexList, + containsInAnyOrder(expectedIndexList.toArray(String[]::new)) + ); + + assertThat(finalIndexResponse.getIndices()[0], is(indexName)); + assertThat(finalIndexResponse.getIndices()[1], is(rolledIndexName)); + } + public void testTriggerRollResultsIndicesIfNecessaryTask() throws Exception { // replace the default set of conditions with an empty set so we can roll the index unconditionally // It's not the conditions or even the rollover itself we are testing but the state of the indices and aliases afterwards. @@ -260,7 +364,7 @@ private String writeAlias(String jobId) { private void blockingCall(Consumer> function) throws InterruptedException { AtomicReference exceptionHolder = new AtomicReference<>(); CountDownLatch latch = new CountDownLatch(1); - ActionListener listener = ActionListener.wrap(r -> { latch.countDown(); }, e -> { + ActionListener listener = ActionListener.wrap(r -> latch.countDown(), e -> { exceptionHolder.set(e); latch.countDown(); }); @@ -275,4 +379,12 @@ private PutJobAction.Response putJob(Job.Builder job) { PutJobAction.Request request = new PutJobAction.Request(job); return client().execute(PutJobAction.INSTANCE, request).actionGet(); } + + private void deleteJob(String jobId) { + try { + client().execute(DeleteJobAction.INSTANCE, new DeleteJobAction.Request(jobId)).actionGet(); + } catch (Exception e) { + // noop + } + } } From 77ab9dfe6d3e30c553cfb3e253a6cd83a1560d56 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 17 Oct 2025 03:51:08 +0000 Subject: [PATCH 32/42] [CI] Auto commit changes from spotless --- ...lyMaintenanceServiceRolloverResultsIndicesIT.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java index 34f5ff21cd478..5b7ab32afd00a 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java @@ -187,11 +187,7 @@ public void testTriggerRollResultsIndicesIfNecessaryTask_givenNoJobAliases() thr deleteJob(jobId); // Verify the index still exists but has no aliases - GetIndexResponse getIndexResponse = client().admin() - .indices() - .prepareGetIndex(TEST_REQUEST_TIMEOUT) - .setIndices(indexName) - .get(); + GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex(TEST_REQUEST_TIMEOUT).setIndices(indexName).get(); assertThat(getIndexResponse.getIndices().length, is(1)); assertThat(getIndexResponse.getAliases().size(), is(0)); @@ -208,11 +204,7 @@ public void testTriggerRollResultsIndicesIfNecessaryTask_givenNoJobAliases() thr List expectedIndexList = List.of(indexName, rolledIndexName); List actualIndexList = Arrays.asList(finalIndexResponse.getIndices()); - assertThat( - "Mismatch for indices", - actualIndexList, - containsInAnyOrder(expectedIndexList.toArray(String[]::new)) - ); + assertThat("Mismatch for indices", actualIndexList, containsInAnyOrder(expectedIndexList.toArray(String[]::new))); assertThat(finalIndexResponse.getIndices()[0], is(indexName)); assertThat(finalIndexResponse.getIndices()[1], is(rolledIndexName)); From cd9b467b85d5fd10d0f6057e3e2e4414ffd43cb0 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Mon, 20 Oct 2025 16:08:35 +1300 Subject: [PATCH 33/42] Refactor integration tests --- .../core/ml/utils/MlIndexAndAliasTests.java | 159 +++++++++++++++++- ...enanceServiceRolloverResultsIndicesIT.java | 12 +- .../xpack/ml/MlAnomaliesIndexUpdateTests.java | 129 -------------- 3 files changed, 154 insertions(+), 146 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java index d46c999850bea..f98c465ecac6a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.client.internal.AdminClient; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.ClusterAdminClient; +import org.elasticsearch.client.internal.ElasticsearchClient; import org.elasticsearch.client.internal.IndicesAdminClient; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -59,6 +60,7 @@ import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.doAnswer; @@ -77,10 +79,7 @@ public class MlIndexAndAliasTests extends ESTestCase { private static final int TEST_TEMPLATE_VERSION = 12345678; - private ThreadPool threadPool; private IndicesAdminClient indicesAdminClient; - private ClusterAdminClient clusterAdminClient; - private AdminClient adminClient; private Client client; private ActionListener listener; @@ -90,7 +89,7 @@ public class MlIndexAndAliasTests extends ESTestCase { @SuppressWarnings("unchecked") @Before public void setUpMocks() { - threadPool = mock(ThreadPool.class); + final ThreadPool threadPool = mock(ThreadPool.class); when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); indicesAdminClient = mock(IndicesAdminClient.class); @@ -104,7 +103,7 @@ public void setUpMocks() { doAnswer(withResponse(IndicesAliasesResponse.ACKNOWLEDGED_NO_ERRORS)).when(indicesAdminClient).aliases(any(), any()); doAnswer(withResponse(IndicesAliasesResponse.ACKNOWLEDGED_NO_ERRORS)).when(indicesAdminClient).putTemplate(any(), any()); - clusterAdminClient = mock(ClusterAdminClient.class); + final ClusterAdminClient clusterAdminClient = mock(ClusterAdminClient.class); doAnswer(invocationOnMock -> { ActionListener actionListener = (ActionListener) invocationOnMock .getArguments()[1]; @@ -112,7 +111,7 @@ public void setUpMocks() { return null; }).when(clusterAdminClient).health(any(ClusterHealthRequest.class), any(ActionListener.class)); - adminClient = mock(AdminClient.class); + final AdminClient adminClient = mock(AdminClient.class); when(adminClient.indices()).thenReturn(indicesAdminClient); when(adminClient.cluster()).thenReturn(clusterAdminClient); @@ -328,7 +327,7 @@ private void assertMlStateWriteAliasAddedToMostRecentMlStateIndex(List e } public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButInitialStateIndexExists() { - assertMlStateWriteAliasAddedToMostRecentMlStateIndex(Arrays.asList(FIRST_CONCRETE_INDEX), FIRST_CONCRETE_INDEX); + assertMlStateWriteAliasAddedToMostRecentMlStateIndex(List.of(FIRST_CONCRETE_INDEX), FIRST_CONCRETE_INDEX); } public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButSubsequentStateIndicesExist() { @@ -405,6 +404,152 @@ public void testHas6DigitSuffix() { assertFalse(MlIndexAndAlias.has6DigitSuffix("index000001")); } + public void testLatestIndexMatchingBaseName_isLatest() { + Metadata.Builder metadata = Metadata.builder(); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo", IndexVersion.current(), List.of("job1"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bar", IndexVersion.current(), List.of("job2"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bax", IndexVersion.current(), List.of("job3"))); + ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); + csBuilder.metadata(metadata); + + var latest = MlIndexAndAlias.latestIndexMatchingBaseName( + ".ml-anomalies-custom-foo", + TestIndexNameExpressionResolver.newInstance(), + csBuilder.build() + ); + assertEquals(".ml-anomalies-custom-foo", latest); + } + + public void testLatestIndexMatchingBaseName_hasLater() { + Metadata.Builder metadata = Metadata.builder(); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo", IndexVersion.current(), List.of("job1"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bar", IndexVersion.current(), List.of("job2"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-000001", IndexVersion.current(), List.of("job3"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-000002", IndexVersion.current(), List.of("job4"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-baz-000001", IndexVersion.current(), List.of("job5"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-baz-000002", IndexVersion.current(), List.of("job6"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-baz-000003", IndexVersion.current(), List.of("job7"))); + ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); + csBuilder.metadata(metadata); + var state = csBuilder.build(); + + assertTrue(MlIndexAndAlias.has6DigitSuffix(".ml-anomalies-custom-foo-000002")); + + var latest = MlIndexAndAlias.latestIndexMatchingBaseName( + ".ml-anomalies-custom-foo", + TestIndexNameExpressionResolver.newInstance(), + state + ); + assertEquals(".ml-anomalies-custom-foo-000002", latest); + + latest = MlIndexAndAlias.latestIndexMatchingBaseName( + ".ml-anomalies-custom-baz-000001", + TestIndexNameExpressionResolver.newInstance(), + state + ); + assertEquals(".ml-anomalies-custom-baz-000003", latest); + } + + public void testLatestIndexMatchingBaseName_CollidingIndexNames() { + Metadata.Builder metadata = Metadata.builder(); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo", IndexVersion.current(), List.of("job1"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bar", IndexVersion.current(), List.of("job2"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foodifferent000001", IndexVersion.current(), List.of("job3"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-notthisone-000001", IndexVersion.current(), List.of("job4"))); + metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-notthisone-000002", IndexVersion.current(), List.of("job5"))); + ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); + csBuilder.metadata(metadata); + var state = csBuilder.build(); + + var latest = MlIndexAndAlias.latestIndexMatchingBaseName( + ".ml-anomalies-custom-foo", + TestIndexNameExpressionResolver.newInstance(), + state + ); + assertEquals(".ml-anomalies-custom-foo", latest); + + latest = MlIndexAndAlias.latestIndexMatchingBaseName( + ".ml-anomalies-custom-foo-notthisone-000001", + TestIndexNameExpressionResolver.newInstance(), + state + ); + assertEquals(".ml-anomalies-custom-foo-notthisone-000002", latest); + } + + public void testBuildIndexAliasesRequest() { + var anomaliesIndex = ".ml-anomalies-sharedindex"; + var jobs = List.of("job1", "job2"); + IndexMetadata.Builder indexMetadata = createSharedResultsIndex(anomaliesIndex, IndexVersion.current(), jobs); + Metadata.Builder metadata = Metadata.builder(); + metadata.put(indexMetadata); + ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); + csBuilder.metadata(metadata); + + IndicesAliasesRequestBuilder aliasRequestBuilder = new IndicesAliasesRequestBuilder( + mock(ElasticsearchClient.class), + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT + ); + + var newIndex = anomaliesIndex + "-000001"; + var request = MlIndexAndAlias.addIndexAliasesRequests(aliasRequestBuilder, anomaliesIndex, newIndex, csBuilder.build()); + var actions = request.request().getAliasActions(); + assertThat(actions, hasSize(6)); + + // The order in which the alias actions are created + // is not preserved so look for the item in the list + for (var job : jobs) { + var expected = new AliasActionMatcher( + AnomalyDetectorsIndex.resultsWriteAlias(job), + newIndex, + IndicesAliasesRequest.AliasActions.Type.ADD + ); + assertThat(actions.stream().filter(expected::matches).count(), equalTo(1L)); + + expected = new AliasActionMatcher( + AnomalyDetectorsIndex.resultsWriteAlias(job), + anomaliesIndex, + IndicesAliasesRequest.AliasActions.Type.REMOVE + ); + assertThat(actions.stream().filter(expected::matches).count(), equalTo(1L)); + + expected = new AliasActionMatcher( + AnomalyDetectorsIndex.jobResultsAliasedName(job), + newIndex, + IndicesAliasesRequest.AliasActions.Type.ADD + ); + assertThat(actions.stream().filter(expected::matches).count(), equalTo(1L)); + } + } + + private record AliasActionMatcher(String aliasName, String index, IndicesAliasesRequest.AliasActions.Type actionType) { + boolean matches(IndicesAliasesRequest.AliasActions aliasAction) { + return aliasAction.actionType() == actionType + && aliasAction.aliases()[0].equals(aliasName) + && aliasAction.indices()[0].equals(index); + } + } + + private IndexMetadata.Builder createSharedResultsIndex(String indexName, IndexVersion indexVersion, List jobs) { + IndexMetadata.Builder indexMetadata = IndexMetadata.builder(indexName); + indexMetadata.settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, indexVersion) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_INDEX_UUID, "_uuid") + ); + + for (var jobId : jobs) { + indexMetadata.putAlias(AliasMetadata.builder(AnomalyDetectorsIndex.jobResultsAliasedName(jobId)).isHidden(true).build()); + indexMetadata.putAlias( + AliasMetadata.builder(AnomalyDetectorsIndex.resultsWriteAlias(jobId)).writeIndex(true).isHidden(true).build() + ); + } + + return indexMetadata; + } + private void createIndexAndAliasIfNecessary(ClusterState clusterState) { MlIndexAndAlias.createIndexAndAliasIfNecessary( client, diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java index 34f5ff21cd478..5b7ab32afd00a 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java @@ -187,11 +187,7 @@ public void testTriggerRollResultsIndicesIfNecessaryTask_givenNoJobAliases() thr deleteJob(jobId); // Verify the index still exists but has no aliases - GetIndexResponse getIndexResponse = client().admin() - .indices() - .prepareGetIndex(TEST_REQUEST_TIMEOUT) - .setIndices(indexName) - .get(); + GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex(TEST_REQUEST_TIMEOUT).setIndices(indexName).get(); assertThat(getIndexResponse.getIndices().length, is(1)); assertThat(getIndexResponse.getAliases().size(), is(0)); @@ -208,11 +204,7 @@ public void testTriggerRollResultsIndicesIfNecessaryTask_givenNoJobAliases() thr List expectedIndexList = List.of(indexName, rolledIndexName); List actualIndexList = Arrays.asList(finalIndexResponse.getIndices()); - assertThat( - "Mismatch for indices", - actualIndexList, - containsInAnyOrder(expectedIndexList.toArray(String[]::new)) - ); + assertThat("Mismatch for indices", actualIndexList, containsInAnyOrder(expectedIndexList.toArray(String[]::new))); assertThat(finalIndexResponse.getIndices()[0], is(indexName)); assertThat(finalIndexResponse.getIndices()[1], is(rolledIndexName)); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java index d77e60bfdd4af..cc32b5a872e69 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java @@ -9,15 +9,12 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.alias.TransportIndicesAliasesAction; import org.elasticsearch.action.admin.indices.rollover.RolloverAction; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.client.internal.Client; -import org.elasticsearch.client.internal.ElasticsearchClient; -import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetadata; @@ -32,14 +29,11 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; -import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasSize; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.doAnswer; @@ -81,57 +75,6 @@ public void testIsAbleToRun_IndicesHaveNoRouting() { assertFalse(updater.isAbleToRun(csBuilder.build())); } - public void testBuildIndexAliasesRequest() { - var anomaliesIndex = ".ml-anomalies-sharedindex"; - var jobs = List.of("job1", "job2"); - IndexMetadata.Builder indexMetadata = createSharedResultsIndex(anomaliesIndex, IndexVersion.current(), jobs); - Metadata.Builder metadata = Metadata.builder(); - metadata.put(indexMetadata); - ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); - csBuilder.metadata(metadata); - - var updater = new MlAnomaliesIndexUpdate( - TestIndexNameExpressionResolver.newInstance(), - new OriginSettingClient(mock(Client.class), "doesn't matter") - ); - - IndicesAliasesRequestBuilder aliasRequestBuilder = new IndicesAliasesRequestBuilder( - mock(ElasticsearchClient.class), - TEST_REQUEST_TIMEOUT, - TEST_REQUEST_TIMEOUT - ); - - var newIndex = anomaliesIndex + "-000001"; - var request = MlIndexAndAlias.addIndexAliasesRequests(aliasRequestBuilder, anomaliesIndex, newIndex, csBuilder.build()); - var actions = request.request().getAliasActions(); - assertThat(actions, hasSize(6)); - - // The order in which the alias actions are created - // is not preserved so look for the item in the list - for (var job : jobs) { - var expected = new AliasActionMatcher( - AnomalyDetectorsIndex.resultsWriteAlias(job), - newIndex, - IndicesAliasesRequest.AliasActions.Type.ADD - ); - assertThat(actions.stream().filter(expected::matches).count(), equalTo(1L)); - - expected = new AliasActionMatcher( - AnomalyDetectorsIndex.resultsWriteAlias(job), - anomaliesIndex, - IndicesAliasesRequest.AliasActions.Type.REMOVE - ); - assertThat(actions.stream().filter(expected::matches).count(), equalTo(1L)); - - expected = new AliasActionMatcher( - AnomalyDetectorsIndex.jobResultsAliasedName(job), - newIndex, - IndicesAliasesRequest.AliasActions.Type.ADD - ); - assertThat(actions.stream().filter(expected::matches).count(), equalTo(1L)); - } - } - public void testRunUpdate_UpToDateIndices() { String indexName = ".ml-anomalies-sharedindex-000001"; var jobs = List.of("job1", "job2"); @@ -174,78 +117,6 @@ public void testRunUpdate_LegacyIndex() { verifyNoMoreInteractions(client); } - public void testLatestIndexMatchingBaseName_isLatest() { - Metadata.Builder metadata = Metadata.builder(); - metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo", IndexVersion.current(), List.of("job1"))); - metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bar", IndexVersion.current(), List.of("job2"))); - metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bax", IndexVersion.current(), List.of("job3"))); - ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); - csBuilder.metadata(metadata); - - var latest = MlIndexAndAlias.latestIndexMatchingBaseName( - ".ml-anomalies-custom-foo", - TestIndexNameExpressionResolver.newInstance(), - csBuilder.build() - ); - assertEquals(".ml-anomalies-custom-foo", latest); - } - - public void testLatestIndexMatchingBaseName_hasLater() { - Metadata.Builder metadata = Metadata.builder(); - metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo", IndexVersion.current(), List.of("job1"))); - metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bar", IndexVersion.current(), List.of("job2"))); - metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-000001", IndexVersion.current(), List.of("job3"))); - metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-000002", IndexVersion.current(), List.of("job4"))); - metadata.put(createSharedResultsIndex(".ml-anomalies-custom-baz-000001", IndexVersion.current(), List.of("job5"))); - metadata.put(createSharedResultsIndex(".ml-anomalies-custom-baz-000002", IndexVersion.current(), List.of("job6"))); - metadata.put(createSharedResultsIndex(".ml-anomalies-custom-baz-000003", IndexVersion.current(), List.of("job7"))); - ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); - csBuilder.metadata(metadata); - var state = csBuilder.build(); - - assertTrue(MlIndexAndAlias.has6DigitSuffix(".ml-anomalies-custom-foo-000002")); - - var latest = MlIndexAndAlias.latestIndexMatchingBaseName( - ".ml-anomalies-custom-foo", - TestIndexNameExpressionResolver.newInstance(), - state - ); - assertEquals(".ml-anomalies-custom-foo-000002", latest); - - latest = MlIndexAndAlias.latestIndexMatchingBaseName( - ".ml-anomalies-custom-baz-000001", - TestIndexNameExpressionResolver.newInstance(), - state - ); - assertEquals(".ml-anomalies-custom-baz-000003", latest); - } - - public void testLatestIndexMatchingBaseName_CollidingIndexNames() { - Metadata.Builder metadata = Metadata.builder(); - metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo", IndexVersion.current(), List.of("job1"))); - metadata.put(createSharedResultsIndex(".ml-anomalies-custom-bar", IndexVersion.current(), List.of("job2"))); - metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foodifferent000001", IndexVersion.current(), List.of("job3"))); - metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-notthisone-000001", IndexVersion.current(), List.of("job4"))); - metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo-notthisone-000002", IndexVersion.current(), List.of("job5"))); - ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); - csBuilder.metadata(metadata); - var state = csBuilder.build(); - - var latest = MlIndexAndAlias.latestIndexMatchingBaseName( - ".ml-anomalies-custom-foo", - TestIndexNameExpressionResolver.newInstance(), - state - ); - assertEquals(".ml-anomalies-custom-foo", latest); - - latest = MlIndexAndAlias.latestIndexMatchingBaseName( - ".ml-anomalies-custom-foo-notthisone-000001", - TestIndexNameExpressionResolver.newInstance(), - state - ); - assertEquals(".ml-anomalies-custom-foo-notthisone-000002", latest); - } - private record AliasActionMatcher(String aliasName, String index, IndicesAliasesRequest.AliasActions.Type actionType) { boolean matches(IndicesAliasesRequest.AliasActions aliasAction) { return aliasAction.actionType() == actionType From 326eedd1725d5b315390c9f2d5615bf4185b4f2d Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 23 Oct 2025 16:37:58 +1300 Subject: [PATCH 34/42] Attend to code review comments --- docs/changelog/136065.yaml | 2 +- .../machine-learning-settings.md | 4 +- .../xpack/core/ml/job/config/Job.java | 46 +----- .../AnomalyDetectorsIndexFields.java | 3 +- .../xpack/core/ml/utils/MlIndexAndAlias.java | 129 ++++++++++++++- .../xpack/core/ml/job/config/JobTests.java | 2 +- .../xpack/ml/integration/MlJobIT.java | 26 ++- .../integration/JobStorageDeletionTaskIT.java | 2 +- ...enanceServiceRolloverResultsIndicesIT.java | 77 ++++----- .../xpack/ml/MachineLearning.java | 8 +- .../xpack/ml/MlAnomaliesIndexUpdate.java | 24 +-- .../xpack/ml/MlDailyMaintenanceService.java | 148 ++++++++---------- .../xpack/ml/MlInitializationService.java | 2 +- .../xpack/ml/job/JobManager.java | 2 +- .../ml/job/persistence/JobDataDeleter.java | 3 +- .../job/persistence/JobResultsProvider.java | 10 +- 16 files changed, 273 insertions(+), 215 deletions(-) diff --git a/docs/changelog/136065.yaml b/docs/changelog/136065.yaml index 45f073600e61e..c455f2e2f2f5d 100644 --- a/docs/changelog/136065.yaml +++ b/docs/changelog/136065.yaml @@ -1,5 +1,5 @@ pr: 136065 -summary: Manage ad results indices +summary: Nightly maintenance for anomaly detection results indices to keep to manageable size. area: Machine Learning type: enhancement issues: [] diff --git a/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md b/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md index 5eff8c71b3baa..68ae51dd26715 100644 --- a/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md +++ b/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md @@ -86,8 +86,8 @@ $$$xpack.ml.max_open_jobs$$$ `xpack.ml.nightly_maintenance_requests_per_second` : ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The rate at which the nightly maintenance task deletes expired model snapshots and results. The setting is a proxy to the [`requests_per_second`](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-delete-by-query) parameter used in the delete by query requests and controls throttling. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Valid values must be greater than `0.0` or equal to `-1.0`, where `-1.0` means a default value is used. Defaults to `-1.0` -`xpack.ml.nightly_maintenance_rollover_max_size` -: ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The maximum size the anomaly detection results indices can reach before being rolled over by the nightly maintenance task. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Valid values must be greater than `0B` or equal to `-1B`. Defaults to `50GB`. +`xpack.ml.results_index_rollover_max_size` +: ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The maximum size the anomaly detection results indices can reach before being rolled over by the nightly maintenance task. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Valid values must be greater than or equal to `0B`. A value of `0B` means the indices will always be rolled over. Defaults to `50GB`. `xpack.ml.node_concurrent_job_allocations` : ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The maximum number of jobs that can concurrently be in the `opening` state on each node. Typically, jobs spend a small amount of time in this state before they move to `open` state. Jobs that must restore large models when they are opening spend more time in the `opening` state. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Defaults to `2`. diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index 6e2ee608f6576..f71dca638ea5a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -8,9 +8,7 @@ import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.SimpleDiffable; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -808,8 +806,6 @@ public static class Builder implements Writeable { private boolean allowLazyOpen; private Blocked blocked = Blocked.none(); private DatafeedConfig.Builder datafeedConfig; - private ClusterState clusterState; - private IndexNameExpressionResolver indexNameExpressionResolver; public Builder() {} @@ -884,14 +880,6 @@ public String getId() { return id; } - private void setClusterState(ClusterState state) { - this.clusterState = state; - } - - private void setIndexNameExpressionResolver(IndexNameExpressionResolver indexNameExpressionResolver) { - this.indexNameExpressionResolver = indexNameExpressionResolver; - } - public void setJobVersion(MlConfigVersion jobVersion) { this.jobVersion = jobVersion; } @@ -1318,16 +1306,6 @@ public void validateDetectorsAreUnique() { } } - public Job build( - @SuppressWarnings("HiddenField") Date createTime, - ClusterState state, - IndexNameExpressionResolver indexNameExpressionResolver - ) { - setClusterState(state); - setIndexNameExpressionResolver(indexNameExpressionResolver); - return build(createTime); - } - /** * Builds a job with the given {@code createTime} and the current version. * This should be used when a new job is created as opposed to {@link #build()}. @@ -1367,24 +1345,12 @@ public Job build() { if (Strings.isNullOrEmpty(resultsIndexName)) { resultsIndexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; - } else if ((resultsIndexName.startsWith(AnomalyDetectorsIndexFields.RESULTS_INDEX_SHARED) - && MlIndexAndAlias.has6DigitSuffix(resultsIndexName) - && resultsIndexName.length() == AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT.length()) == false) { - // User-defined names are prepended with "custom" and end with a 6 digit suffix - // Conditional guards against multiple prepending due to updates instead of first creation - resultsIndexName = resultsIndexName.startsWith("custom-") ? resultsIndexName : "custom-" + resultsIndexName; - } - - resultsIndexName = MlIndexAndAlias.has6DigitSuffix(resultsIndexName) ? resultsIndexName : resultsIndexName + "-000001"; - - if (indexNameExpressionResolver != null && clusterState != null) { - String tmpResultsIndexName = MlIndexAndAlias.latestIndexMatchingBaseName( - AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + resultsIndexName, - indexNameExpressionResolver, - clusterState - ); - - resultsIndexName = tmpResultsIndexName.substring(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX.length()); + } else if (MlIndexAndAlias.isAnomaliesSharedIndex( + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + resultsIndexName + ) == false) { + // User-defined names are prepended with "custom" + // Conditional guards against multiple prepending due to updates instead of first creation + resultsIndexName = resultsIndexName.startsWith("custom-") ? resultsIndexName : "custom-" + resultsIndexName; } if (datafeedConfig != null) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java index 3c4a65155af7f..2a0fff86ba494 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java @@ -14,8 +14,7 @@ public final class AnomalyDetectorsIndexFields { // ".write" rather than simply "write" to avoid the danger of clashing // with the read alias of a job whose name begins with "write-" public static final String RESULTS_INDEX_WRITE_PREFIX = RESULTS_INDEX_PREFIX + ".write-"; - public static final String RESULTS_INDEX_SHARED = "shared"; - public static final String RESULTS_INDEX_DEFAULT = "shared-000001"; + public static final String RESULTS_INDEX_DEFAULT = "shared"; private AnomalyDetectorsIndexFields() {} } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java index 58b3c51894a06..e534c04e64046 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java @@ -31,6 +31,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -75,6 +76,10 @@ public final class MlIndexAndAlias { private static final Logger logger = LogManager.getLogger(MlIndexAndAlias.class); private static final Predicate HAS_SIX_DIGIT_SUFFIX = Pattern.compile("\\d{6}").asMatchPredicate(); + private static final Predicate IS_ANOMALIES_SHARED_INDEX = Pattern.compile( + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT + "-\\d{6}" + ).asMatchPredicate(); + private static final String ROLLOVER_ALIAS_SUFFIX = ".rollover_alias"; static final Comparator INDEX_NAME_COMPARATOR = (index1, index2) -> { String[] index1Parts = index1.split("-"); @@ -229,6 +234,21 @@ public static void createIndexAndAliasIfNecessary( loggingListener.onResponse(false); } + /** + * Creates a system index based on the provided descriptor if it does not already exist. + *

+ * The check for existence is simple and will return the listener on the calling thread if successful. + * If the index needs to be created an async call will be made and this method will wait for the index to reach at least + * a yellow health status before notifying the listener, ensuring it is ready for use + * upon a successful response. A {@link ResourceAlreadyExistsException} during creation + * is handled gracefully and treated as a success. + * + * @param client The client to use for the create index request. + * @param clusterState The current cluster state, used for the initial existence check. + * @param descriptor The descriptor containing the index name, settings, and mappings. + * @param masterNodeTimeout The timeout for waiting on the master node. + * @param finalListener Async listener + */ public static void createSystemIndexIfNecessary( Client client, ClusterState clusterState, @@ -328,6 +348,16 @@ private static void createFirstConcreteIndex( ); } + /** + * Creates or moves a write alias from one index to another. + * + * @param client The client to use for the add alias request. + * @param alias The alias to update. + * @param currentIndex The index the alias is currently pointing to. + * @param newIndex The new index the alias should point to. + * @param masterNodeTimeout The timeout for waiting on the master node. + * @param listener Async listener + */ public static void updateWriteAlias( Client client, String alias, @@ -362,7 +392,7 @@ public static void updateWriteAlias( /** * Installs the index template specified by {@code templateConfig} if it is not in already * installed in {@code clusterState}. - * + *

* The check for presence is simple and will return the listener on * the calling thread if successful. If the template has to be installed * an async call will be made. @@ -432,17 +462,38 @@ public static void installIndexTemplateIfRequired( executeAsyncWithOrigin(client, ML_ORIGIN, TransportPutComposableIndexTemplateAction.TYPE, templateRequest, innerListener); } - public static boolean hasIndexTemplate(ClusterState state, String templateName, long version) { + private static boolean hasIndexTemplate(ClusterState state, String templateName, long version) { var template = state.getMetadata().getProject().templatesV2().get(templateName); return template != null && Long.valueOf(version).equals(template.version()); } + public static String ensureValidResultsIndexName(String indexName) { + // The results index name is either the original one provided or the original with a suffix appended. + return has6DigitSuffix(indexName) ? indexName : indexName + FIRST_INDEX_SIX_DIGIT_SUFFIX; + } + + /** + * Checks if an index name ends with a 6-digit suffix (e.g., "-000001"). + * + * @param indexName The name of the index to check. + * @return {@code true} if the index name has a 6-digit suffix, {@code false} otherwise. + */ public static boolean has6DigitSuffix(String indexName) { String[] indexParts = indexName.split("-"); String suffix = indexParts[indexParts.length - 1]; return HAS_SIX_DIGIT_SUFFIX.test(suffix); } + /** + * Checks if an index name matches the pattern for the default ML anomalies indices (e.g., ".ml-anomalies-shared-000001"). + * + * @param indexName The name of the index to check. + * @return {@code true} if the index is a shared anomalies index, {@code false} otherwise. + */ + public static boolean isAnomaliesSharedIndex(String indexName) { + return IS_ANOMALIES_SHARED_INDEX.test(indexName); + } + /** * Returns the latest index. Latest is the index with the highest * 6 digit suffix. @@ -503,6 +554,15 @@ public static String latestIndexMatchingBaseName( return MlIndexAndAlias.latestIndex(filtered); } + /** + * Executes a rollover request. It handles {@link ResourceAlreadyExistsException} gracefully by treating it as a success + * and returning the name of the existing index. + * + * @param client The client to use for the rollover request. + * @param rolloverRequest The rollover request to execute. + * @param listener A listener that will be notified with the name of the new (or pre-existing) index on success, + * or an exception on failure. + */ public static void rollover(Client client, RolloverRequest rolloverRequest, ActionListener listener) { client.admin() .indices() @@ -516,6 +576,35 @@ public static void rollover(Client client, RolloverRequest rolloverRequest, Acti })); } + public static Tuple createRolloverAliasAndNewIndexName(String index) { + // Create an alias specifically for rolling over. + // The ml-anomalies index has aliases for each job, any + // of which could be used but that means one alias is + // treated differently. + // Using a `.` in the alias name avoids any conflicts + // as AD job Ids cannot start with `.` + String rolloverAlias = index + ROLLOVER_ALIAS_SUFFIX; + + // If the index does not end in a digit then rollover does not know + // what to name the new index so it must be specified in the request. + // Otherwise leave null and rollover will calculate the new name + String newIndexName = MlIndexAndAlias.has6DigitSuffix(index) ? null : index + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; + + return new Tuple<>(rolloverAlias, newIndexName); + } + + public static IndicesAliasesRequestBuilder createIndicesAliasesRequestBuilder(Client client) { + return client.admin().indices().prepareAliases(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS); + } + + /** + * Creates a hidden alias for an index, typically used as a rollover target. + * + * @param client The client to use for the alias request. + * @param indexName The name of the index to which the alias will be added. + * @param aliasName The name of the alias to create. + * @param listener A listener that will be notified with the response. + */ public static void createAliasForRollover( Client client, String indexName, @@ -523,17 +612,31 @@ public static void createAliasForRollover( ActionListener listener ) { logger.info("creating rollover [{}] alias for [{}]", aliasName, indexName); - client.admin() - .indices() - .prepareAliases(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS) - .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(indexName).alias(aliasName).isHidden(true)) - .execute(listener); + createIndicesAliasesRequestBuilder(client).addAliasAction( + IndicesAliasesRequest.AliasActions.add().index(indexName).alias(aliasName).isHidden(true) + ).execute(listener); } + /** + * Executes a prepared {@link IndicesAliasesRequestBuilder} and notifies the listener of the result. + * + * @param request The prepared request builder containing alias actions. + * @param listener A listener that will be notified with {@code true} on success. + */ public static void updateAliases(IndicesAliasesRequestBuilder request, ActionListener listener) { request.execute(listener.delegateFailure((l, response) -> l.onResponse(Boolean.TRUE))); } + /** + * Adds alias actions to a request builder to move ML job aliases from an old index to a new one after a rollover. + * This includes moving the write alias and re-creating the filtered read aliases on the new index. + * + * @param aliasRequestBuilder The request builder to add actions to. + * @param oldIndex The index from which aliases are being moved. + * @param newIndex The new index to which aliases will be moved. + * @param clusterState The current cluster state, used to inspect existing aliases on the old index. + * @return The modified {@link IndicesAliasesRequestBuilder}. + */ public static IndicesAliasesRequestBuilder addIndexAliasesRequests( IndicesAliasesRequestBuilder aliasRequestBuilder, String oldIndex, @@ -570,10 +673,22 @@ public static IndicesAliasesRequestBuilder addIndexAliasesRequests( return aliasRequestBuilder; } + /** + * Determines if an alias name is an ML anomalies write alias. + * + * @param aliasName The alias name to check. + * @return {@code true} if the name matches the write alias pattern, {@code false} otherwise. + */ public static boolean isAnomaliesWriteAlias(String aliasName) { return aliasName.startsWith(AnomalyDetectorsIndexFields.RESULTS_INDEX_WRITE_PREFIX); } + /** + * Determines if an alias name is an ML anomalies read alias. + * + * @param aliasName The alias name to check. + * @return {@code true} if the name matches the read alias pattern, {@code false} otherwise. + */ public static boolean isAnomaliesReadAlias(String aliasName) { if (aliasName.startsWith(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX) == false) { return false; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java index 82f139d49ef8f..4f4c90aacbf35 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java @@ -500,7 +500,7 @@ public void testBuilder_setsIndexName() { Job.Builder builder = buildJobBuilder("foo"); builder.setResultsIndexName("carol"); Job job = builder.build(); - assertEquals(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-carol-000001", job.getInitialResultsIndexName()); + assertEquals(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-carol", job.getInitialResultsIndexName()); } public void testBuilder_withInvalidIndexNameThrows() { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java index 56ef442ee6ac8..e0f318dbefd1e 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats; +import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.ml.MachineLearning; import org.junit.After; @@ -82,7 +83,7 @@ public void testPutJob_GivenFarequoteConfig() throws Exception { Response response = createFarequoteJob("given-farequote-config-job"); String responseAsString = EntityUtils.toString(response.getEntity()); assertThat(responseAsString, containsString("\"job_id\":\"given-farequote-config-job\"")); - assertThat(responseAsString, containsString("\"results_index_name\":\"shared-000001\"")); + assertThat(responseAsString, containsString("\"results_index_name\":\"shared\"")); String mlIndicesResponseAsString = getMlResultsIndices(); assertThat(mlIndicesResponseAsString, containsString("green open .ml-anomalies-shared-000001")); @@ -537,7 +538,10 @@ public void testCreateJobInSharedIndexUpdatesMapping() throws Exception { // Check the index mapping contains the first by_field_name Request getResultsMappingRequest = new Request( "GET", - AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT + "/_mapping" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT + + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX + + "/_mapping" ); getResultsMappingRequest.addParameter("pretty", null); String resultsMappingAfterJob1 = EntityUtils.toString(client().performRequest(getResultsMappingRequest).getEntity()); @@ -660,7 +664,8 @@ public void testOpenJobFailsWhenPersistentTaskAssignmentDisabled() throws Except public void testDeleteJob() throws Exception { String jobId = "delete-job-job"; - String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; + String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT + + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; createFarequoteJob(jobId); // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652 @@ -726,7 +731,8 @@ public void testOutOfOrderData() throws Exception { public void testDeleteJob_TimingStatsDocumentIsDeleted() throws Exception { String jobId = "delete-job-with-timing-stats-document-job"; - String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; + String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT + + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; createFarequoteJob(jobId); assertThat( @@ -779,7 +785,8 @@ public void testDeleteJob_TimingStatsDocumentIsDeleted() throws Exception { public void testDeleteJobAsync() throws Exception { String jobId = "delete-job-async-job"; - String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; + String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT + + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; createFarequoteJob(jobId); // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652 @@ -836,7 +843,8 @@ private static String extractTaskId(Response response) throws IOException { public void testDeleteJobAfterMissingIndex() throws Exception { String jobId = "delete-job-after-missing-index-job"; String aliasName = AnomalyDetectorsIndex.jobResultsAliasedName(jobId); - String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; + String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT + + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; createFarequoteJob(jobId); // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652 @@ -870,7 +878,8 @@ public void testDeleteJobAfterMissingAliases() throws Exception { String jobId = "delete-job-after-missing-alias-job"; String readAliasName = AnomalyDetectorsIndex.jobResultsAliasedName(jobId); String writeAliasName = AnomalyDetectorsIndex.resultsWriteAlias(jobId); - String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; + String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT + + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; createFarequoteJob(jobId); // With security enabled cat aliases throws an index_not_found_exception @@ -900,7 +909,8 @@ public void testDeleteJobAfterMissingAliases() throws Exception { public void testMultiIndexDelete() throws Exception { String jobId = "multi-index-delete-job"; - String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; + String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT + + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; createFarequoteJob(jobId); // Make the job's results span an extra two indices, i.e. three in total. diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobStorageDeletionTaskIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobStorageDeletionTaskIT.java index 61acf64fa5efe..f02c5ecb84c0e 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobStorageDeletionTaskIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobStorageDeletionTaskIT.java @@ -119,7 +119,7 @@ public void testDeleteDedicatedJobWithDataInShared() throws Exception { ensureStableCluster(1); String jobIdDedicated = "delete-test-job-dedicated"; - Job.Builder job = createJob(jobIdDedicated, ByteSizeValue.ofMb(2)).setResultsIndexName(jobIdDedicated); + Job.Builder job = createJob(jobIdDedicated, ByteSizeValue.ofMb(2)).setResultsIndexName(jobIdDedicated + "-000001"); client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(job)).actionGet(); client().execute(OpenJobAction.INSTANCE, new OpenJobAction.Request(job.getId())).actionGet(); String dedicatedIndex = job.build().getInitialResultsIndexName(); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java index 5b7ab32afd00a..16de2e5a18962 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java @@ -7,11 +7,12 @@ package org.elasticsearch.xpack.ml.integration; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; -import org.elasticsearch.action.admin.indices.rollover.RolloverConditions; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESIntegTestCase; @@ -71,9 +72,9 @@ private void initClusterAndJob() { public void testTriggerRollResultsIndicesIfNecessaryTask_givenNoIndices() throws Exception { // The null case, nothing to do. - // replace the default set of conditions with an empty set so we can roll the index unconditionally + // set the rollover max size to 0B so we can roll the index unconditionally // It's not the conditions or even the rollover itself we are testing but the state of the indices and aliases afterwards. - maintenanceService.setRolloverConditions(RolloverConditions.newBuilder().build()); + maintenanceService.setRolloverMaxSize(ByteSizeValue.ZERO); { GetIndexResponse getIndexResponse = client().admin() .indices() @@ -115,7 +116,7 @@ public void testTriggerRollResultsIndicesIfNecessaryTask_givenUnmetConditions() } public void testTriggerRollResultsIndicesIfNecessaryTask_withMixedIndexTypes() throws Exception { - maintenanceService.setRolloverConditions(RolloverConditions.newBuilder().build()); + maintenanceService.setRolloverMaxSize(ByteSizeValue.ZERO); // 1. Create a job using the default shared index Job.Builder sharedJob = createJob("shared-job"); @@ -169,51 +170,30 @@ public void testTriggerRollResultsIndicesIfNecessaryTask_withMixedIndexTypes() t } public void testTriggerRollResultsIndicesIfNecessaryTask_givenNoJobAliases() throws Exception { - maintenanceService.setRolloverConditions(RolloverConditions.newBuilder().build()); - - String jobId = "job-to-be-deleted"; - Job.Builder job = createJob(jobId); - putJob(job); + maintenanceService.setRolloverMaxSize(ByteSizeValue.ZERO); String indexName = AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared-000001"; String rolledIndexName = AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared-000002"; - assertIndicesAndAliases( - "Before job deletion", - AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared*", - Map.of(indexName, List.of(writeAlias(jobId), readAlias(jobId))) - ); + String indexWildcard = AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared*"; - // Delete the job, which also removes its aliases - deleteJob(jobId); + // 1. Create an index that looks like an ML results index but has no aliases + CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName); + client().admin().indices().create(createIndexRequest).actionGet(); - // Verify the index still exists but has no aliases - GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex(TEST_REQUEST_TIMEOUT).setIndices(indexName).get(); - assertThat(getIndexResponse.getIndices().length, is(1)); - assertThat(getIndexResponse.getAliases().size(), is(0)); + // Expect the index to exist with no aliases + assertIndicesAndAliases("Before rollover attempt", indexWildcard, Map.of(indexName, List.of())); - // Trigger maintenance + // 2. Trigger maintenance blockingCall(maintenanceService::triggerRollResultsIndicesIfNecessaryTask); // Verify that the index was rolled over, even though it had no ML aliases - GetIndexResponse finalIndexResponse = client().admin() - .indices() - .prepareGetIndex(TEST_REQUEST_TIMEOUT) - .setIndices(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared*") - .get(); - assertThat(finalIndexResponse.getIndices().length, is(2)); - List expectedIndexList = List.of(indexName, rolledIndexName); - List actualIndexList = Arrays.asList(finalIndexResponse.getIndices()); - - assertThat("Mismatch for indices", actualIndexList, containsInAnyOrder(expectedIndexList.toArray(String[]::new))); - - assertThat(finalIndexResponse.getIndices()[0], is(indexName)); - assertThat(finalIndexResponse.getIndices()[1], is(rolledIndexName)); + assertIndicesAndAliases("After rollover attempt", indexWildcard, Map.of(indexName, List.of(), rolledIndexName, List.of())); } public void testTriggerRollResultsIndicesIfNecessaryTask() throws Exception { // replace the default set of conditions with an empty set so we can roll the index unconditionally // It's not the conditions or even the rollover itself we are testing but the state of the indices and aliases afterwards. - maintenanceService.setRolloverConditions(RolloverConditions.newBuilder().build()); + maintenanceService.setRolloverMaxSize(ByteSizeValue.ZERO); // Create jobs that will use the default results indices - ".ml-anomalies-shared-*" Job.Builder[] jobs_with_default_index = { createJob("job_using_default_index"), createJob("another_job_using_default_index") }; @@ -326,21 +306,28 @@ private void assertIndicesAndAliases(String context, String indexWildcard, Map { - assertTrue("Expected index [" + indexName + "] was not found. Context: " + context, aliases.containsKey(indexName)); - List actualAliasMetadata = aliases.get(indexName); - List actualAliasList = actualAliasMetadata.stream().map(AliasMetadata::alias).toList(); - assertThat( - "Alias mismatch for index [" + indexName + "]. Context: " + context, - actualAliasList, - containsInAnyOrder(expectedAliasList.toArray(String[]::new)) - ); - sb.append(" Index [").append(indexName).append("]: ").append(actualAliasList).append("\n"); + assertThat("Context: " + context, indices.size(), is(expectedAliases.size())); + if (expectedAliasList.isEmpty()) { + assertThat("Context: " + context, aliases.size(), is(0)); + } else { + List actualAliasMetadata = aliases.get(indexName); + List actualAliasList = actualAliasMetadata.stream().map(AliasMetadata::alias).toList(); + assertThat( + "Alias mismatch for index [" + indexName + "]. Context: " + context, + actualAliasList, + containsInAnyOrder(expectedAliasList.toArray(String[]::new)) + ); + sb.append(" Index [").append(indexName).append("]: ").append(actualAliasList).append("\n"); + } }); logger.warn(sb.toString().trim()); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 29be1a5938efc..4a0dd89ee9dbe 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -719,9 +719,11 @@ public void loadExtensions(ExtensionLoader loader) { Property.NodeScope ); - public static final Setting NIGHTLY_MAINTENANCE_ROLLOVER_MAX_SIZE = Setting.byteSizeSetting( - "xpack.ml.nightly_maintenance_rollover_max_size", + public static final Setting RESULTS_INDEX_ROLLOVER_MAX_SIZE = Setting.byteSizeSetting( + "xpack.ml.results_index_rollover_max_size", ByteSizeValue.of(50, ByteSizeUnit.GB), + ByteSizeValue.ofBytes(0L), + ByteSizeValue.ofBytes(Long.MAX_VALUE), Property.OperatorDynamic, Property.NodeScope ); @@ -849,7 +851,7 @@ public List> getSettings() { ModelLoadingService.INFERENCE_MODEL_CACHE_TTL, ResultsPersisterService.PERSIST_RESULTS_MAX_RETRIES, NIGHTLY_MAINTENANCE_REQUESTS_PER_SECOND, - NIGHTLY_MAINTENANCE_ROLLOVER_MAX_SIZE, + RESULTS_INDEX_ROLLOVER_MAX_SIZE, MachineLearningField.USE_AUTO_MACHINE_MEMORY_PERCENT, MAX_ML_NODE_SIZE, DELAYED_DATA_CHECK_FREQ, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java index f0c287760087b..202cc020471f6 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Tuple; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.rest.RestStatus; @@ -142,24 +143,11 @@ public void runUpdate(ClusterState latestState) { } private void rollAndUpdateAliases(ClusterState clusterState, String index, ActionListener listener) { - // Create an alias specifically for rolling over. - // The ml-anomalies index has aliases for each job, any - // of which could be used but that means one alias is - // treated differently. - // Using a `.` in the alias name avoids any conflicts - // as AD job Ids cannot start with `.` - String rolloverAlias = index + ".rollover_alias"; - - // If the index does not end in a digit then rollover does not know - // what to name the new index so it must be specified in the request. - // Otherwise leave null and rollover will calculate the new name - String newIndexName = MlIndexAndAlias.has6DigitSuffix(index) ? null : index + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; - IndicesAliasesRequestBuilder aliasRequestBuilder = client.admin() - .indices() - .prepareAliases( - MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT, - MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT - ); + Tuple newIndexNameAndRolloverAlias = MlIndexAndAlias.createRolloverAliasAndNewIndexName(index); + String rolloverAlias = newIndexNameAndRolloverAlias.v1(); + String newIndexName = newIndexNameAndRolloverAlias.v2(); + + IndicesAliasesRequestBuilder aliasRequestBuilder = MlIndexAndAlias.createIndicesAliasesRequestBuilder(client); SubscribableListener.newForked( l -> { createAliasForRollover(index, rolloverAlias, l.map(AcknowledgedResponse::isAcknowledged)); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index 300eab06360af..1ec27cfac620a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -41,6 +41,7 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.TaskInfo; @@ -79,7 +80,8 @@ */ public class MlDailyMaintenanceService implements Releasable { - private static final org.elasticsearch.logging.Logger logger = LogManager.getLogger(MlDailyMaintenanceService.class); + // The service to use for creating log messages. + private static final Logger logger = LogManager.getLogger(MlDailyMaintenanceService.class); private static final int MAX_TIME_OFFSET_MINUTES = 120; @@ -100,8 +102,6 @@ public class MlDailyMaintenanceService implements Releasable { private final boolean isDataFrameAnalyticsEnabled; private final boolean isNlpEnabled; - private RolloverConditions rolloverConditions; - private volatile Scheduler.Cancellable cancellable; private volatile float deleteExpiredDataRequestsPerSecond; private volatile ByteSizeValue rolloverMaxSize; @@ -125,12 +125,10 @@ public class MlDailyMaintenanceService implements Releasable { this.schedulerProvider = Objects.requireNonNull(schedulerProvider); this.expressionResolver = Objects.requireNonNull(expressionResolver); this.deleteExpiredDataRequestsPerSecond = MachineLearning.NIGHTLY_MAINTENANCE_REQUESTS_PER_SECOND.get(settings); - this.rolloverMaxSize = MachineLearning.NIGHTLY_MAINTENANCE_ROLLOVER_MAX_SIZE.get(settings); + this.rolloverMaxSize = MachineLearning.RESULTS_INDEX_ROLLOVER_MAX_SIZE.get(settings); this.isAnomalyDetectionEnabled = isAnomalyDetectionEnabled; this.isDataFrameAnalyticsEnabled = isDataFrameAnalyticsEnabled; this.isNlpEnabled = isNlpEnabled; - - this.rolloverConditions = RolloverConditions.newBuilder().addMaxIndexSizeCondition(rolloverMaxSize).build(); } public MlDailyMaintenanceService( @@ -163,7 +161,13 @@ void setDeleteExpiredDataRequestsPerSecond(float value) { this.deleteExpiredDataRequestsPerSecond = value; } - void setRolloverMaxSize(ByteSizeValue value) { + // Public for testing + /** + * Set the value of the rollover max size. + * + * @param value The value of the rollover max size. + */ + public void setRolloverMaxSize(ByteSizeValue value) { this.rolloverMaxSize = value; } @@ -187,7 +191,7 @@ private static TimeValue delayToNextTime(ClusterName clusterName) { } public synchronized void start() { - logger.info("Starting ML daily maintenance service"); + logger.debug("Starting ML daily maintenance service"); scheduleNext(); } @@ -292,7 +296,8 @@ private void triggerNlpMaintenance() { // Currently a NOOP } - void removeRolloverAlias( + // Helper function to remove an alias from a given index. + private void removeAlias( String index, String alias, IndicesAliasesRequestBuilder aliasRequestBuilder, @@ -308,33 +313,20 @@ private void rollover(Client client, String rolloverAlias, @Nullable String newI client, new RolloverRequestBuilder(client).setRolloverTarget(rolloverAlias) .setNewIndexName(newIndexName) - .setConditions(rolloverConditions) + .setConditions(RolloverConditions.newBuilder().addMaxIndexSizeCondition(rolloverMaxSize).build()) .request(), listener ); } private void rollAndUpdateAliases(ClusterState clusterState, String index, ActionListener listener) { - // Create an alias specifically for rolling over. - // The ml-anomalies index has aliases for each job, any - // of which could be used but that means one alias is - // treated differently. - // Using a `.` in the alias name avoids any conflicts - // as AD job Ids cannot start with `.` - String rolloverAlias = index + ".rollover_alias"; - OriginSettingClient originSettingClient = new OriginSettingClient(client, ML_ORIGIN); - // If the index does not end in a digit then rollover does not know - // what to name the new index so it must be specified in the request. - // Otherwise leave null and rollover will calculate the new name - String newIndexName = MlIndexAndAlias.has6DigitSuffix(index) ? null : index + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; - IndicesAliasesRequestBuilder aliasRequestBuilder = originSettingClient.admin() - .indices() - .prepareAliases( - MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT, - MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT - ); + Tuple newIndexNameAndRolloverAlias = MlIndexAndAlias.createRolloverAliasAndNewIndexName(index); + String rolloverAlias = newIndexNameAndRolloverAlias.v1(); + String newIndexName = newIndexNameAndRolloverAlias.v2(); + + IndicesAliasesRequestBuilder aliasRequestBuilder = MlIndexAndAlias.createIndicesAliasesRequestBuilder(client); // 4 Clean up any dangling aliases ActionListener aliasListener = ActionListener.wrap(listener::onResponse, e -> { @@ -342,19 +334,14 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio // Removal of the rollover alias may have failed in the case of rollover not occurring, e.g. when the rollover conditions // were not satisfied. // We must still clean up the temporary alias from the original index. - // The index name is either the original one provided or the original with a suffix appended. - var indexName = MlIndexAndAlias.has6DigitSuffix(index) ? index : index + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; - + var indexName = MlIndexAndAlias.ensureValidResultsIndexName(index); // Make sure we use a fresh IndicesAliasesRequestBuilder, the original one may have changed internal state. - IndicesAliasesRequestBuilder localAliasRequestBuilder = originSettingClient.admin() - .indices() - .prepareAliases( - MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT, - MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT - ); + IndicesAliasesRequestBuilder localAliasRequestBuilder = MlIndexAndAlias.createIndicesAliasesRequestBuilder( + originSettingClient + ); // Execute the cleanup, no need to propagate the original failure. - removeRolloverAlias(indexName, rolloverAlias, localAliasRequestBuilder, listener); + removeAlias(indexName, rolloverAlias, localAliasRequestBuilder, listener); } else { listener.onFailure(e); } @@ -367,13 +354,13 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio // Note that the rollover request is considered "successful" even if it didn't occur due to a condition not being met // (no exception will be thrown). In which case the attempt to remove the alias here will fail with an // IndexNotFoundException. We handle this case with a secondary listener. - removeRolloverAlias(newIndexNameResponse, rolloverAlias, aliasRequestBuilder, aliasListener); + removeAlias(newIndexNameResponse, rolloverAlias, aliasRequestBuilder, aliasListener); }, e -> { // If rollover fails, we must still clean up the temporary alias from the original index. // The index name is either the original one provided or the original with a suffix appended. - var indexName = MlIndexAndAlias.has6DigitSuffix(index) ? index : index + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; + var targetIndexName = MlIndexAndAlias.ensureValidResultsIndexName(index); // Execute the cleanup, no need to propagate the original failure. - removeRolloverAlias(indexName, rolloverAlias, aliasRequestBuilder, aliasListener); + removeAlias(targetIndexName, rolloverAlias, aliasRequestBuilder, aliasListener); }); // 2 rollover the index alias to the new index name @@ -385,16 +372,7 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio MlIndexAndAlias.createAliasForRollover(originSettingClient, index, rolloverAlias, getIndicesAliasesListener); } - // public for testing - public void setRolloverConditions(RolloverConditions rolloverConditions) { - this.rolloverConditions = Objects.requireNonNull(rolloverConditions); - } - - // public for testing - public void triggerRollResultsIndicesIfNecessaryTask(ActionListener finalListener) { - List failures = new ArrayList<>(); - - ClusterState clusterState = clusterService.state(); + private String[] findIndicesNeedingRollover(ClusterState clusterState) { // list all indices starting .ml-anomalies- // this includes the shared index and all custom results indices String[] indices = expressionResolver.concreteIndexNames( @@ -402,41 +380,29 @@ public void triggerRollResultsIndicesIfNecessaryTask(ActionListener updated = new PlainActionFuture<>(); - rollAndUpdateAliases(clusterState, index, updated); - try { - updated.actionGet(); - } catch (Exception ex) { - var message = "failed rolling over ml anomalies index [" + index + "]"; - logger.warn(message, ex); - if (ex instanceof ElasticsearchException elasticsearchException) { - failures.add(new ElasticsearchStatusException(message, elasticsearchException.status(), elasticsearchException)); - } else { - failures.add(new ElasticsearchStatusException(message, RestStatus.REQUEST_TIMEOUT, ex)); - } + private void rolloverIndexSafely(ClusterState clusterState, String index, List failures) { + PlainActionFuture updated = new PlainActionFuture<>(); + rollAndUpdateAliases(clusterState, index, updated); + try { + updated.actionGet(); + } catch (Exception ex) { + String message = String.format("Failed to rollover ML anomalies index [%s]: %s", index, ex.getMessage()); + logger.warn(message); + if (ex instanceof ElasticsearchException elasticsearchException) { + failures.add(new ElasticsearchStatusException(message, elasticsearchException.status(), elasticsearchException)); + } else { + failures.add(new ElasticsearchStatusException(message, RestStatus.REQUEST_TIMEOUT, ex)); } } + } + private void handleRolloverResults(String[] indices, List failures, ActionListener finalListener) { if (failures.isEmpty()) { - logger.info("ml anomalies indices [{}] rolled over and aliases updated", String.join(",", indices)); + logger.debug("ML anomalies indices [{}] rolled over and aliases updated", String.join(",", indices)); finalListener.onResponse(AcknowledgedResponse.TRUE); return; } @@ -445,6 +411,28 @@ public void triggerRollResultsIndicesIfNecessaryTask(ActionListener finalListener) { + logger.info("[ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask"); + + ClusterState clusterState = clusterService.state(); + + String[] indices = findIndicesNeedingRollover(clusterState); + if (indices.length == 0) { + // Early bath + finalListener.onResponse(AcknowledgedResponse.TRUE); + return; + } + + List failures = new ArrayList<>(); + + Arrays.stream(indices) + .filter(index -> MlIndexAndAlias.latestIndexMatchingBaseName(index, expressionResolver, clusterState).equals(index)) + .forEach(index -> rolloverIndexSafely(clusterState, index, failures)); + + handleRolloverResults(indices, failures, finalListener); + } + private void triggerDeleteExpiredDataTask(ActionListener finalListener) { ActionListener deleteExpiredDataActionListener = finalListener.delegateFailureAndWrap( (l, deleteExpiredDataResponse) -> { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java index 87efa40fb57dc..77ed3a978fa3c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java @@ -116,7 +116,7 @@ public void afterStart() { ); clusterService.getClusterSettings() .addSettingsUpdateConsumer( - MachineLearning.NIGHTLY_MAINTENANCE_ROLLOVER_MAX_SIZE, + MachineLearning.RESULTS_INDEX_ROLLOVER_MAX_SIZE, mlDailyMaintenanceService::setRolloverMaxSize ); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index 39c3fa8190459..164a6ea8ad560 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -240,7 +240,7 @@ public void putJob( jobBuilder.validateModelSnapshotRetentionSettingsAndSetDefaults(); validateCategorizationAnalyzerOrSetDefault(jobBuilder, analysisRegistry, minNodeVersion); - Job job = jobBuilder.build(new Date(), state, indexNameExpressionResolver); + Job job = jobBuilder.build(new Date()); ActionListener putJobListener = new ActionListener<>() { @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java index 923135fe6b23a..e084c38483ac1 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java @@ -65,6 +65,7 @@ import org.elasticsearch.xpack.core.ml.job.results.ModelPlot; import org.elasticsearch.xpack.core.ml.job.results.Result; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias; import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.job.retention.WritableIndexExpander; @@ -408,7 +409,7 @@ public void deleteJobDocuments( } } SearchResponse searchResponse = item.getResponse(); - if (searchResponse.getHits().getTotalHits().value() > 0 || indexNames.get()[i].equals(defaultSharedIndex)) { + if (searchResponse.getHits().getTotalHits().value() > 0 || MlIndexAndAlias.isAnomaliesSharedIndex(indexNames.get()[i])) { needToRunDBQTemp = true; } else { indicesToDelete.add(indexNames.get()[i]); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java index 1475d1c27815a..344522a822171 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java @@ -307,16 +307,18 @@ public void createJobResultIndex(Job job, ClusterState state, final ActionListen String readAliasName = AnomalyDetectorsIndex.jobResultsAliasedName(job.getId()); String writeAliasName = AnomalyDetectorsIndex.resultsWriteAlias(job.getId()); String tempIndexName = job.getInitialResultsIndexName(); + + // Ensure the index name is valid + tempIndexName = MlIndexAndAlias.ensureValidResultsIndexName(tempIndexName); + // Find all indices starting with this name and pick the latest one - String[] concreteIndices = resolver.concreteIndexNames(state, IndicesOptions.lenientExpandOpen(), tempIndexName + "*"); - if (concreteIndices.length > 0) { - tempIndexName = MlIndexAndAlias.latestIndex(concreteIndices); - } + tempIndexName = MlIndexAndAlias.latestIndexMatchingBaseName(tempIndexName, resolver, state); // Our read/write aliases should point to the concrete index // If the initial index is NOT an alias, either it is already a concrete index, or it does not exist yet if (state.getMetadata().getProject().hasAlias(tempIndexName)) { + String[] concreteIndices = resolver.concreteIndexNames(state, IndicesOptions.lenientExpandOpen(), tempIndexName + "*"); // SHOULD NOT be closed as in typical call flow checkForLeftOverDocuments already verified this // if it is closed, we bailout and return an error if (concreteIndices.length == 0) { From 21ea400aa3a6ddac50ac2ef14c9e4e698f2a785c Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 23 Oct 2025 16:43:51 +1300 Subject: [PATCH 35/42] Remove unneeded variable --- .../elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java index e084c38483ac1..36517e37ed72c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java @@ -391,8 +391,6 @@ public void deleteJobDocuments( deleteByQueryExecutor.onResponse(true); // We need to run DBQ and alias deletion return; } - String defaultSharedIndex = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX - + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; List indicesToDelete = new ArrayList<>(); boolean needToRunDBQTemp = false; assert multiSearchResponse.getResponses().length == indexNames.get().length; From f90e4b8005a9a4834d0ab81b26c238f7ff11f1d2 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 23 Oct 2025 03:50:10 +0000 Subject: [PATCH 36/42] [CI] Auto commit changes from spotless --- .../elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java index 36517e37ed72c..0bf165060f803 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java @@ -53,7 +53,6 @@ import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; -import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.CategorizerState; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; From 000e1b3d09b9e2b5f8d939d8e9ed7895ba9ee5ee Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 24 Oct 2025 11:04:00 +1300 Subject: [PATCH 37/42] Bugfix and typo --- .../java/org/elasticsearch/xpack/core/ml/job/config/Job.java | 5 +---- .../elasticsearch/xpack/ml/MlDailyMaintenanceService.java | 3 ++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index f71dca638ea5a..4dc54aef62cef 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -30,7 +30,6 @@ import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; -import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias; import org.elasticsearch.xpack.core.ml.utils.MlStrings; import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; @@ -1345,9 +1344,7 @@ public Job build() { if (Strings.isNullOrEmpty(resultsIndexName)) { resultsIndexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; - } else if (MlIndexAndAlias.isAnomaliesSharedIndex( - AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + resultsIndexName - ) == false) { + } else if (resultsIndexName.equals(AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT) == false) { // User-defined names are prepended with "custom" // Conditional guards against multiple prepending due to updates instead of first creation resultsIndexName = resultsIndexName.startsWith("custom-") ? resultsIndexName : "custom-" + resultsIndexName; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index 1ec27cfac620a..5656a94fe8eb3 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; @@ -390,7 +391,7 @@ private void rolloverIndexSafely(ClusterState clusterState, String index, List Date: Fri, 24 Oct 2025 13:30:04 +1300 Subject: [PATCH 38/42] More tests and fixes --- .../xpack/core/ml/utils/MlIndexAndAlias.java | 11 ++++--- .../core/ml/utils/MlIndexAndAliasTests.java | 31 +++++++++++++++++++ .../ml/integration/JobResultsProviderIT.java | 3 +- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java index e534c04e64046..cf63a75acf5e4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java @@ -47,6 +47,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Comparator; +import java.util.Objects; import java.util.Optional; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -79,7 +80,7 @@ public final class MlIndexAndAlias { private static final Predicate IS_ANOMALIES_SHARED_INDEX = Pattern.compile( AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT + "-\\d{6}" ).asMatchPredicate(); - private static final String ROLLOVER_ALIAS_SUFFIX = ".rollover_alias"; + public static final String ROLLOVER_ALIAS_SUFFIX = ".rollover_alias"; static final Comparator INDEX_NAME_COMPARATOR = (index1, index2) -> { String[] index1Parts = index1.split("-"); @@ -577,18 +578,20 @@ public static void rollover(Client client, RolloverRequest rolloverRequest, Acti } public static Tuple createRolloverAliasAndNewIndexName(String index) { + String indexName = Objects.requireNonNull(index); + // Create an alias specifically for rolling over. // The ml-anomalies index has aliases for each job, any // of which could be used but that means one alias is // treated differently. - // Using a `.` in the alias name avoids any conflicts + // ROLLOVER_ALIAS_SUFFIX puts a `.` in the alias name to avoid any conflicts // as AD job Ids cannot start with `.` - String rolloverAlias = index + ROLLOVER_ALIAS_SUFFIX; + String rolloverAlias = indexName + ROLLOVER_ALIAS_SUFFIX; // If the index does not end in a digit then rollover does not know // what to name the new index so it must be specified in the request. // Otherwise leave null and rollover will calculate the new name - String newIndexName = MlIndexAndAlias.has6DigitSuffix(index) ? null : index + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; + String newIndexName = MlIndexAndAlias.has6DigitSuffix(index) ? null : indexName + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; return new Tuple<>(rolloverAlias, newIndexName); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java index f98c465ecac6a..64d25f1c957e6 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java @@ -404,6 +404,37 @@ public void testHas6DigitSuffix() { assertFalse(MlIndexAndAlias.has6DigitSuffix("index000001")); } + public void testIsAnomaliesSharedIndex() { + assertTrue(MlIndexAndAlias.isAnomaliesSharedIndex(".ml-anomalies-shared-000001")); + assertTrue(MlIndexAndAlias.isAnomaliesSharedIndex(".ml-anomalies-shared-000007")); + assertTrue(MlIndexAndAlias.isAnomaliesSharedIndex(".ml-anomalies-shared-100000")); + assertTrue(MlIndexAndAlias.isAnomaliesSharedIndex(".ml-anomalies-shared-999999")); + assertFalse(MlIndexAndAlias.isAnomaliesSharedIndex(".ml-anomalies-shared-1000000")); + assertFalse(MlIndexAndAlias.isAnomaliesSharedIndex(".ml-anomalies-shared-00001")); + assertFalse(MlIndexAndAlias.isAnomaliesSharedIndex(".ml-anomalies-custom-fred-000007")); + assertFalse(MlIndexAndAlias.isAnomaliesSharedIndex("shared-000007")); + assertFalse(MlIndexAndAlias.isAnomaliesSharedIndex(".ml-anomalies-shared")); + assertFalse(MlIndexAndAlias.isAnomaliesSharedIndex(".ml-anomalies-shared000007")); + assertFalse(MlIndexAndAlias.isAnomaliesSharedIndex(".ml-anomalies-state-000007")); + assertFalse(MlIndexAndAlias.isAnomaliesSharedIndex(".ml-anomalies-annotations-000007")); + assertFalse(MlIndexAndAlias.isAnomaliesSharedIndex(".ml-anomalies-stats-000007")); + } + + public void testCreateRolloverAliasAndNewIndexName() { + var alias_index1 = MlIndexAndAlias.createRolloverAliasAndNewIndexName("fred"); + assertThat(alias_index1.v1(), equalTo("fred" + MlIndexAndAlias.ROLLOVER_ALIAS_SUFFIX)); + assertThat(alias_index1.v2(), equalTo("fred" + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX)); + + var alias_index2 = MlIndexAndAlias.createRolloverAliasAndNewIndexName("derf" + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX); + assertThat( + alias_index2.v1(), + equalTo("derf" + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX + MlIndexAndAlias.ROLLOVER_ALIAS_SUFFIX) + ); + assertThat(alias_index2.v2(), equalTo(null)); + + assertThrows(NullPointerException.class, () -> MlIndexAndAlias.createRolloverAliasAndNewIndexName(null)); + } + public void testLatestIndexMatchingBaseName_isLatest() { Metadata.Builder metadata = Metadata.builder(); metadata.put(createSharedResultsIndex(".ml-anomalies-custom-foo", IndexVersion.current(), List.of("job1"))); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java index d676e2405ace3..eadf6a1b598a7 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java @@ -155,7 +155,8 @@ public void testPutJob_CreatesResultsIndex() { // Put fist job. This should create the results index as it's the first job. client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(job1)).actionGet(); - String sharedResultsIndex = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; + String sharedResultsIndex = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT + + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; Map mappingProperties = getIndexMappingProperties(sharedResultsIndex); // Assert mappings have a few fields from the template From 74d92aed6bfce79faeb62a1b076314d0f6d740e2 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 30 Oct 2025 10:47:35 +1300 Subject: [PATCH 39/42] * Clarified documentation regarding results_index_rollover_max_size * Added the ability to set results_index_rollover_max_size to -1B to configure the nightly maintenance task not to trigger indices rollover --- .../machine-learning-settings.md | 2 +- ...enanceServiceRolloverResultsIndicesIT.java | 24 ++++++++++++++++--- .../xpack/ml/MachineLearning.java | 2 +- .../xpack/ml/MlDailyMaintenanceService.java | 5 ++-- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md b/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md index 68ae51dd26715..a5233f364de93 100644 --- a/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md +++ b/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md @@ -87,7 +87,7 @@ $$$xpack.ml.max_open_jobs$$$ : ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The rate at which the nightly maintenance task deletes expired model snapshots and results. The setting is a proxy to the [`requests_per_second`](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-delete-by-query) parameter used in the delete by query requests and controls throttling. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Valid values must be greater than `0.0` or equal to `-1.0`, where `-1.0` means a default value is used. Defaults to `-1.0` `xpack.ml.results_index_rollover_max_size` -: ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The maximum size the anomaly detection results indices can reach before being rolled over by the nightly maintenance task. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Valid values must be greater than or equal to `0B`. A value of `0B` means the indices will always be rolled over. Defaults to `50GB`. +: ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The maximum size the anomaly detection results indices can reach before being rolled over by the nightly maintenance task. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Valid values must be greater than or equal to `-1B`. A value of `-1B` means the indices will never be rolled over. A value of `0B` means the indices will always be rolled over, regardless of size. Defaults to `50GB`. `xpack.ml.node_concurrent_job_allocations` : ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The maximum number of jobs that can concurrently be in the `opening` state on each node. Typically, jobs spend a small amount of time in this state before they move to `open` state. Jobs that must restore large models when they are opening spend more time in the `opening` state. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Defaults to `2`. diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java index 16de2e5a18962..5e894d401e403 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java @@ -102,6 +102,24 @@ public void testTriggerRollResultsIndicesIfNecessaryTask_givenNoIndices() throws } } + public void testTriggerRollResultsIndicesIfNecessaryTask_givenMinusOnRolloverMaxSize() throws Exception { + // The null case, nothing to do. + + // set the rollover max size to -1B so the indices should not be rolled over + maintenanceService.setRolloverMaxSize(ByteSizeValue.MINUS_ONE); + + // Create jobs that will use the default results indices - ".ml-anomalies-shared-*" + Job.Builder[] jobs_with_default_index = { createJob("job_using_default_index"), createJob("another_job_using_default_index") }; + + // Create jobs that will use custom results indices - ".ml-anomalies-custom-fred-*" + Job.Builder[] jobs_with_custom_index = { + createJob("job_using_custom_index").setResultsIndexName("fred"), + createJob("another_job_using_custom_index").setResultsIndexName("fred") }; + + runTestScenarioWithNoRolloverOccurring(jobs_with_default_index, "shared"); + runTestScenarioWithNoRolloverOccurring(jobs_with_custom_index, "custom-fred"); + } + public void testTriggerRollResultsIndicesIfNecessaryTask_givenUnmetConditions() throws Exception { // Create jobs that will use the default results indices - ".ml-anomalies-shared-*" Job.Builder[] jobs_with_default_index = { createJob("job_using_default_index"), createJob("another_job_using_default_index") }; @@ -111,8 +129,8 @@ public void testTriggerRollResultsIndicesIfNecessaryTask_givenUnmetConditions() createJob("job_using_custom_index").setResultsIndexName("fred"), createJob("another_job_using_custom_index").setResultsIndexName("fred") }; - runTestScenarioWithUnmetConditions(jobs_with_default_index, "shared"); - runTestScenarioWithUnmetConditions(jobs_with_custom_index, "custom-fred"); + runTestScenarioWithNoRolloverOccurring(jobs_with_default_index, "shared"); + runTestScenarioWithNoRolloverOccurring(jobs_with_custom_index, "custom-fred"); } public void testTriggerRollResultsIndicesIfNecessaryTask_withMixedIndexTypes() throws Exception { @@ -207,7 +225,7 @@ public void testTriggerRollResultsIndicesIfNecessaryTask() throws Exception { runTestScenario(jobs_with_custom_index, "custom-fred"); } - private void runTestScenarioWithUnmetConditions(Job.Builder[] jobs, String indexNamePart) throws Exception { + private void runTestScenarioWithNoRolloverOccurring(Job.Builder[] jobs, String indexNamePart) throws Exception { String firstJobId = jobs[0].getId(); String secondJobId = jobs[1].getId(); String indexWildcard = AnomalyDetectorsIndex.jobResultsIndexPrefix() + indexNamePart + "*"; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 838d1c3ec96c5..20b2bc0bd5de0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -722,7 +722,7 @@ public void loadExtensions(ExtensionLoader loader) { public static final Setting RESULTS_INDEX_ROLLOVER_MAX_SIZE = Setting.byteSizeSetting( "xpack.ml.results_index_rollover_max_size", ByteSizeValue.of(50, ByteSizeUnit.GB), - ByteSizeValue.ofBytes(0L), + ByteSizeValue.ofBytes(-1L), ByteSizeValue.ofBytes(Long.MAX_VALUE), Property.OperatorDynamic, Property.NodeScope diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index 5656a94fe8eb3..348cdf0e11a2b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -81,9 +81,10 @@ */ public class MlDailyMaintenanceService implements Releasable { - // The service to use for creating log messages. private static final Logger logger = LogManager.getLogger(MlDailyMaintenanceService.class); + // The maximum value of a random offset used to calculate the time the nightly maintenance tasks are triggered on a given cluster. + // This is added in order to avoid multiple clusters running the maintenance tasks at the same time. private static final int MAX_TIME_OFFSET_MINUTES = 120; private final ThreadPool threadPool; @@ -419,7 +420,7 @@ public void triggerRollResultsIndicesIfNecessaryTask(ActionListener Date: Wed, 5 Nov 2025 16:40:56 +1300 Subject: [PATCH 40/42] [ML] Add daily task to manage .ml-state indices Add a daily maintenance task to roll over .ml-state indices if the index size exceeds a configurable default size (default 50GB). This replaces the previous method of using ILM to manage the state indices, as that was not a workable solution for serverless. This builds on the work done in PR #136065 which provides similar functionality for results indices. --- .../xpack/core/ml/utils/MlIndexAndAlias.java | 56 ++++- .../core/ml/utils/MlIndexAndAliasTests.java | 2 +- .../state_index_template.json | 4 +- ...yMaintenanceServiceRolloverIndicesIT.java} | 226 +++++++++++++++--- .../xpack/ml/MlAnomaliesIndexUpdate.java | 2 +- .../xpack/ml/MlDailyMaintenanceService.java | 115 +++++---- .../xpack/ml/MlIndexTemplateRegistry.java | 3 - .../ml/MlIndexTemplateRegistryTests.java | 2 - 8 files changed, 315 insertions(+), 95 deletions(-) rename x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/{MlDailyMaintenanceServiceRolloverResultsIndicesIT.java => MlDailyMaintenanceServiceRolloverIndicesIT.java} (66%) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java index cf63a75acf5e4..d3f213b4e0c64 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java @@ -80,6 +80,9 @@ public final class MlIndexAndAlias { private static final Predicate IS_ANOMALIES_SHARED_INDEX = Pattern.compile( AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT + "-\\d{6}" ).asMatchPredicate(); + private static final Predicate IS_ANOMALIES_STATE_INDEX = Pattern.compile( + AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX + "-\\d{6}" + ).asMatchPredicate(); public static final String ROLLOVER_ALIAS_SUFFIX = ".rollover_alias"; static final Comparator INDEX_NAME_COMPARATOR = (index1, index2) -> { @@ -495,6 +498,16 @@ public static boolean isAnomaliesSharedIndex(String indexName) { return IS_ANOMALIES_SHARED_INDEX.test(indexName); } + /** + * Checks if an index name matches the pattern for the ML anomalies state indices (e.g., ".ml-state-000001"). + * + * @param indexName The name of the index to check. + * @return {@code true} if the index is an anomalies state index, {@code false} otherwise. + */ + public static boolean isAnomaliesStateIndex(String indexName) { + return IS_ANOMALIES_STATE_INDEX.test(indexName); + } + /** * Returns the latest index. Latest is the index with the highest * 6 digit suffix. @@ -630,6 +643,47 @@ public static void updateAliases(IndicesAliasesRequestBuilder request, ActionLis request.execute(listener.delegateFailure((l, response) -> l.onResponse(Boolean.TRUE))); } + /** + * Adds alias actions to a request builder to move the ML state write alias from an old index to a new one after a rollover. + * This method is robust and will move the correct alias regardless of the current alias state on the old index. + * + * @param aliasRequestBuilder The request builder to add actions to. + * @param oldIndex The index from which the alias is being moved. + * @param newIndex The new index to which the alias will be moved. + * @param clusterState The current cluster state, used to inspect existing aliases on the old index. + * @param allStateIndices A list of all current .ml-state indices + * @return The modified {@link IndicesAliasesRequestBuilder}. + */ + public static IndicesAliasesRequestBuilder addStateIndexRolloverAliasActions( + IndicesAliasesRequestBuilder aliasRequestBuilder, + String oldIndex, + String newIndex, + ClusterState clusterState, + String[] allStateIndices + ) { + var meta = clusterState.metadata().getProject().index(oldIndex); + if (meta == null) { + // This should not happen in practice as we are iterating over existing indices, but we defend against it. + return aliasRequestBuilder; + } + + // Remove the write alias from ALL state indices to handle any inconsistencies where it might exist on more than one. + aliasRequestBuilder.addAliasAction( + IndicesAliasesRequest.AliasActions.remove().indices(allStateIndices).alias(AnomalyDetectorsIndex.jobStateIndexWriteAlias()) + ); + + aliasRequestBuilder.addAliasAction( + IndicesAliasesRequest.AliasActions.add() + .index(newIndex) + .alias(AnomalyDetectorsIndex.jobStateIndexWriteAlias()) + .isHidden(true) + .writeIndex(true) + ); + + return aliasRequestBuilder; + + } + /** * Adds alias actions to a request builder to move ML job aliases from an old index to a new one after a rollover. * This includes moving the write alias and re-creating the filtered read aliases on the new index. @@ -640,7 +694,7 @@ public static void updateAliases(IndicesAliasesRequestBuilder request, ActionLis * @param clusterState The current cluster state, used to inspect existing aliases on the old index. * @return The modified {@link IndicesAliasesRequestBuilder}. */ - public static IndicesAliasesRequestBuilder addIndexAliasesRequests( + public static IndicesAliasesRequestBuilder addResultsIndexRolloverAliasActions( IndicesAliasesRequestBuilder aliasRequestBuilder, String oldIndex, String newIndex, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java index 64d25f1c957e6..87d1af0e652a2 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java @@ -523,7 +523,7 @@ public void testBuildIndexAliasesRequest() { ); var newIndex = anomaliesIndex + "-000001"; - var request = MlIndexAndAlias.addIndexAliasesRequests(aliasRequestBuilder, anomaliesIndex, newIndex, csBuilder.build()); + var request = MlIndexAndAlias.addResultsIndexRolloverAliasActions(aliasRequestBuilder, anomaliesIndex, newIndex, csBuilder.build()); var actions = request.request().getAliasActions(); assertThat(actions, hasSize(6)); diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/state_index_template.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/state_index_template.json index 6f4d39fdb939a..19c9e4172b58e 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/state_index_template.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/state_index_template.json @@ -9,9 +9,7 @@ "index" : { "auto_expand_replicas" : "0-1", "hidden": true - }, - "index.lifecycle.name": "${xpack.ml.index.lifecycle.name}", - "index.lifecycle.rollover_alias": "${xpack.ml.index.lifecycle.rollover_alias}" + } }, "mappings" : { "_meta": { diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverIndicesIT.java similarity index 66% rename from x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java rename to x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverIndicesIT.java index 5e894d401e403..416f1c20ae3c0 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverIndicesIT.java @@ -8,8 +8,12 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.unit.ByteSizeValue; @@ -17,7 +21,6 @@ import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.ml.action.DeleteJobAction; import org.elasticsearch.xpack.core.ml.action.PutJobAction; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; @@ -33,12 +36,14 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import static org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex.createStateIndexAndAliasIfNecessary; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.mock; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1, numClientNodes = 0, supportsDedicatedMasters = false) -public class MlDailyMaintenanceServiceRolloverResultsIndicesIT extends BaseMlIntegTestCase { +public class MlDailyMaintenanceServiceRolloverIndicesIT extends BaseMlIntegTestCase { private MlDailyMaintenanceService maintenanceService; @@ -64,45 +69,80 @@ public void createComponents() throws Exception { ); } + /** + * In production the only way to create a model snapshot is to open a job, and + * opening a job ensures that the state index exists. This suite does not open jobs + * but instead inserts snapshot and state documents directly to the results and + * state indices. This means it needs to create the state index explicitly. This + * method should not be copied to test suites that run jobs in the way they are + * run in production. + */ + @Before + public void addMlState() { + PlainActionFuture future = new PlainActionFuture<>(); + createStateIndexAndAliasIfNecessary( + client(), + ClusterState.EMPTY_STATE, + TestIndexNameExpressionResolver.newInstance(), + TEST_REQUEST_TIMEOUT, + future + ); + future.actionGet(); + } + private void initClusterAndJob() { internalCluster().ensureAtLeastNumDataNodes(1); ensureStableCluster(1); } - public void testTriggerRollResultsIndicesIfNecessaryTask_givenNoIndices() throws Exception { + public void testTriggerIndicesIfNecessaryTask_givenNoIndices() throws Exception { // The null case, nothing to do. - // set the rollover max size to 0B so we can roll the index unconditionally + // Delete the .ml-state-000001 index for this particular test + PlainActionFuture future = new PlainActionFuture<>(); + DeleteIndexRequest request = new DeleteIndexRequest(".ml-state-000001"); + client().admin().indices().delete(request).actionGet(); + + // set the rollover max size to 0B so we can roll the indices unconditionally // It's not the conditions or even the rollover itself we are testing but the state of the indices and aliases afterwards. maintenanceService.setRolloverMaxSize(ByteSizeValue.ZERO); - { - GetIndexResponse getIndexResponse = client().admin() - .indices() - .prepareGetIndex(TEST_REQUEST_TIMEOUT) - .setIndices(".ml-anomalies*") - .get(); - logger.warn("get_index_response: {}", getIndexResponse.toString()); - assertThat(getIndexResponse.getIndices().length, is(0)); - var aliases = getIndexResponse.getAliases(); - assertThat(aliases.size(), is(0)); - } - blockingCall(maintenanceService::triggerRollResultsIndicesIfNecessaryTask); + Map>> params = Map.of(".ml-anomalies*", (listener) -> { + maintenanceService.triggerRollResultsIndicesIfNecessaryTask(listener); + }, ".ml-state*", (listener) -> { maintenanceService.triggerRollStateIndicesIfNecessaryTask(listener); }); + + for (Map.Entry>> param : params.entrySet()) { + String indexPattern = param.getKey(); + Consumer> function = param.getValue(); + { + GetIndexResponse getIndexResponse = client().admin() + .indices() + .prepareGetIndex(TEST_REQUEST_TIMEOUT) + .setIndices(indexPattern) + .get(); + logger.warn("get_index_response: {}", getIndexResponse.toString()); + assertThat(getIndexResponse.getIndices().length, is(0)); + var aliases = getIndexResponse.getAliases(); + assertThat(aliases.size(), is(0)); + } - { - GetIndexResponse getIndexResponse = client().admin() - .indices() - .prepareGetIndex(TEST_REQUEST_TIMEOUT) - .setIndices(".ml-anomalies*") - .get(); - logger.warn("get_index_response: {}", getIndexResponse.toString()); - assertThat(getIndexResponse.getIndices().length, is(0)); - var aliases = getIndexResponse.getAliases(); - assertThat(aliases.size(), is(0)); + blockingCall(function); + + { + GetIndexResponse getIndexResponse = client().admin() + .indices() + .prepareGetIndex(TEST_REQUEST_TIMEOUT) + .setIndices(indexPattern) + .get(); + logger.warn("get_index_response: {}", getIndexResponse.toString()); + assertThat(getIndexResponse.getIndices().length, is(0)); + var aliases = getIndexResponse.getAliases(); + assertThat(aliases.size(), is(0)); + } } } - public void testTriggerRollResultsIndicesIfNecessaryTask_givenMinusOnRolloverMaxSize() throws Exception { + public void testTriggerRollResultsIndicesIfNecessaryTask_givenMinusOneRolloverMaxSize() throws Exception { // The null case, nothing to do. // set the rollover max size to -1B so the indices should not be rolled over @@ -225,6 +265,127 @@ public void testTriggerRollResultsIndicesIfNecessaryTask() throws Exception { runTestScenario(jobs_with_custom_index, "custom-fred"); } + public void testTriggerRollStateIndicesIfNecessaryTask() throws Exception { + // 1. Ensure that rollover tasks will always execute + maintenanceService.setRolloverMaxSize(ByteSizeValue.ZERO); + + // 2. Check the state index exists and has the expected write alias + assertIndicesAndAliases( + "Before rollover (state)", + AnomalyDetectorsIndex.jobStateIndexPattern(), + Map.of(".ml-state-000001", List.of(".ml-state-write")) + ); + + // 3. Trigger a single maintenance run + blockingCall(maintenanceService::triggerRollStateIndicesIfNecessaryTask); + + // 4. Verify state index was rolled over correctly + assertIndicesAndAliases( + "After rollover (state)", + AnomalyDetectorsIndex.jobStateIndexPattern(), + Map.of(".ml-state-000001", List.of(), ".ml-state-000002", List.of(".ml-state-write")) + ); + + // 5. Trigger another maintenance run + blockingCall(maintenanceService::triggerRollStateIndicesIfNecessaryTask); + + // 6. Verify state index was rolled over correctly + assertIndicesAndAliases( + "After rollover (state)", + AnomalyDetectorsIndex.jobStateIndexPattern(), + Map.of(".ml-state-000001", List.of(), ".ml-state-000002", List.of(), ".ml-state-000003", List.of(".ml-state-write")) + ); + } + + public void testTriggerRollStateIndicesIfNecessaryTask_givenMinusOneRolloverMaxSize() throws Exception { + // The null case, nothing to do. + + // set the rollover max size to -1B so the indices should not be rolled over + maintenanceService.setRolloverMaxSize(ByteSizeValue.MINUS_ONE); + { + GetIndexResponse getIndexResponse = client().admin() + .indices() + .prepareGetIndex(TEST_REQUEST_TIMEOUT) + .setIndices(".ml-state*") + .get(); + logger.warn("get_index_response: {}", getIndexResponse.toString()); + assertIndicesAndAliases( + "Before rollover (state)", + AnomalyDetectorsIndex.jobStateIndexPattern(), + Map.of(".ml-state-000001", List.of(".ml-state-write")) + ); + } + + blockingCall(maintenanceService::triggerRollStateIndicesIfNecessaryTask); + + { + GetIndexResponse getIndexResponse = client().admin() + .indices() + .prepareGetIndex(TEST_REQUEST_TIMEOUT) + .setIndices(".ml-state*") + .get(); + assertIndicesAndAliases( + "After rollover (state)", + AnomalyDetectorsIndex.jobStateIndexPattern(), + Map.of(".ml-state-000001", List.of(".ml-state-write")) + ); + } + } + + public void testTriggerRollStateIndicesIfNecessaryTask_givenMissingWriteAlias() throws Exception { + // 1. Ensure that rollover tasks will always attempt to execute + maintenanceService.setRolloverMaxSize(ByteSizeValue.ZERO); + + // 2. Remove the write alias to create an inconsistent state + client().admin() + .indices() + .prepareAliases(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT) + .removeAlias(".ml-state-000001", AnomalyDetectorsIndex.jobStateIndexWriteAlias()) + .get(); + + assertIndicesAndAliases( + "Before rollover (state, missing alias)", + AnomalyDetectorsIndex.jobStateIndexPattern(), + Map.of(".ml-state-000001", List.of()) + ); + + // 3. Trigger a maintenance run and expect it to gracefully handle the missing write alias + blockingCall(maintenanceService::triggerRollStateIndicesIfNecessaryTask); + + // 4. Verify the index rolled over correctly and the write alias was added + assertIndicesAndAliases( + "After rollover (state, missing alias)", + AnomalyDetectorsIndex.jobStateIndexPattern(), + Map.of(".ml-state-000001", List.of(), ".ml-state-000002", List.of(".ml-state-write")) + ); + } + + public void testTriggerRollStateIndicesIfNecessaryTask_givenWriteAliasOnWrongIndex() throws Exception { + // 1. Ensure that rollover tasks will always attempt to execute + maintenanceService.setRolloverMaxSize(ByteSizeValue.ZERO); + + // 2. Create a second, newer state index + createIndex(".ml-state-000002"); + + // 3. Verify the initial state (write alias is on the older index) + assertIndicesAndAliases( + "Before rollover (state, alias on wrong index)", + AnomalyDetectorsIndex.jobStateIndexPattern(), + Map.of(".ml-state-000001", List.of(".ml-state-write"), ".ml-state-000002", List.of()) + ); + + // 4. The service finds .ml-state-000002 as the latest, but the rollover alias points to ...000001 + // Trigger a maintenance run and expect it to gracefully repair the wrongly seated write alias + blockingCall(maintenanceService::triggerRollStateIndicesIfNecessaryTask); + + // 5. Verify the index rolled over correctly and the write alias was moved to the latest index + assertIndicesAndAliases( + "After rollover (state, alias on wrong index)", + AnomalyDetectorsIndex.jobStateIndexPattern(), + Map.of(".ml-state-000001", List.of(), ".ml-state-000002", List.of(), ".ml-state-000003", List.of(".ml-state-write")) + ); + } + private void runTestScenarioWithNoRolloverOccurring(Job.Builder[] jobs, String indexNamePart) throws Exception { String firstJobId = jobs[0].getId(); String secondJobId = jobs[1].getId(); @@ -335,7 +496,8 @@ private void assertIndicesAndAliases(String context, String indexWildcard, Map { assertThat("Context: " + context, indices.size(), is(expectedAliases.size())); if (expectedAliasList.isEmpty()) { - assertThat("Context: " + context, aliases.size(), is(0)); + List actualAliasMetadata = aliases.get(indexName); + assertThat("Context: " + context, actualAliasMetadata, is(nullValue())); } else { List actualAliasMetadata = aliases.get(indexName); List actualAliasList = actualAliasMetadata.stream().map(AliasMetadata::alias).toList(); @@ -376,12 +538,4 @@ private PutJobAction.Response putJob(Job.Builder job) { PutJobAction.Request request = new PutJobAction.Request(job); return client().execute(PutJobAction.INSTANCE, request).actionGet(); } - - private void deleteJob(String jobId) { - try { - client().execute(DeleteJobAction.INSTANCE, new DeleteJobAction.Request(jobId)).actionGet(); - } catch (Exception e) { - // noop - } - } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java index 202cc020471f6..027afe839a664 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java @@ -154,7 +154,7 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio ).andThen((l, success) -> { rollover(rolloverAlias, newIndexName, l); }).andThen((l, newIndexNameResponse) -> { - MlIndexAndAlias.addIndexAliasesRequests(aliasRequestBuilder, index, newIndexNameResponse, clusterState); + MlIndexAndAlias.addResultsIndexRolloverAliasActions(aliasRequestBuilder, index, newIndexNameResponse, clusterState); // Delete the new alias created for the rollover action aliasRequestBuilder.removeAlias(newIndexNameResponse, rolloverAlias); MlIndexAndAlias.updateAliases(aliasRequestBuilder, l); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index 348cdf0e11a2b..792cb4d77dd5a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -253,41 +253,42 @@ private void triggerTasks() { } private void triggerAnomalyDetectionMaintenance() { - // Step 5: Log any error that could have happened - ActionListener finalListener = ActionListener.wrap(response -> { - if (response.isAcknowledged() == false) { - logger.warn("[ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask failed"); - } else { - logger.info("[ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask succeeded"); - } - }, e -> logger.warn("An error occurred during [ML] maintenance tasks execution ", e)); + // The maintenance tasks are chained, where each subsequent task is executed regardless of whether the previous one + // succeeded or failed. - // Step 4: Roll over results indices if necessary - ActionListener rollResultsIndicesIfNecessaryListener = ActionListener.wrap(unused -> { - triggerRollResultsIndicesIfNecessaryTask(finalListener); - }, e -> { - // Note: Steps 1-4 are independent, so continue upon errors. - triggerRollResultsIndicesIfNecessaryTask(finalListener); - }); + // Final step: Log completion + ActionListener finalListener = ActionListener.wrap( + response -> logger.info("Completed [ML] maintenance tasks"), + e -> logger.warn("An error occurred during [ML] maintenance tasks execution", e) + ); + + // Step 5: Roll over state indices + Runnable rollStateIndices = () -> triggerRollStateIndicesIfNecessaryTask(finalListener); + + // Step 4: Roll over results indices + Runnable rollResultsIndices = () -> triggerRollResultsIndicesIfNecessaryTask( + continueOnFailureListener("roll-state-indices", rollStateIndices) + ); // Step 3: Delete expired data - ActionListener deleteJobsListener = ActionListener.wrap(unused -> { - triggerDeleteExpiredDataTask(rollResultsIndicesIfNecessaryListener); - }, e -> { - // Note: Steps 1-4 are independent, so continue upon errors. - triggerDeleteExpiredDataTask(rollResultsIndicesIfNecessaryListener); - }); + Runnable deleteExpiredData = () -> triggerDeleteExpiredDataTask( + continueOnFailureListener("roll-results-indices", rollResultsIndices) + ); - // Step 2: Reset jobs that are in resetting state without task - ActionListener resetJobsListener = ActionListener.wrap(unused -> { - triggerResetJobsInStateResetWithoutResetTask(deleteJobsListener); - }, e -> { - // Note: Steps 1-4 are independent, so continue upon errors. - triggerResetJobsInStateResetWithoutResetTask(deleteJobsListener); - }); + // Step 2: Reset jobs that are in resetting state without a task + Runnable resetJobs = () -> triggerResetJobsInStateResetWithoutResetTask( + continueOnFailureListener("delete-expired-data", deleteExpiredData) + ); + + // Step 1: Delete jobs that are in deleting state without a task + triggerDeleteJobsInStateDeletingWithoutDeletionTask(continueOnFailureListener("reset-jobs", resetJobs)); + } - // Step 1: Delete jobs that are in deleting state without task - triggerDeleteJobsInStateDeletingWithoutDeletionTask(resetJobsListener); + private ActionListener continueOnFailureListener(String nextTaskName, Runnable next) { + return ActionListener.wrap(response -> next.run(), e -> { + logger.warn(() -> "A maintenance task failed, but maintenance will continue. Triggering next task [" + nextTaskName + "].", e); + next.run(); + }); } private void triggerDataFrameAnalyticsMaintenance() { @@ -321,7 +322,7 @@ private void rollover(Client client, String rolloverAlias, @Nullable String newI ); } - private void rollAndUpdateAliases(ClusterState clusterState, String index, ActionListener listener) { + private void rollAndUpdateAliases(ClusterState clusterState, String index, String[] allIndices, ActionListener listener) { OriginSettingClient originSettingClient = new OriginSettingClient(client, ML_ORIGIN); Tuple newIndexNameAndRolloverAlias = MlIndexAndAlias.createRolloverAliasAndNewIndexName(index); @@ -351,7 +352,17 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio // 3 Update aliases ActionListener rolloverListener = ActionListener.wrap(newIndexNameResponse -> { - MlIndexAndAlias.addIndexAliasesRequests(aliasRequestBuilder, index, newIndexNameResponse, clusterState); + if (MlIndexAndAlias.isAnomaliesStateIndex(index)) { + MlIndexAndAlias.addStateIndexRolloverAliasActions( + aliasRequestBuilder, + index, + newIndexNameResponse, + clusterState, + allIndices + ); + } else { + MlIndexAndAlias.addResultsIndexRolloverAliasActions(aliasRequestBuilder, index, newIndexNameResponse, clusterState); + } // On success, the rollover alias may have been moved to the new index, so we attempt to remove it from there. // Note that the rollover request is considered "successful" even if it didn't occur due to a condition not being met // (no exception will be thrown). In which case the attempt to remove the alias here will fail with an @@ -374,21 +385,16 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio MlIndexAndAlias.createAliasForRollover(originSettingClient, index, rolloverAlias, getIndicesAliasesListener); } - private String[] findIndicesNeedingRollover(ClusterState clusterState) { - // list all indices starting .ml-anomalies- - // this includes the shared index and all custom results indices - String[] indices = expressionResolver.concreteIndexNames( - clusterState, - IndicesOptions.lenientExpandOpenHidden(), - AnomalyDetectorsIndex.jobResultsIndexPattern() - ); - logger.trace("triggerRollResultsIndicesIfNecessaryTask: indices found: {}", Arrays.toString(indices)); + private String[] findIndicesMatchingPattern(ClusterState clusterState, String indexPattern) { + // list all indices matching the given index pattern + String[] indices = expressionResolver.concreteIndexNames(clusterState, IndicesOptions.lenientExpandOpenHidden(), indexPattern); + logger.trace("findIndicesMatchingPattern: indices found: {} matching pattern [{}]", Arrays.toString(indices), indexPattern); return indices; } - private void rolloverIndexSafely(ClusterState clusterState, String index, List failures) { + private void rolloverIndexSafely(ClusterState clusterState, String index, String[] allIndices, List failures) { PlainActionFuture updated = new PlainActionFuture<>(); - rollAndUpdateAliases(clusterState, index, updated); + rollAndUpdateAliases(clusterState, index, allIndices, updated); try { updated.actionGet(); } catch (Exception ex) { @@ -413,13 +419,16 @@ private void handleRolloverResults(String[] indices, List failures, A finalListener.onResponse(AcknowledgedResponse.FALSE); } - // public for testing - public void triggerRollResultsIndicesIfNecessaryTask(ActionListener finalListener) { - logger.info("[ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask"); + private void triggerRollIndicesIfNecessaryTask( + String taskName, + String indexPattern, + ActionListener finalListener + ) { + logger.info("[ML] maintenance task: [{}] for index pattern [{}]", taskName, indexPattern); ClusterState clusterState = clusterService.state(); - String[] indices = findIndicesNeedingRollover(clusterState); + String[] indices = findIndicesMatchingPattern(clusterState, indexPattern); if (rolloverMaxSize == ByteSizeValue.MINUS_ONE || indices.length == 0) { // Early bath finalListener.onResponse(AcknowledgedResponse.TRUE); @@ -430,11 +439,21 @@ public void triggerRollResultsIndicesIfNecessaryTask(ActionListener MlIndexAndAlias.latestIndexMatchingBaseName(index, expressionResolver, clusterState).equals(index)) - .forEach(index -> rolloverIndexSafely(clusterState, index, failures)); + .forEach(latestIndex -> rolloverIndexSafely(clusterState, latestIndex, indices, failures)); handleRolloverResults(indices, failures, finalListener); } + // public for testing + public void triggerRollResultsIndicesIfNecessaryTask(ActionListener finalListener) { + triggerRollIndicesIfNecessaryTask("roll-state-indices", AnomalyDetectorsIndex.jobResultsIndexPattern(), finalListener); + } + + // public for testing + public void triggerRollStateIndicesIfNecessaryTask(ActionListener finalListener) { + triggerRollIndicesIfNecessaryTask("roll-results-indices", AnomalyDetectorsIndex.jobStateIndexPattern(), finalListener); + } + private void triggerDeleteExpiredDataTask(ActionListener finalListener) { ActionListener deleteExpiredDataActionListener = finalListener.delegateFailureAndWrap( (l, deleteExpiredDataResponse) -> { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java index 02fcc2b4465f3..57a1dcb9bd0b0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java @@ -60,9 +60,6 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry { private IndexTemplateConfig stateTemplate() { Map variables = new HashMap<>(); variables.put(VERSION_ID_PATTERN, String.valueOf(ML_INDEX_TEMPLATE_VERSION)); - // In serverless a different version of "state_index_template.json" is shipped that won't substitute the ILM policy variable - variables.put(INDEX_LIFECYCLE_NAME, ML_SIZE_BASED_ILM_POLICY_NAME); - variables.put(INDEX_LIFECYCLE_ROLLOVER_ALIAS, AnomalyDetectorsIndex.jobStateIndexWriteAlias()); return new IndexTemplateConfig( AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX, diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java index acda7e981489d..5da433f09a1e6 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java @@ -103,8 +103,6 @@ public void testStateTemplate() { .findFirst() .orElseThrow(() -> new AssertionError("expected the ml state index template to be put")); ComposableIndexTemplate indexTemplate = req.indexTemplate(); - assertThat(indexTemplate.template().settings().get("index.lifecycle.name"), equalTo("ml-size-based-ilm-policy")); - assertThat(indexTemplate.template().settings().get("index.lifecycle.rollover_alias"), equalTo(".ml-state-write")); } public void testStatsTemplate() { From e95a85a5bcec72261aff384791df89cbc8ced6f1 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Wed, 5 Nov 2025 16:59:19 +1300 Subject: [PATCH 41/42] Update docs/changelog/137603.yaml --- docs/changelog/137603.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/137603.yaml diff --git a/docs/changelog/137603.yaml b/docs/changelog/137603.yaml new file mode 100644 index 0000000000000..8934c3fbecc27 --- /dev/null +++ b/docs/changelog/137603.yaml @@ -0,0 +1,5 @@ +pr: 137603 +summary: Add daily task to manage .ml-state indices +area: Machine Learning +type: enhancement +issues: [] From f6ac2754b65e05354abe294c4fd0832071d63e70 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Mon, 6 Oct 2025 15:20:36 +1300 Subject: [PATCH 42/42] [ML] Add daily task to manage .ml-state indices Add a daily maintenance task to roll over .ml-state indices if the index size exceeds a configurable default size (default 50GB). This replaces the previous method of using ILM to manage the state indices, as that was not a workable solution for serverless. This builds on the work done in PR #136065 which provides similar functionality for results indices. WIP --- .../xpack/core/ml/utils/MlIndexAndAlias.java | 56 ++++- .../core/ml/utils/MlIndexAndAliasTests.java | 2 +- .../state_index_template.json | 4 +- ...yMaintenanceServiceRolloverIndicesIT.java} | 226 +++++++++++++++--- .../xpack/ml/MlAnomaliesIndexUpdate.java | 2 +- .../xpack/ml/MlDailyMaintenanceService.java | 118 +++++---- .../xpack/ml/MlIndexTemplateRegistry.java | 3 - .../ml/MlIndexTemplateRegistryTests.java | 2 - 8 files changed, 317 insertions(+), 96 deletions(-) rename x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/{MlDailyMaintenanceServiceRolloverResultsIndicesIT.java => MlDailyMaintenanceServiceRolloverIndicesIT.java} (66%) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java index cf63a75acf5e4..d3f213b4e0c64 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java @@ -80,6 +80,9 @@ public final class MlIndexAndAlias { private static final Predicate IS_ANOMALIES_SHARED_INDEX = Pattern.compile( AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT + "-\\d{6}" ).asMatchPredicate(); + private static final Predicate IS_ANOMALIES_STATE_INDEX = Pattern.compile( + AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX + "-\\d{6}" + ).asMatchPredicate(); public static final String ROLLOVER_ALIAS_SUFFIX = ".rollover_alias"; static final Comparator INDEX_NAME_COMPARATOR = (index1, index2) -> { @@ -495,6 +498,16 @@ public static boolean isAnomaliesSharedIndex(String indexName) { return IS_ANOMALIES_SHARED_INDEX.test(indexName); } + /** + * Checks if an index name matches the pattern for the ML anomalies state indices (e.g., ".ml-state-000001"). + * + * @param indexName The name of the index to check. + * @return {@code true} if the index is an anomalies state index, {@code false} otherwise. + */ + public static boolean isAnomaliesStateIndex(String indexName) { + return IS_ANOMALIES_STATE_INDEX.test(indexName); + } + /** * Returns the latest index. Latest is the index with the highest * 6 digit suffix. @@ -630,6 +643,47 @@ public static void updateAliases(IndicesAliasesRequestBuilder request, ActionLis request.execute(listener.delegateFailure((l, response) -> l.onResponse(Boolean.TRUE))); } + /** + * Adds alias actions to a request builder to move the ML state write alias from an old index to a new one after a rollover. + * This method is robust and will move the correct alias regardless of the current alias state on the old index. + * + * @param aliasRequestBuilder The request builder to add actions to. + * @param oldIndex The index from which the alias is being moved. + * @param newIndex The new index to which the alias will be moved. + * @param clusterState The current cluster state, used to inspect existing aliases on the old index. + * @param allStateIndices A list of all current .ml-state indices + * @return The modified {@link IndicesAliasesRequestBuilder}. + */ + public static IndicesAliasesRequestBuilder addStateIndexRolloverAliasActions( + IndicesAliasesRequestBuilder aliasRequestBuilder, + String oldIndex, + String newIndex, + ClusterState clusterState, + String[] allStateIndices + ) { + var meta = clusterState.metadata().getProject().index(oldIndex); + if (meta == null) { + // This should not happen in practice as we are iterating over existing indices, but we defend against it. + return aliasRequestBuilder; + } + + // Remove the write alias from ALL state indices to handle any inconsistencies where it might exist on more than one. + aliasRequestBuilder.addAliasAction( + IndicesAliasesRequest.AliasActions.remove().indices(allStateIndices).alias(AnomalyDetectorsIndex.jobStateIndexWriteAlias()) + ); + + aliasRequestBuilder.addAliasAction( + IndicesAliasesRequest.AliasActions.add() + .index(newIndex) + .alias(AnomalyDetectorsIndex.jobStateIndexWriteAlias()) + .isHidden(true) + .writeIndex(true) + ); + + return aliasRequestBuilder; + + } + /** * Adds alias actions to a request builder to move ML job aliases from an old index to a new one after a rollover. * This includes moving the write alias and re-creating the filtered read aliases on the new index. @@ -640,7 +694,7 @@ public static void updateAliases(IndicesAliasesRequestBuilder request, ActionLis * @param clusterState The current cluster state, used to inspect existing aliases on the old index. * @return The modified {@link IndicesAliasesRequestBuilder}. */ - public static IndicesAliasesRequestBuilder addIndexAliasesRequests( + public static IndicesAliasesRequestBuilder addResultsIndexRolloverAliasActions( IndicesAliasesRequestBuilder aliasRequestBuilder, String oldIndex, String newIndex, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java index 64d25f1c957e6..87d1af0e652a2 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java @@ -523,7 +523,7 @@ public void testBuildIndexAliasesRequest() { ); var newIndex = anomaliesIndex + "-000001"; - var request = MlIndexAndAlias.addIndexAliasesRequests(aliasRequestBuilder, anomaliesIndex, newIndex, csBuilder.build()); + var request = MlIndexAndAlias.addResultsIndexRolloverAliasActions(aliasRequestBuilder, anomaliesIndex, newIndex, csBuilder.build()); var actions = request.request().getAliasActions(); assertThat(actions, hasSize(6)); diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/state_index_template.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/state_index_template.json index 6f4d39fdb939a..19c9e4172b58e 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/state_index_template.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/state_index_template.json @@ -9,9 +9,7 @@ "index" : { "auto_expand_replicas" : "0-1", "hidden": true - }, - "index.lifecycle.name": "${xpack.ml.index.lifecycle.name}", - "index.lifecycle.rollover_alias": "${xpack.ml.index.lifecycle.rollover_alias}" + } }, "mappings" : { "_meta": { diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverIndicesIT.java similarity index 66% rename from x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java rename to x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverIndicesIT.java index 5e894d401e403..416f1c20ae3c0 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverResultsIndicesIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceRolloverIndicesIT.java @@ -8,8 +8,12 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.unit.ByteSizeValue; @@ -17,7 +21,6 @@ import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.ml.action.DeleteJobAction; import org.elasticsearch.xpack.core.ml.action.PutJobAction; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; @@ -33,12 +36,14 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import static org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex.createStateIndexAndAliasIfNecessary; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.mock; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1, numClientNodes = 0, supportsDedicatedMasters = false) -public class MlDailyMaintenanceServiceRolloverResultsIndicesIT extends BaseMlIntegTestCase { +public class MlDailyMaintenanceServiceRolloverIndicesIT extends BaseMlIntegTestCase { private MlDailyMaintenanceService maintenanceService; @@ -64,45 +69,80 @@ public void createComponents() throws Exception { ); } + /** + * In production the only way to create a model snapshot is to open a job, and + * opening a job ensures that the state index exists. This suite does not open jobs + * but instead inserts snapshot and state documents directly to the results and + * state indices. This means it needs to create the state index explicitly. This + * method should not be copied to test suites that run jobs in the way they are + * run in production. + */ + @Before + public void addMlState() { + PlainActionFuture future = new PlainActionFuture<>(); + createStateIndexAndAliasIfNecessary( + client(), + ClusterState.EMPTY_STATE, + TestIndexNameExpressionResolver.newInstance(), + TEST_REQUEST_TIMEOUT, + future + ); + future.actionGet(); + } + private void initClusterAndJob() { internalCluster().ensureAtLeastNumDataNodes(1); ensureStableCluster(1); } - public void testTriggerRollResultsIndicesIfNecessaryTask_givenNoIndices() throws Exception { + public void testTriggerIndicesIfNecessaryTask_givenNoIndices() throws Exception { // The null case, nothing to do. - // set the rollover max size to 0B so we can roll the index unconditionally + // Delete the .ml-state-000001 index for this particular test + PlainActionFuture future = new PlainActionFuture<>(); + DeleteIndexRequest request = new DeleteIndexRequest(".ml-state-000001"); + client().admin().indices().delete(request).actionGet(); + + // set the rollover max size to 0B so we can roll the indices unconditionally // It's not the conditions or even the rollover itself we are testing but the state of the indices and aliases afterwards. maintenanceService.setRolloverMaxSize(ByteSizeValue.ZERO); - { - GetIndexResponse getIndexResponse = client().admin() - .indices() - .prepareGetIndex(TEST_REQUEST_TIMEOUT) - .setIndices(".ml-anomalies*") - .get(); - logger.warn("get_index_response: {}", getIndexResponse.toString()); - assertThat(getIndexResponse.getIndices().length, is(0)); - var aliases = getIndexResponse.getAliases(); - assertThat(aliases.size(), is(0)); - } - blockingCall(maintenanceService::triggerRollResultsIndicesIfNecessaryTask); + Map>> params = Map.of(".ml-anomalies*", (listener) -> { + maintenanceService.triggerRollResultsIndicesIfNecessaryTask(listener); + }, ".ml-state*", (listener) -> { maintenanceService.triggerRollStateIndicesIfNecessaryTask(listener); }); + + for (Map.Entry>> param : params.entrySet()) { + String indexPattern = param.getKey(); + Consumer> function = param.getValue(); + { + GetIndexResponse getIndexResponse = client().admin() + .indices() + .prepareGetIndex(TEST_REQUEST_TIMEOUT) + .setIndices(indexPattern) + .get(); + logger.warn("get_index_response: {}", getIndexResponse.toString()); + assertThat(getIndexResponse.getIndices().length, is(0)); + var aliases = getIndexResponse.getAliases(); + assertThat(aliases.size(), is(0)); + } - { - GetIndexResponse getIndexResponse = client().admin() - .indices() - .prepareGetIndex(TEST_REQUEST_TIMEOUT) - .setIndices(".ml-anomalies*") - .get(); - logger.warn("get_index_response: {}", getIndexResponse.toString()); - assertThat(getIndexResponse.getIndices().length, is(0)); - var aliases = getIndexResponse.getAliases(); - assertThat(aliases.size(), is(0)); + blockingCall(function); + + { + GetIndexResponse getIndexResponse = client().admin() + .indices() + .prepareGetIndex(TEST_REQUEST_TIMEOUT) + .setIndices(indexPattern) + .get(); + logger.warn("get_index_response: {}", getIndexResponse.toString()); + assertThat(getIndexResponse.getIndices().length, is(0)); + var aliases = getIndexResponse.getAliases(); + assertThat(aliases.size(), is(0)); + } } } - public void testTriggerRollResultsIndicesIfNecessaryTask_givenMinusOnRolloverMaxSize() throws Exception { + public void testTriggerRollResultsIndicesIfNecessaryTask_givenMinusOneRolloverMaxSize() throws Exception { // The null case, nothing to do. // set the rollover max size to -1B so the indices should not be rolled over @@ -225,6 +265,127 @@ public void testTriggerRollResultsIndicesIfNecessaryTask() throws Exception { runTestScenario(jobs_with_custom_index, "custom-fred"); } + public void testTriggerRollStateIndicesIfNecessaryTask() throws Exception { + // 1. Ensure that rollover tasks will always execute + maintenanceService.setRolloverMaxSize(ByteSizeValue.ZERO); + + // 2. Check the state index exists and has the expected write alias + assertIndicesAndAliases( + "Before rollover (state)", + AnomalyDetectorsIndex.jobStateIndexPattern(), + Map.of(".ml-state-000001", List.of(".ml-state-write")) + ); + + // 3. Trigger a single maintenance run + blockingCall(maintenanceService::triggerRollStateIndicesIfNecessaryTask); + + // 4. Verify state index was rolled over correctly + assertIndicesAndAliases( + "After rollover (state)", + AnomalyDetectorsIndex.jobStateIndexPattern(), + Map.of(".ml-state-000001", List.of(), ".ml-state-000002", List.of(".ml-state-write")) + ); + + // 5. Trigger another maintenance run + blockingCall(maintenanceService::triggerRollStateIndicesIfNecessaryTask); + + // 6. Verify state index was rolled over correctly + assertIndicesAndAliases( + "After rollover (state)", + AnomalyDetectorsIndex.jobStateIndexPattern(), + Map.of(".ml-state-000001", List.of(), ".ml-state-000002", List.of(), ".ml-state-000003", List.of(".ml-state-write")) + ); + } + + public void testTriggerRollStateIndicesIfNecessaryTask_givenMinusOneRolloverMaxSize() throws Exception { + // The null case, nothing to do. + + // set the rollover max size to -1B so the indices should not be rolled over + maintenanceService.setRolloverMaxSize(ByteSizeValue.MINUS_ONE); + { + GetIndexResponse getIndexResponse = client().admin() + .indices() + .prepareGetIndex(TEST_REQUEST_TIMEOUT) + .setIndices(".ml-state*") + .get(); + logger.warn("get_index_response: {}", getIndexResponse.toString()); + assertIndicesAndAliases( + "Before rollover (state)", + AnomalyDetectorsIndex.jobStateIndexPattern(), + Map.of(".ml-state-000001", List.of(".ml-state-write")) + ); + } + + blockingCall(maintenanceService::triggerRollStateIndicesIfNecessaryTask); + + { + GetIndexResponse getIndexResponse = client().admin() + .indices() + .prepareGetIndex(TEST_REQUEST_TIMEOUT) + .setIndices(".ml-state*") + .get(); + assertIndicesAndAliases( + "After rollover (state)", + AnomalyDetectorsIndex.jobStateIndexPattern(), + Map.of(".ml-state-000001", List.of(".ml-state-write")) + ); + } + } + + public void testTriggerRollStateIndicesIfNecessaryTask_givenMissingWriteAlias() throws Exception { + // 1. Ensure that rollover tasks will always attempt to execute + maintenanceService.setRolloverMaxSize(ByteSizeValue.ZERO); + + // 2. Remove the write alias to create an inconsistent state + client().admin() + .indices() + .prepareAliases(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT) + .removeAlias(".ml-state-000001", AnomalyDetectorsIndex.jobStateIndexWriteAlias()) + .get(); + + assertIndicesAndAliases( + "Before rollover (state, missing alias)", + AnomalyDetectorsIndex.jobStateIndexPattern(), + Map.of(".ml-state-000001", List.of()) + ); + + // 3. Trigger a maintenance run and expect it to gracefully handle the missing write alias + blockingCall(maintenanceService::triggerRollStateIndicesIfNecessaryTask); + + // 4. Verify the index rolled over correctly and the write alias was added + assertIndicesAndAliases( + "After rollover (state, missing alias)", + AnomalyDetectorsIndex.jobStateIndexPattern(), + Map.of(".ml-state-000001", List.of(), ".ml-state-000002", List.of(".ml-state-write")) + ); + } + + public void testTriggerRollStateIndicesIfNecessaryTask_givenWriteAliasOnWrongIndex() throws Exception { + // 1. Ensure that rollover tasks will always attempt to execute + maintenanceService.setRolloverMaxSize(ByteSizeValue.ZERO); + + // 2. Create a second, newer state index + createIndex(".ml-state-000002"); + + // 3. Verify the initial state (write alias is on the older index) + assertIndicesAndAliases( + "Before rollover (state, alias on wrong index)", + AnomalyDetectorsIndex.jobStateIndexPattern(), + Map.of(".ml-state-000001", List.of(".ml-state-write"), ".ml-state-000002", List.of()) + ); + + // 4. The service finds .ml-state-000002 as the latest, but the rollover alias points to ...000001 + // Trigger a maintenance run and expect it to gracefully repair the wrongly seated write alias + blockingCall(maintenanceService::triggerRollStateIndicesIfNecessaryTask); + + // 5. Verify the index rolled over correctly and the write alias was moved to the latest index + assertIndicesAndAliases( + "After rollover (state, alias on wrong index)", + AnomalyDetectorsIndex.jobStateIndexPattern(), + Map.of(".ml-state-000001", List.of(), ".ml-state-000002", List.of(), ".ml-state-000003", List.of(".ml-state-write")) + ); + } + private void runTestScenarioWithNoRolloverOccurring(Job.Builder[] jobs, String indexNamePart) throws Exception { String firstJobId = jobs[0].getId(); String secondJobId = jobs[1].getId(); @@ -335,7 +496,8 @@ private void assertIndicesAndAliases(String context, String indexWildcard, Map { assertThat("Context: " + context, indices.size(), is(expectedAliases.size())); if (expectedAliasList.isEmpty()) { - assertThat("Context: " + context, aliases.size(), is(0)); + List actualAliasMetadata = aliases.get(indexName); + assertThat("Context: " + context, actualAliasMetadata, is(nullValue())); } else { List actualAliasMetadata = aliases.get(indexName); List actualAliasList = actualAliasMetadata.stream().map(AliasMetadata::alias).toList(); @@ -376,12 +538,4 @@ private PutJobAction.Response putJob(Job.Builder job) { PutJobAction.Request request = new PutJobAction.Request(job); return client().execute(PutJobAction.INSTANCE, request).actionGet(); } - - private void deleteJob(String jobId) { - try { - client().execute(DeleteJobAction.INSTANCE, new DeleteJobAction.Request(jobId)).actionGet(); - } catch (Exception e) { - // noop - } - } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java index 202cc020471f6..027afe839a664 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java @@ -154,7 +154,7 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio ).andThen((l, success) -> { rollover(rolloverAlias, newIndexName, l); }).andThen((l, newIndexNameResponse) -> { - MlIndexAndAlias.addIndexAliasesRequests(aliasRequestBuilder, index, newIndexNameResponse, clusterState); + MlIndexAndAlias.addResultsIndexRolloverAliasActions(aliasRequestBuilder, index, newIndexNameResponse, clusterState); // Delete the new alias created for the rollover action aliasRequestBuilder.removeAlias(newIndexNameResponse, rolloverAlias); MlIndexAndAlias.updateAliases(aliasRequestBuilder, l); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index 348cdf0e11a2b..24a90295df93b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -253,41 +253,42 @@ private void triggerTasks() { } private void triggerAnomalyDetectionMaintenance() { - // Step 5: Log any error that could have happened - ActionListener finalListener = ActionListener.wrap(response -> { - if (response.isAcknowledged() == false) { - logger.warn("[ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask failed"); - } else { - logger.info("[ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask succeeded"); - } - }, e -> logger.warn("An error occurred during [ML] maintenance tasks execution ", e)); + // The maintenance tasks are chained, where each subsequent task is executed regardless of whether the previous one + // succeeded or failed. - // Step 4: Roll over results indices if necessary - ActionListener rollResultsIndicesIfNecessaryListener = ActionListener.wrap(unused -> { - triggerRollResultsIndicesIfNecessaryTask(finalListener); - }, e -> { - // Note: Steps 1-4 are independent, so continue upon errors. - triggerRollResultsIndicesIfNecessaryTask(finalListener); - }); + // Final step: Log completion + ActionListener finalListener = ActionListener.wrap( + response -> logger.info("Completed [ML] maintenance tasks"), + e -> logger.warn("An error occurred during [ML] maintenance tasks execution", e) + ); + + // Step 5: Roll over state indices + Runnable rollStateIndices = () -> triggerRollStateIndicesIfNecessaryTask(finalListener); + + // Step 4: Roll over results indices + Runnable rollResultsIndices = () -> triggerRollResultsIndicesIfNecessaryTask( + continueOnFailureListener("roll-state-indices", rollStateIndices) + ); // Step 3: Delete expired data - ActionListener deleteJobsListener = ActionListener.wrap(unused -> { - triggerDeleteExpiredDataTask(rollResultsIndicesIfNecessaryListener); - }, e -> { - // Note: Steps 1-4 are independent, so continue upon errors. - triggerDeleteExpiredDataTask(rollResultsIndicesIfNecessaryListener); - }); + Runnable deleteExpiredData = () -> triggerDeleteExpiredDataTask( + continueOnFailureListener("roll-results-indices", rollResultsIndices) + ); - // Step 2: Reset jobs that are in resetting state without task - ActionListener resetJobsListener = ActionListener.wrap(unused -> { - triggerResetJobsInStateResetWithoutResetTask(deleteJobsListener); - }, e -> { - // Note: Steps 1-4 are independent, so continue upon errors. - triggerResetJobsInStateResetWithoutResetTask(deleteJobsListener); - }); + // Step 2: Reset jobs that are in resetting state without a task + Runnable resetJobs = () -> triggerResetJobsInStateResetWithoutResetTask( + continueOnFailureListener("delete-expired-data", deleteExpiredData) + ); - // Step 1: Delete jobs that are in deleting state without task - triggerDeleteJobsInStateDeletingWithoutDeletionTask(resetJobsListener); + // Step 1: Delete jobs that are in deleting state without a task + triggerDeleteJobsInStateDeletingWithoutDeletionTask(continueOnFailureListener("reset-jobs", resetJobs)); + } + + private ActionListener continueOnFailureListener(String nextTaskName, Runnable next) { + return ActionListener.wrap(response -> next.run(), e -> { + logger.warn(() -> "A maintenance task failed, but maintenance will continue. Triggering next task [" + nextTaskName + "].", e); + next.run(); + }); } private void triggerDataFrameAnalyticsMaintenance() { @@ -321,7 +322,7 @@ private void rollover(Client client, String rolloverAlias, @Nullable String newI ); } - private void rollAndUpdateAliases(ClusterState clusterState, String index, ActionListener listener) { + private void rollAndUpdateAliases(ClusterState clusterState, String index, String[] allIndices, ActionListener listener) { OriginSettingClient originSettingClient = new OriginSettingClient(client, ML_ORIGIN); Tuple newIndexNameAndRolloverAlias = MlIndexAndAlias.createRolloverAliasAndNewIndexName(index); @@ -351,7 +352,17 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio // 3 Update aliases ActionListener rolloverListener = ActionListener.wrap(newIndexNameResponse -> { - MlIndexAndAlias.addIndexAliasesRequests(aliasRequestBuilder, index, newIndexNameResponse, clusterState); + if (MlIndexAndAlias.isAnomaliesStateIndex(index)) { + MlIndexAndAlias.addStateIndexRolloverAliasActions( + aliasRequestBuilder, + index, + newIndexNameResponse, + clusterState, + allIndices + ); + } else { + MlIndexAndAlias.addResultsIndexRolloverAliasActions(aliasRequestBuilder, index, newIndexNameResponse, clusterState); + } // On success, the rollover alias may have been moved to the new index, so we attempt to remove it from there. // Note that the rollover request is considered "successful" even if it didn't occur due to a condition not being met // (no exception will be thrown). In which case the attempt to remove the alias here will fail with an @@ -374,21 +385,16 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio MlIndexAndAlias.createAliasForRollover(originSettingClient, index, rolloverAlias, getIndicesAliasesListener); } - private String[] findIndicesNeedingRollover(ClusterState clusterState) { - // list all indices starting .ml-anomalies- - // this includes the shared index and all custom results indices - String[] indices = expressionResolver.concreteIndexNames( - clusterState, - IndicesOptions.lenientExpandOpenHidden(), - AnomalyDetectorsIndex.jobResultsIndexPattern() - ); - logger.trace("triggerRollResultsIndicesIfNecessaryTask: indices found: {}", Arrays.toString(indices)); + private String[] findIndicesMatchingPattern(ClusterState clusterState, String indexPattern) { + // list all indices matching the given index pattern + String[] indices = expressionResolver.concreteIndexNames(clusterState, IndicesOptions.lenientExpandOpenHidden(), indexPattern); + logger.trace("findIndicesMatchingPattern: indices found: {} matching pattern [{}]", Arrays.toString(indices), indexPattern); return indices; } - private void rolloverIndexSafely(ClusterState clusterState, String index, List failures) { + private void rolloverIndexSafely(ClusterState clusterState, String index, String[] allIndices, List failures) { PlainActionFuture updated = new PlainActionFuture<>(); - rollAndUpdateAliases(clusterState, index, updated); + rollAndUpdateAliases(clusterState, index, allIndices, updated); try { updated.actionGet(); } catch (Exception ex) { @@ -413,13 +419,16 @@ private void handleRolloverResults(String[] indices, List failures, A finalListener.onResponse(AcknowledgedResponse.FALSE); } - // public for testing - public void triggerRollResultsIndicesIfNecessaryTask(ActionListener finalListener) { - logger.info("[ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask"); + private void triggerRollIndicesIfNecessaryTask( + String taskName, + String indexPattern, + ActionListener finalListener + ) { + logger.info("[ML] maintenance task: [{}] for index pattern [{}]", taskName, indexPattern); ClusterState clusterState = clusterService.state(); - String[] indices = findIndicesNeedingRollover(clusterState); + String[] indices = findIndicesMatchingPattern(clusterState, indexPattern); if (rolloverMaxSize == ByteSizeValue.MINUS_ONE || indices.length == 0) { // Early bath finalListener.onResponse(AcknowledgedResponse.TRUE); @@ -430,11 +439,21 @@ public void triggerRollResultsIndicesIfNecessaryTask(ActionListener MlIndexAndAlias.latestIndexMatchingBaseName(index, expressionResolver, clusterState).equals(index)) - .forEach(index -> rolloverIndexSafely(clusterState, index, failures)); + .forEach(latestIndex -> rolloverIndexSafely(clusterState, latestIndex, indices, failures)); handleRolloverResults(indices, failures, finalListener); } + // public for testing + public void triggerRollResultsIndicesIfNecessaryTask(ActionListener finalListener) { + triggerRollIndicesIfNecessaryTask("roll-state-indices", AnomalyDetectorsIndex.jobResultsIndexPattern(), finalListener); + } + + // public for testing + public void triggerRollStateIndicesIfNecessaryTask(ActionListener finalListener) { + triggerRollIndicesIfNecessaryTask("roll-results-indices", AnomalyDetectorsIndex.jobStateIndexPattern(), finalListener); + } + private void triggerDeleteExpiredDataTask(ActionListener finalListener) { ActionListener deleteExpiredDataActionListener = finalListener.delegateFailureAndWrap( (l, deleteExpiredDataResponse) -> { @@ -550,6 +569,7 @@ private void triggerJobsInStateWithoutMatchingTask( } chainTaskExecutor.execute(jobsActionListener); }, finalListener::onFailure); + ActionListener getJobsActionListener = ActionListener.wrap(getJobsResponse -> { Set jobsInState = getJobsResponse.getResponse().results().stream().filter(jobFilter).map(Job::getId).collect(toSet()); if (jobsInState.isEmpty()) { @@ -557,7 +577,6 @@ private void triggerJobsInStateWithoutMatchingTask( return; } jobsInStateHolder.set(jobsInState); - executeAsyncWithOrigin( client, ML_ORIGIN, @@ -566,6 +585,7 @@ private void triggerJobsInStateWithoutMatchingTask( listTasksActionListener ); }, finalListener::onFailure); + executeAsyncWithOrigin(client, ML_ORIGIN, GetJobsAction.INSTANCE, new GetJobsAction.Request("*"), getJobsActionListener); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java index 02fcc2b4465f3..57a1dcb9bd0b0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java @@ -60,9 +60,6 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry { private IndexTemplateConfig stateTemplate() { Map variables = new HashMap<>(); variables.put(VERSION_ID_PATTERN, String.valueOf(ML_INDEX_TEMPLATE_VERSION)); - // In serverless a different version of "state_index_template.json" is shipped that won't substitute the ILM policy variable - variables.put(INDEX_LIFECYCLE_NAME, ML_SIZE_BASED_ILM_POLICY_NAME); - variables.put(INDEX_LIFECYCLE_ROLLOVER_ALIAS, AnomalyDetectorsIndex.jobStateIndexWriteAlias()); return new IndexTemplateConfig( AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX, diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java index acda7e981489d..5da433f09a1e6 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java @@ -103,8 +103,6 @@ public void testStateTemplate() { .findFirst() .orElseThrow(() -> new AssertionError("expected the ml state index template to be put")); ComposableIndexTemplate indexTemplate = req.indexTemplate(); - assertThat(indexTemplate.template().settings().get("index.lifecycle.name"), equalTo("ml-size-based-ilm-policy")); - assertThat(indexTemplate.template().settings().get("index.lifecycle.rollover_alias"), equalTo(".ml-state-write")); } public void testStatsTemplate() {