diff --git a/docs/changelog/136065.yaml b/docs/changelog/136065.yaml new file mode 100644 index 0000000000000..45f073600e61e --- /dev/null +++ b/docs/changelog/136065.yaml @@ -0,0 +1,5 @@ +pr: 136065 +summary: Manage ad results indices +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md b/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md index 118fb5a480381..5eff8c71b3baa 100644 --- a/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md +++ b/docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md @@ -86,6 +86,9 @@ $$$xpack.ml.max_open_jobs$$$ `xpack.ml.nightly_maintenance_requests_per_second` : ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The rate at which the nightly maintenance task deletes expired model snapshots and results. The setting is a proxy to the [`requests_per_second`](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-delete-by-query) parameter used in the delete by query requests and controls throttling. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Valid values must be greater than `0.0` or equal to `-1.0`, where `-1.0` means a default value is used. Defaults to `-1.0` +`xpack.ml.nightly_maintenance_rollover_max_size` +: ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The maximum size the anomaly detection results indices can reach before being rolled over by the nightly maintenance task. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Valid values must be greater than `0B` or equal to `-1B`. Defaults to `50GB`. + `xpack.ml.node_concurrent_job_allocations` : ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The maximum number of jobs that can concurrently be in the `opening` state on each node. Typically, jobs spend a small amount of time in this state before they move to `open` state. Jobs that must restore large models when they are opening spend more time in the `opening` state. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Defaults to `2`. diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index ea715b0d5c921..6bde49708693c 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -85,6 +85,12 @@ tasks.named("precommit").configure { } tasks.named("yamlRestCompatTestTransform").configure({ task -> + task.replaceIsTrue("\\.ml-anomalies-shared.mappings._meta.version", "\\.ml-anomalies-shared-000001.mappings._meta.version") + task.replaceKeyInMatch("\\.ml-anomalies-shared.mappings.new_field.mapping.new_field.type", "\\.ml-anomalies-shared-000001.mappings.new_field.mapping.new_field.type") + task.replaceValueTextByKeyValue("index", ".ml-anomalies-shared", ".ml-anomalies-shared-000001") + task.replaceValueTextByKeyValue("index", ".ml-anomalies-custom-all-test-1,.ml-anomalies-custom-all-test-2", ".ml-anomalies-custom-all-test-1-000001,.ml-anomalies-custom-all-test-2-000001") + task.replaceValueTextByKeyValue("index", ".ml-anomalies-custom-all-test-1", ".ml-anomalies-custom-all-test-1-000001") + task.replaceValueTextByKeyValue("index", ".ml-anomalies-custom-all-test-2", ".ml-anomalies-custom-all-test-2-000001") task.skipTest("esql/60_usage/Basic ESQL usage output (telemetry)", "The telemetry output changed. We dropped a column. That's safe.") task.skipTest("inference/inference_crud/Test get all", "Assertions on number of inference models break due to default configs") task.skipTest("esql/60_usage/Basic ESQL usage output (telemetry) snapshot version", "The number of functions is constantly increasing") diff --git a/x-pack/plugin/core/src/main/java/module-info.java b/x-pack/plugin/core/src/main/java/module-info.java index bb183f8d3845e..42c4910d82782 100644 --- a/x-pack/plugin/core/src/main/java/module-info.java +++ b/x-pack/plugin/core/src/main/java/module-info.java @@ -26,6 +26,7 @@ requires org.apache.httpcomponents.client5.httpclient5; requires org.apache.httpcomponents.core5.httpcore5; requires org.slf4j; + requires org.elasticsearch.logging; exports org.elasticsearch.index.engine.frozen; exports org.elasticsearch.license; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index e663bbd6800bd..e98c551a536de 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -6,9 +6,12 @@ */ package org.elasticsearch.xpack.core.ml.job.config; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.SimpleDiffable; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -30,6 +33,7 @@ import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias; import org.elasticsearch.xpack.core.ml.utils.MlStrings; import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; @@ -805,6 +809,8 @@ public static class Builder implements Writeable { private boolean allowLazyOpen; private Blocked blocked = Blocked.none(); private DatafeedConfig.Builder datafeedConfig; + private SetOnce clusterState = new SetOnce<>(); + private SetOnce indexNameExpressionResolver = new SetOnce<>(); public Builder() {} @@ -879,6 +885,14 @@ public String getId() { return id; } + private void setClusterState(ClusterState state) { + this.clusterState.set(state); + } + + private void setIndexNameExpressionResolver(IndexNameExpressionResolver indexNameExpressionResolver) { + this.indexNameExpressionResolver.set(indexNameExpressionResolver); + } + public void setJobVersion(MlConfigVersion jobVersion) { this.jobVersion = jobVersion; } @@ -1305,6 +1319,16 @@ public void validateDetectorsAreUnique() { } } + public Job build( + @SuppressWarnings("HiddenField") Date createTime, + ClusterState state, + IndexNameExpressionResolver indexNameExpressionResolver + ) { + setClusterState(state); + setIndexNameExpressionResolver(indexNameExpressionResolver); + return build(createTime); + } + /** * Builds a job with the given {@code createTime} and the current version. * This should be used when a new job is created as opposed to {@link #build()}. @@ -1345,10 +1369,23 @@ public Job build() { if (Strings.isNullOrEmpty(resultsIndexName)) { resultsIndexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; } else if (resultsIndexName.equals(AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT) == false) { - // User-defined names are prepended with "custom" + // User-defined names are prepended with "custom" and end with a 6 digit suffix // Conditional guards against multiple prepending due to updates instead of first creation resultsIndexName = resultsIndexName.startsWith("custom-") ? resultsIndexName : "custom-" + resultsIndexName; } + + resultsIndexName = MlIndexAndAlias.has6DigitSuffix(resultsIndexName) ? resultsIndexName : resultsIndexName + "-000001"; + + if (indexNameExpressionResolver.get() != null && clusterState.get() != null) { + String tmpResultsIndexName = MlIndexAndAlias.latestIndexMatchingBaseName( + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + resultsIndexName, + indexNameExpressionResolver.get(), + clusterState.get() + ); + + resultsIndexName = tmpResultsIndexName.substring(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX.length()); + } + if (datafeedConfig != null) { if (datafeedConfig.getId() == null) { datafeedConfig.setId(id); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java index 2a0fff86ba494..d36d031a6a4c3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java @@ -14,7 +14,7 @@ public final class AnomalyDetectorsIndexFields { // ".write" rather than simply "write" to avoid the danger of clashing // with the read alias of a job whose name begins with "write-" public static final String RESULTS_INDEX_WRITE_PREFIX = RESULTS_INDEX_PREFIX + ".write-"; - public static final String RESULTS_INDEX_DEFAULT = "shared"; + public static final String RESULTS_INDEX_DEFAULT = "shared-000001"; private AnomalyDetectorsIndexFields() {} } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java index 2d8b2bbd19410..d9f2b5181a1d7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java @@ -20,6 +20,7 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; @@ -33,9 +34,13 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; import org.elasticsearch.xpack.core.template.IndexTemplateConfig; import java.io.IOException; @@ -456,4 +461,129 @@ public static String latestIndex(String[] concreteIndices) { public static boolean indexIsReadWriteCompatibleInV9(IndexVersion version) { return version.onOrAfter(IndexVersions.V_8_0_0); } + + /** + * Strip any suffix from the index name and find any other indices + * that match the base name. Then return the latest index from the + * matching ones. + * + * @param index The index to check + * @param expressionResolver The expression resolver + * @param latestState The latest cluster state + * @return The latest index that matches the base name of the given index + */ + public static String latestIndexMatchingBaseName( + String index, + IndexNameExpressionResolver expressionResolver, + ClusterState latestState + ) { + String baseIndexName = MlIndexAndAlias.has6DigitSuffix(index) + ? index.substring(0, index.length() - FIRST_INDEX_SIX_DIGIT_SUFFIX.length()) + : index; + + String[] matching = expressionResolver.concreteIndexNames( + latestState, + IndicesOptions.lenientExpandOpenHidden(), + baseIndexName + "*" + ); + + // We used to assert here if no matching indices could be found. However, when called _before_ a job is created it may be the case + // that no .ml-anomalies-shared* indices yet exist + if (matching.length == 0) { + return index; + } + + // Exclude indices that start with the same base name but are a different index + // e.g. .ml-anomalies-foobar should not be included when the index name is + // .ml-anomalies-foo + String[] filtered = Arrays.stream(matching).filter(i -> { + return i.equals(index) || (has6DigitSuffix(i) && i.length() == baseIndexName.length() + FIRST_INDEX_SIX_DIGIT_SUFFIX.length()); + }).toArray(String[]::new); + + return MlIndexAndAlias.latestIndex(filtered); + } + + public static void rollover(Client client, RolloverRequest rolloverRequest, ActionListener listener) { + client.admin() + .indices() + .rolloverIndex(rolloverRequest, ActionListener.wrap(response -> listener.onResponse(response.getNewIndex()), e -> { + if (e instanceof ResourceAlreadyExistsException alreadyExistsException) { + // The destination index already exists possibly because it has been rolled over already. + listener.onResponse(alreadyExistsException.getIndex().getName()); + } else { + listener.onFailure(e); + } + })); + } + + public static void createAliasForRollover( + org.elasticsearch.logging.Logger logger, + Client client, + String indexName, + String aliasName, + ActionListener listener + ) { + logger.warn("creating rollover [{}] alias for [{}]", aliasName, indexName); + client.admin() + .indices() + .prepareAliases(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS) + .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(indexName).alias(aliasName).isHidden(true)) + .execute(listener); + } + + public static void updateAliases(IndicesAliasesRequestBuilder request, ActionListener listener) { + request.execute(listener.delegateFailure((l, response) -> l.onResponse(Boolean.TRUE))); + } + + public static IndicesAliasesRequestBuilder addIndexAliasesRequests( + IndicesAliasesRequestBuilder aliasRequestBuilder, + String oldIndex, + String newIndex, + ClusterState clusterState + ) { + // Multiple jobs can share the same index each job + // has a read and write alias that needs updating + // after the rollover + var meta = clusterState.metadata().getProject().index(oldIndex); + assert meta != null; + if (meta == null) { + return aliasRequestBuilder; + } + + for (var alias : meta.getAliases().values()) { + if (isAnomaliesWriteAlias(alias.alias())) { + aliasRequestBuilder.addAliasAction( + IndicesAliasesRequest.AliasActions.add().index(newIndex).alias(alias.alias()).isHidden(true).writeIndex(true) + ); + aliasRequestBuilder.addAliasAction(IndicesAliasesRequest.AliasActions.remove().index(oldIndex).alias(alias.alias())); + } else if (isAnomaliesReadAlias(alias.alias())) { + String jobId = AnomalyDetectorsIndex.jobIdFromAlias(alias.alias()); + aliasRequestBuilder.addAliasAction( + IndicesAliasesRequest.AliasActions.add() + .index(newIndex) + .alias(alias.alias()) + .isHidden(true) + .filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), jobId)) + ); + } + } + + return aliasRequestBuilder; + } + + public static boolean isAnomaliesWriteAlias(String aliasName) { + return aliasName.startsWith(AnomalyDetectorsIndexFields.RESULTS_INDEX_WRITE_PREFIX); + } + + public static boolean isAnomaliesReadAlias(String aliasName) { + if (aliasName.startsWith(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX) == false) { + return false; + } + + // See {@link AnomalyDetectorsIndex#jobResultsAliasedName} + String jobIdPart = aliasName.substring(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX.length()); + // If this is a write alias it will start with a `.` character + // which is not a valid job id. + return MlStrings.isValidId(jobIdPart); + } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java index 4f4c90aacbf35..82f139d49ef8f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java @@ -500,7 +500,7 @@ public void testBuilder_setsIndexName() { Job.Builder builder = buildJobBuilder("foo"); builder.setResultsIndexName("carol"); Job job = builder.build(); - assertEquals(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-carol", job.getInitialResultsIndexName()); + assertEquals(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-carol-000001", job.getInitialResultsIndexName()); } public void testBuilder_withInvalidIndexNameThrows() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java index b4bdb45bfb7b4..d46c999850bea 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java @@ -38,6 +38,7 @@ import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; import org.elasticsearch.xpack.core.template.IndexTemplateConfig; import org.junit.After; @@ -357,6 +358,18 @@ public void testCreateStateIndexAndAliasIfNecessary_WriteAliasDoesNotExistButLeg assertThat(createRequest.aliases(), equalTo(Collections.singleton(new Alias(TEST_INDEX_ALIAS).isHidden(true)))); } + public void testIsAnomaliesWriteAlias() { + assertTrue(MlIndexAndAlias.isAnomaliesWriteAlias(AnomalyDetectorsIndex.resultsWriteAlias("foo"))); + assertFalse(MlIndexAndAlias.isAnomaliesWriteAlias(AnomalyDetectorsIndex.jobResultsAliasedName("foo"))); + assertFalse(MlIndexAndAlias.isAnomaliesWriteAlias("some-index")); + } + + public void testIsAnomaliesAlias() { + assertTrue(MlIndexAndAlias.isAnomaliesReadAlias(AnomalyDetectorsIndex.jobResultsAliasedName("foo"))); + assertFalse(MlIndexAndAlias.isAnomaliesReadAlias(AnomalyDetectorsIndex.resultsWriteAlias("foo"))); + assertFalse(MlIndexAndAlias.isAnomaliesReadAlias("some-index")); + } + public void testIndexNameComparator() { Comparator comparator = MlIndexAndAlias.INDEX_NAME_COMPARATOR; assertThat(Stream.of("test-000001").max(comparator).get(), equalTo("test-000001")); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/BulkFailureRetryIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/BulkFailureRetryIT.java index 39b1d711a9206..2d327324466cb 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/BulkFailureRetryIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/BulkFailureRetryIT.java @@ -44,7 +44,7 @@ public class BulkFailureRetryIT extends MlNativeAutodetectIntegTestCase { private final long now = System.currentTimeMillis(); private static final long DAY = Duration.ofDays(1).toMillis(); private final String jobId = "bulk-failure-retry-job"; - private final String resultsIndex = ".ml-anomalies-custom-bulk-failure-retry-job"; + private final String resultsIndex = ".ml-anomalies-custom-bulk-failure-retry-job-000001"; @Before public void putPastDataIntoIndex() { diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceIT.java index 4fe3ed61114c3..fdfb0216d5d9f 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlDailyMaintenanceServiceIT.java @@ -8,6 +8,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexVersion; @@ -54,6 +55,7 @@ public void testTriggerDeleteJobsInStateDeletingWithoutDeletionTask() throws Int client(), mock(ClusterService.class), mock(MlAssignmentNotifier.class), + mock(IndexNameExpressionResolver.class), true, true, true diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java index cebcb6631c9bf..1c1acc99f02ed 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java @@ -212,7 +212,7 @@ public void testCreateJobsWithIndexNameOption() throws Exception { "results_index_name" : "%s"}"""; String jobId1 = "create-jobs-with-index-name-option-job-1"; - String indexName = "non-default-index"; + String indexName = "non-default-index-000001"; putJob(jobId1, Strings.format(jobTemplate, indexName)); String jobId2 = "create-jobs-with-index-name-option-job-2"; @@ -406,7 +406,7 @@ public void testCreateJobInCustomSharedIndexUpdatesMapping() throws Exception { // Check the index mapping contains the first by_field_name Request getResultsMappingRequest = new Request( "GET", - AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-shared-index/_mapping" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-shared-index-000001/_mapping" ); getResultsMappingRequest.addParameter("pretty", null); String resultsMappingAfterJob1 = EntityUtils.toString(client().performRequest(getResultsMappingRequest).getEntity()); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnomalyJobCRUDIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnomalyJobCRUDIT.java index dcc16c0bea23b..ee8f3c24e90bd 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnomalyJobCRUDIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnomalyJobCRUDIT.java @@ -139,7 +139,7 @@ public void testCreateWithExistingQuantilesDocs() { public void testCreateWithExistingResultsDocs() { String jobId = "job-id-with-existing-docs"; testCreateWithExistingDocs( - prepareIndex(".ml-anomalies-shared").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + prepareIndex(".ml-anomalies-shared-000001").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .setId(jobId + "_1464739200000_1") .setSource("{\"job_id\": \"" + jobId + "\"}", XContentType.JSON) .request(), @@ -149,14 +149,14 @@ public void testCreateWithExistingResultsDocs() { public void testPutJobWithClosedResultsIndex() { String jobId = "job-with-closed-results-index"; - client().admin().indices().prepareCreate(".ml-anomalies-shared").get(); - client().admin().indices().prepareClose(".ml-anomalies-shared").get(); + client().admin().indices().prepareCreate(".ml-anomalies-shared-000001").get(); + client().admin().indices().prepareClose(".ml-anomalies-shared-000001").get(); ElasticsearchStatusException ex = expectThrows(ElasticsearchStatusException.class, () -> createJob(jobId)); assertThat( ex.getMessage(), containsString("Cannot create job [job-with-closed-results-index] as it requires closed index [.ml-anomalies-*]") ); - client().admin().indices().prepareDelete(".ml-anomalies-shared").get(); + client().admin().indices().prepareDelete(".ml-anomalies-shared-000001").get(); } public void testPutJobWithClosedStateIndex() { diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java index 842ed7a4a2a2e..194747e44cffe 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java @@ -369,7 +369,7 @@ public void testMlStateAndResultsIndicesNotAvailable() throws Exception { // Create the indices (using installed templates) and set the routing to specific nodes // State and results go on the state-and-results node, config goes on the config node - indicesAdmin().prepareCreate(".ml-anomalies-shared") + indicesAdmin().prepareCreate(".ml-anomalies-shared-000001") .setSettings( Settings.builder() .put("index.routing.allocation.include.ml-indices", "state-and-results") @@ -435,7 +435,7 @@ public void testMlStateAndResultsIndicesNotAvailable() throws Exception { ); assertThat(detailedMessage, containsString("because not all primary shards are active for the following indices")); assertThat(detailedMessage, containsString(".ml-state")); - assertThat(detailedMessage, containsString(".ml-anomalies-shared")); + assertThat(detailedMessage, containsString(".ml-anomalies-shared-000001")); logger.info("Start data node"); String nonMlNode = internalCluster().startNode( diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java index 9a513f2690917..d676e2405ace3 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java @@ -208,7 +208,7 @@ public void testPutJob_WithCustomResultsIndex() { client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(job)).actionGet(); - String customIndex = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-bar"; + String customIndex = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-bar-000001"; Map mappingProperties = getIndexMappingProperties(customIndex); // Assert mappings have a few fields from the template diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobStorageDeletionTaskIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobStorageDeletionTaskIT.java index 7e46f260ea475..61acf64fa5efe 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobStorageDeletionTaskIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobStorageDeletionTaskIT.java @@ -138,14 +138,14 @@ public void testDeleteDedicatedJobWithDataInShared() throws Exception { IndicesAliasesRequest.AliasActions.add() .alias(AnomalyDetectorsIndex.jobResultsAliasedName(jobIdDedicated)) .isHidden(true) - .index(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared") + .index(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared-000001") .writeIndex(false) .filter(QueryBuilders.boolQuery().filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), jobIdDedicated))) ) .addAliasAction( IndicesAliasesRequest.AliasActions.add() .alias(AnomalyDetectorsIndex.resultsWriteAlias(jobIdDedicated)) - .index(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared") + .index(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "shared-000001") .isHidden(true) .writeIndex(true) ) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 704b0d7634db4..29be1a5938efc 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -39,6 +39,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; +import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.Processors; import org.elasticsearch.common.util.FeatureFlag; @@ -718,6 +719,13 @@ public void loadExtensions(ExtensionLoader loader) { Property.NodeScope ); + public static final Setting NIGHTLY_MAINTENANCE_ROLLOVER_MAX_SIZE = Setting.byteSizeSetting( + "xpack.ml.nightly_maintenance_rollover_max_size", + ByteSizeValue.of(50, ByteSizeUnit.GB), + Property.OperatorDynamic, + Property.NodeScope + ); + /** * This is the maximum possible node size for a machine learning node. It is useful when determining if a job could ever be opened * on the cluster. @@ -841,6 +849,7 @@ public List> getSettings() { ModelLoadingService.INFERENCE_MODEL_CACHE_TTL, ResultsPersisterService.PERSIST_RESULTS_MAX_RETRIES, NIGHTLY_MAINTENANCE_REQUESTS_PER_SECOND, + NIGHTLY_MAINTENANCE_ROLLOVER_MAX_SIZE, MachineLearningField.USE_AUTO_MACHINE_MEMORY_PERCENT, MAX_ML_NODE_SIZE, DELAYED_DATA_CHECK_FREQ, @@ -1348,6 +1357,7 @@ public Collection createComponents(PluginServices services) { client, adaptiveAllocationsScalerService, mlAssignmentNotifier, + indexNameExpressionResolver, machineLearningExtension.get().isAnomalyDetectionEnabled(), machineLearningExtension.get().isDataFrameAnalyticsEnabled(), machineLearningExtension.get().isNlpEnabled() diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java index 2aeaa948799d2..65d7fda17e17a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdate.java @@ -9,10 +9,8 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; @@ -26,23 +24,16 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.core.Nullable; -import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; -import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias; -import org.elasticsearch.xpack.core.ml.utils.MlStrings; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; -import static org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; -import static org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias.has6DigitSuffix; /** * Rollover the various .ml-anomalies result indices @@ -107,12 +98,15 @@ public void runUpdate(ClusterState latestState) { latestState.metadata().getProject().index(index).getCreationVersion() ); - if (isCompatibleIndexVersion) { + // Ensure the index name is of a format amenable to simplifying maintenance + boolean isCompatibleIndexFormat = MlIndexAndAlias.has6DigitSuffix(index); + + if (isCompatibleIndexVersion && isCompatibleIndexFormat) { continue; } // Check if this index has already been rolled over - String latestIndex = latestIndexMatchingBaseName(index, expressionResolver, latestState); + String latestIndex = MlIndexAndAlias.latestIndexMatchingBaseName(index, expressionResolver, latestState); if (index.equals(latestIndex) == false) { logger.debug("index [{}] will not be rolled over as there is a later index [{}]", index, latestIndex); @@ -171,131 +165,18 @@ private void rollAndUpdateAliases(ClusterState clusterState, String index, Actio ).andThen((l, success) -> { rollover(rolloverAlias, newIndexName, l); }).andThen((l, newIndexNameResponse) -> { - addIndexAliasesRequests(aliasRequestBuilder, index, newIndexNameResponse, clusterState); + MlIndexAndAlias.addIndexAliasesRequests(aliasRequestBuilder, index, newIndexNameResponse, clusterState); // Delete the new alias created for the rollover action aliasRequestBuilder.removeAlias(newIndexNameResponse, rolloverAlias); - updateAliases(aliasRequestBuilder, l); + MlIndexAndAlias.updateAliases(aliasRequestBuilder, l); }).addListener(listener); } private void rollover(String alias, @Nullable String newIndexName, ActionListener listener) { - client.admin() - .indices() - .rolloverIndex( - new RolloverRequest(alias, newIndexName), - ActionListener.wrap(response -> listener.onResponse(response.getNewIndex()), e -> { - if (e instanceof ResourceAlreadyExistsException alreadyExistsException) { - // The destination index already exists possibly because it has been rolled over already. - listener.onResponse(alreadyExistsException.getIndex().getName()); - } else { - listener.onFailure(e); - } - }) - ); + MlIndexAndAlias.rollover(client, new RolloverRequest(alias, newIndexName), listener); } private void createAliasForRollover(String indexName, String aliasName, ActionListener listener) { - logger.info("creating alias for rollover [{}]", aliasName); - client.admin() - .indices() - .prepareAliases( - MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT, - MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT - ) - .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(indexName).alias(aliasName).isHidden(true)) - .execute(listener); - } - - private void updateAliases(IndicesAliasesRequestBuilder request, ActionListener listener) { - request.execute(listener.delegateFailure((l, response) -> l.onResponse(Boolean.TRUE))); - } - - IndicesAliasesRequestBuilder addIndexAliasesRequests( - IndicesAliasesRequestBuilder aliasRequestBuilder, - String oldIndex, - String newIndex, - ClusterState clusterState - ) { - // Multiple jobs can share the same index each job - // has a read and write alias that needs updating - // after the rollover - var meta = clusterState.metadata().getProject().index(oldIndex); - assert meta != null; - if (meta == null) { - return aliasRequestBuilder; - } - - for (var alias : meta.getAliases().values()) { - if (isAnomaliesWriteAlias(alias.alias())) { - aliasRequestBuilder.addAliasAction( - IndicesAliasesRequest.AliasActions.add().index(newIndex).alias(alias.alias()).isHidden(true).writeIndex(true) - ); - aliasRequestBuilder.addAliasAction(IndicesAliasesRequest.AliasActions.remove().index(oldIndex).alias(alias.alias())); - } else if (isAnomaliesReadAlias(alias.alias())) { - String jobId = AnomalyDetectorsIndex.jobIdFromAlias(alias.alias()); - aliasRequestBuilder.addAliasAction( - IndicesAliasesRequest.AliasActions.add() - .index(newIndex) - .alias(alias.alias()) - .isHidden(true) - .filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), jobId)) - ); - } - } - - return aliasRequestBuilder; - } - - static boolean isAnomaliesWriteAlias(String aliasName) { - return aliasName.startsWith(AnomalyDetectorsIndexFields.RESULTS_INDEX_WRITE_PREFIX); - } - - static boolean isAnomaliesReadAlias(String aliasName) { - if (aliasName.startsWith(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX) == false) { - return false; - } - - // See {@link AnomalyDetectorsIndex#jobResultsAliasedName} - String jobIdPart = aliasName.substring(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX.length()); - // If this is a write alias it will start with a `.` character - // which is not a valid job id. - return MlStrings.isValidId(jobIdPart); - } - - /** - * Strip any suffix from the index name and find any other indices - * that match the base name. Then return the latest index from the - * matching ones. - * - * @param index The index to check - * @param expressionResolver The expression resolver - * @param latestState The latest cluster state - * @return The latest index that matches the base name of the given index - */ - static String latestIndexMatchingBaseName(String index, IndexNameExpressionResolver expressionResolver, ClusterState latestState) { - String baseIndexName = MlIndexAndAlias.has6DigitSuffix(index) - ? index.substring(0, index.length() - FIRST_INDEX_SIX_DIGIT_SUFFIX.length()) - : index; - - String[] matching = expressionResolver.concreteIndexNames( - latestState, - IndicesOptions.lenientExpandOpenHidden(), - baseIndexName + "*" - ); - - // This should never happen - assert matching.length > 0 : "No indices matching [" + baseIndexName + "*]"; - if (matching.length == 0) { - return index; - } - - // Exclude indices that start with the same base name but are a different index - // e.g. .ml-anomalies-foobar should not be included when the index name is - // .ml-anomalies-foo - String[] filtered = Arrays.stream(matching).filter(i -> { - return i.equals(index) || (has6DigitSuffix(i) && i.length() == baseIndexName.length() + FIRST_INDEX_SIX_DIGIT_SUFFIX.length()); - }).toArray(String[]::new); - - return MlIndexAndAlias.latestIndex(filtered); + MlIndexAndAlias.createAliasForRollover(logger, client, indexName, aliasName, listener); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index e7e6e713f123f..25a9109539f45 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -6,22 +6,28 @@ */ package org.elasticsearch.xpack.ml; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; +import org.elasticsearch.action.admin.indices.rollover.RolloverConditions; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequestBuilder; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.set.Sets; @@ -29,6 +35,8 @@ import org.elasticsearch.core.Releasable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.logging.LogManager; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.threadpool.Scheduler; @@ -40,10 +48,13 @@ import org.elasticsearch.xpack.core.ml.action.GetJobsAction; import org.elasticsearch.xpack.core.ml.action.ResetJobAction; import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias; import org.elasticsearch.xpack.ml.utils.TypedChainTaskExecutor; import java.time.Clock; import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Random; @@ -62,7 +73,7 @@ */ public class MlDailyMaintenanceService implements Releasable { - private static final Logger logger = LogManager.getLogger(MlDailyMaintenanceService.class); + private static final org.elasticsearch.logging.Logger logger = LogManager.getLogger(MlDailyMaintenanceService.class); private static final int MAX_TIME_OFFSET_MINUTES = 120; @@ -77,12 +88,15 @@ public class MlDailyMaintenanceService implements Releasable { */ private final Supplier schedulerProvider; + private final IndexNameExpressionResolver expressionResolver; + private final boolean isAnomalyDetectionEnabled; private final boolean isDataFrameAnalyticsEnabled; private final boolean isNlpEnabled; private volatile Scheduler.Cancellable cancellable; private volatile float deleteExpiredDataRequestsPerSecond; + private volatile ByteSizeValue rolloverMaxSize; MlDailyMaintenanceService( Settings settings, @@ -90,7 +104,8 @@ public class MlDailyMaintenanceService implements Releasable { Client client, ClusterService clusterService, MlAssignmentNotifier mlAssignmentNotifier, - Supplier scheduleProvider, + Supplier schedulerProvider, + IndexNameExpressionResolver expressionResolver, boolean isAnomalyDetectionEnabled, boolean isDataFrameAnalyticsEnabled, boolean isNlpEnabled @@ -99,8 +114,10 @@ public class MlDailyMaintenanceService implements Releasable { this.client = Objects.requireNonNull(client); this.clusterService = Objects.requireNonNull(clusterService); this.mlAssignmentNotifier = Objects.requireNonNull(mlAssignmentNotifier); - this.schedulerProvider = Objects.requireNonNull(scheduleProvider); + this.schedulerProvider = Objects.requireNonNull(schedulerProvider); + this.expressionResolver = Objects.requireNonNull(expressionResolver); this.deleteExpiredDataRequestsPerSecond = MachineLearning.NIGHTLY_MAINTENANCE_REQUESTS_PER_SECOND.get(settings); + this.rolloverMaxSize = MachineLearning.NIGHTLY_MAINTENANCE_ROLLOVER_MAX_SIZE.get(settings); this.isAnomalyDetectionEnabled = isAnomalyDetectionEnabled; this.isDataFrameAnalyticsEnabled = isDataFrameAnalyticsEnabled; this.isNlpEnabled = isNlpEnabled; @@ -113,6 +130,7 @@ public MlDailyMaintenanceService( Client client, ClusterService clusterService, MlAssignmentNotifier mlAssignmentNotifier, + IndexNameExpressionResolver expressionResolver, boolean isAnomalyDetectionEnabled, boolean isDataFrameAnalyticsEnabled, boolean isNlpEnabled @@ -124,6 +142,7 @@ public MlDailyMaintenanceService( clusterService, mlAssignmentNotifier, () -> delayToNextTime(clusterName), + expressionResolver, isAnomalyDetectionEnabled, isDataFrameAnalyticsEnabled, isNlpEnabled @@ -134,6 +153,10 @@ void setDeleteExpiredDataRequestsPerSecond(float value) { this.deleteExpiredDataRequestsPerSecond = value; } + void setRolloverMaxSize(ByteSizeValue value) { + this.rolloverMaxSize = value; + } + /** * Calculates the delay until the next time the maintenance should be triggered. * The next time is 30 minutes past midnight of the following day plus a random @@ -154,7 +177,7 @@ private static TimeValue delayToNextTime(ClusterName clusterName) { } public synchronized void start() { - logger.debug("Starting ML daily maintenance service"); + logger.info("Starting ML daily maintenance service"); scheduleNext(); } @@ -214,31 +237,38 @@ private void triggerTasks() { } private void triggerAnomalyDetectionMaintenance() { - // Step 4: Log any error that could have happened - ActionListener finalListener = ActionListener.wrap( - unused -> {}, - e -> logger.warn("An error occurred during [ML] maintenance tasks execution", e) - ); + // Step 5: Log any error that could have happened + ActionListener finalListener = ActionListener.wrap(response -> { + if (response.isAcknowledged() == false) { + logger.warn("[ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask failed"); + } else { + logger.info("[ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask succeeded"); + } + }, e -> logger.warn("An error occurred during [ML] maintenance tasks execution ", e)); + + // Step 4: Roll over results indices if necessary + ActionListener rollResultsIndicesIfNecessaryListener = ActionListener.wrap(unused -> { + triggerRollResultsIndicesIfNecessaryTask(finalListener); + }, e -> { + // Note: Steps 1-4 are independent, so continue upon errors. + triggerRollResultsIndicesIfNecessaryTask(finalListener); + }); // Step 3: Delete expired data - ActionListener deleteJobsListener = ActionListener.wrap( - unused -> triggerDeleteExpiredDataTask(finalListener), - e -> { - logger.warn("[ML] maintenance task: triggerResetJobsInStateResetWithoutResetTask failed", e); - // Note: Steps 1-3 are independent, so continue upon errors. - triggerDeleteExpiredDataTask(finalListener); - } - ); + ActionListener deleteJobsListener = ActionListener.wrap(unused -> { + triggerDeleteExpiredDataTask(rollResultsIndicesIfNecessaryListener); + }, e -> { + // Note: Steps 1-4 are independent, so continue upon errors. + triggerDeleteExpiredDataTask(rollResultsIndicesIfNecessaryListener); + }); // Step 2: Reset jobs that are in resetting state without task - ActionListener resetJobsListener = ActionListener.wrap( - unused -> triggerResetJobsInStateResetWithoutResetTask(deleteJobsListener), - e -> { - logger.warn("[ML] maintenance task: triggerDeleteJobsInStateDeletingWithoutDeletionTask failed", e); - // Note: Steps 1-3 are independent, so continue upon errors. - triggerResetJobsInStateResetWithoutResetTask(deleteJobsListener); - } - ); + ActionListener resetJobsListener = ActionListener.wrap(unused -> { + triggerResetJobsInStateResetWithoutResetTask(deleteJobsListener); + }, e -> { + // Note: Steps 1-4 are independent, so continue upon errors. + triggerResetJobsInStateResetWithoutResetTask(deleteJobsListener); + }); // Step 1: Delete jobs that are in deleting state without task triggerDeleteJobsInStateDeletingWithoutDeletionTask(resetJobsListener); @@ -252,6 +282,139 @@ private void triggerNlpMaintenance() { // Currently a NOOP } + void removeRolloverAlias( + String index, + String alias, + IndicesAliasesRequestBuilder aliasRequestBuilder, + ActionListener listener + ) { + aliasRequestBuilder.removeAlias(index, alias); + MlIndexAndAlias.updateAliases(aliasRequestBuilder, listener); + } + + private void rollAndUpdateAliases(ClusterState clusterState, String index, ActionListener listener) { + // Create an alias specifically for rolling over. + // The ml-anomalies index has aliases for each job, any + // of which could be used but that means one alias is + // treated differently. + // Using a `.` in the alias name avoids any conflicts + // as AD job Ids cannot start with `.` + String rolloverAlias = index + ".rollover_alias"; + + OriginSettingClient originSettingClient = new OriginSettingClient(client, ML_ORIGIN); + + // If the index does not end in a digit then rollover does not know + // what to name the new index so it must be specified in the request. + // Otherwise leave null and rollover will calculate the new name + String newIndexName = MlIndexAndAlias.has6DigitSuffix(index) ? null : index + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; + IndicesAliasesRequestBuilder aliasRequestBuilder = originSettingClient.admin() + .indices() + .prepareAliases( + MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT, + MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT + ); + + // 4 Clean up any dangling aliases + ActionListener aliasListener = ActionListener.wrap(r -> { listener.onResponse(r); }, e -> { + if (e instanceof IndexNotFoundException) { + // Removal of the rollover alias may have failed in the case of rollover not occurring, e.g. when the rollover conditions + // were not satisfied. + // We must still clean up the temporary alias from the original index. + // The index name is either the original one provided or the original with a suffix appended. + var indexName = MlIndexAndAlias.has6DigitSuffix(index) ? index : index + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; + + // Make sure we use a fresh IndicesAliasesRequestBuilder, the original one may have changed internal state. + IndicesAliasesRequestBuilder localAliasRequestBuilder = originSettingClient.admin() + .indices() + .prepareAliases( + MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT, + MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT + ); + + // Execute the cleanup, no need to propagate the original failure. + removeRolloverAlias(indexName, rolloverAlias, localAliasRequestBuilder, listener); + } else { + listener.onFailure(e); + } + }); + + // 3 Update aliases + ActionListener rolloverListener = ActionListener.wrap(newIndexNameResponse -> { + MlIndexAndAlias.addIndexAliasesRequests(aliasRequestBuilder, index, newIndexNameResponse, clusterState); + // On success, the rollover alias may have been moved to the new index, so we attempt to remove it from there. + // Note that the rollover request is considered "successful" even if it didn't occur due to a condition not being met + // (no exception will be thrown). In which case the attempt to remove the alias here will fail with an + // IndexNotFoundException. We handle this case with a secondary listener. + removeRolloverAlias(newIndexNameResponse, rolloverAlias, aliasRequestBuilder, aliasListener); + }, e -> { + // If rollover fails, we must still clean up the temporary alias from the original index. + // The index name is either the original one provided or the original with a suffix appended. + var indexName = MlIndexAndAlias.has6DigitSuffix(index) ? index : index + MlIndexAndAlias.FIRST_INDEX_SIX_DIGIT_SUFFIX; + // Execute the cleanup, no need to propagate the original failure. + removeRolloverAlias(indexName, rolloverAlias, aliasRequestBuilder, aliasListener); + }); + + // 2 rollover the index alias to the new index name + ActionListener getIndicesAliasesListener = ActionListener.wrap(getIndicesAliasesResponse -> { + MlIndexAndAlias.rollover( + originSettingClient, + new RolloverRequestBuilder(originSettingClient).setRolloverTarget(rolloverAlias) + .setNewIndexName(newIndexName) + .setConditions(RolloverConditions.newBuilder().addMaxIndexSizeCondition(rolloverMaxSize).build()) + .request(), + rolloverListener + ); + }, rolloverListener::onFailure); + + // 1. Create necessary aliases + MlIndexAndAlias.createAliasForRollover(logger, originSettingClient, index, rolloverAlias, getIndicesAliasesListener); + } + + // TODO make public for testing? + private void triggerRollResultsIndicesIfNecessaryTask(ActionListener finalListener) { + + List failures = new ArrayList<>(); + + ClusterState clusterState = clusterService.state(); + // list all indices starting .ml-anomalies- + // this includes the shared index and all custom results indices + String[] indices = expressionResolver.concreteIndexNames( + clusterState, + IndicesOptions.lenientExpandOpenHidden(), + AnomalyDetectorsIndex.jobResultsIndexPattern() + ); + + logger.info("[ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask"); + + for (String index : indices) { + // Check if this index has already been rolled over + String latestIndex = MlIndexAndAlias.latestIndexMatchingBaseName(index, expressionResolver, clusterState); + + if (index.equals(latestIndex) == false) { + continue; + } + + ActionListener rollAndUpdateAliasesResponseListener = finalListener.delegateFailureAndWrap( + (l, rolledAndUpdatedAliasesResponse) -> { + if (rolledAndUpdatedAliasesResponse) { + logger.info( + "Successfully completed [ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask for index [{}]", + index + ); + } else { + logger.warn( + "Unsuccessful run of [ML] maintenance task: triggerRollResultsIndicesIfNecessaryTask for index [{}]", + index + ); + } + l.onResponse(AcknowledgedResponse.TRUE); // TODO return false if operation failed for any index? + } + ); + + rollAndUpdateAliases(clusterState, index, rollAndUpdateAliasesResponseListener); + } + } + private void triggerDeleteExpiredDataTask(ActionListener finalListener) { ActionListener deleteExpiredDataActionListener = finalListener.delegateFailureAndWrap( (l, deleteExpiredDataResponse) -> { @@ -367,7 +530,6 @@ private void triggerJobsInStateWithoutMatchingTask( } chainTaskExecutor.execute(jobsActionListener); }, finalListener::onFailure); - ActionListener getJobsActionListener = ActionListener.wrap(getJobsResponse -> { Set jobsInState = getJobsResponse.getResponse().results().stream().filter(jobFilter).map(Job::getId).collect(toSet()); if (jobsInState.isEmpty()) { @@ -375,6 +537,7 @@ private void triggerJobsInStateWithoutMatchingTask( return; } jobsInStateHolder.set(jobsInState); + executeAsyncWithOrigin( client, ML_ORIGIN, @@ -383,7 +546,6 @@ private void triggerJobsInStateWithoutMatchingTask( listTasksActionListener ); }, finalListener::onFailure); - executeAsyncWithOrigin(client, ML_ORIGIN, GetJobsAction.INSTANCE, new GetJobsAction.Request("*"), getJobsActionListener); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java index 5b37b7e75b737..87efa40fb57dc 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.component.LifecycleListener; import org.elasticsearch.common.settings.Settings; @@ -67,6 +68,7 @@ public final class MlInitializationService implements ClusterStateListener { Client client, AdaptiveAllocationsScalerService adaptiveAllocationsScalerService, MlAssignmentNotifier mlAssignmentNotifier, + IndexNameExpressionResolver indexNameExpressionResolver, boolean isAnomalyDetectionEnabled, boolean isDataFrameAnalyticsEnabled, boolean isNlpEnabled @@ -81,6 +83,7 @@ public final class MlInitializationService implements ClusterStateListener { client, clusterService, mlAssignmentNotifier, + indexNameExpressionResolver, isAnomalyDetectionEnabled, isDataFrameAnalyticsEnabled, isNlpEnabled @@ -111,6 +114,12 @@ public void afterStart() { MachineLearning.NIGHTLY_MAINTENANCE_REQUESTS_PER_SECOND, mlDailyMaintenanceService::setDeleteExpiredDataRequestsPerSecond ); + clusterService.getClusterSettings() + .addSettingsUpdateConsumer( + MachineLearning.NIGHTLY_MAINTENANCE_ROLLOVER_MAX_SIZE, + mlDailyMaintenanceService::setRolloverMaxSize + ); + } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index 164a6ea8ad560..39c3fa8190459 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -240,7 +240,7 @@ public void putJob( jobBuilder.validateModelSnapshotRetentionSettingsAndSetDefaults(); validateCategorizationAnalyzerOrSetDefault(jobBuilder, analysisRegistry, minNodeVersion); - Job job = jobBuilder.build(new Date()); + Job job = jobBuilder.build(new Date(), state, indexNameExpressionResolver); ActionListener putJobListener = new ActionListener<>() { @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java index a1140aff0bdee..d77e60bfdd4af 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAnomaliesIndexUpdateTests.java @@ -51,18 +51,6 @@ public class MlAnomaliesIndexUpdateTests extends ESTestCase { - public void testIsAnomaliesWriteAlias() { - assertTrue(MlAnomaliesIndexUpdate.isAnomaliesWriteAlias(AnomalyDetectorsIndex.resultsWriteAlias("foo"))); - assertFalse(MlAnomaliesIndexUpdate.isAnomaliesWriteAlias(AnomalyDetectorsIndex.jobResultsAliasedName("foo"))); - assertFalse(MlAnomaliesIndexUpdate.isAnomaliesWriteAlias("some-index")); - } - - public void testIsAnomaliesAlias() { - assertTrue(MlAnomaliesIndexUpdate.isAnomaliesReadAlias(AnomalyDetectorsIndex.jobResultsAliasedName("foo"))); - assertFalse(MlAnomaliesIndexUpdate.isAnomaliesReadAlias(AnomalyDetectorsIndex.resultsWriteAlias("foo"))); - assertFalse(MlAnomaliesIndexUpdate.isAnomaliesReadAlias("some-index")); - } - public void testIsAbleToRun_IndicesDoNotExist() { RoutingTable.Builder routingTable = RoutingTable.builder(); var updater = new MlAnomaliesIndexUpdate(TestIndexNameExpressionResolver.newInstance(), mock(Client.class)); @@ -73,7 +61,7 @@ public void testIsAbleToRun_IndicesDoNotExist() { } public void testIsAbleToRun_IndicesHaveNoRouting() { - IndexMetadata.Builder indexMetadata = IndexMetadata.builder(".ml-anomalies-shared"); + IndexMetadata.Builder indexMetadata = IndexMetadata.builder(".ml-anomalies-shared-000001"); indexMetadata.settings( Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) @@ -114,7 +102,7 @@ public void testBuildIndexAliasesRequest() { ); var newIndex = anomaliesIndex + "-000001"; - var request = updater.addIndexAliasesRequests(aliasRequestBuilder, anomaliesIndex, newIndex, csBuilder.build()); + var request = MlIndexAndAlias.addIndexAliasesRequests(aliasRequestBuilder, anomaliesIndex, newIndex, csBuilder.build()); var actions = request.request().getAliasActions(); assertThat(actions, hasSize(6)); @@ -145,7 +133,7 @@ public void testBuildIndexAliasesRequest() { } public void testRunUpdate_UpToDateIndices() { - String indexName = ".ml-anomalies-sharedindex"; + String indexName = ".ml-anomalies-sharedindex-000001"; var jobs = List.of("job1", "job2"); IndexMetadata.Builder indexMetadata = createSharedResultsIndex(indexName, IndexVersion.current(), jobs); @@ -194,7 +182,7 @@ public void testLatestIndexMatchingBaseName_isLatest() { ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); csBuilder.metadata(metadata); - var latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName( + var latest = MlIndexAndAlias.latestIndexMatchingBaseName( ".ml-anomalies-custom-foo", TestIndexNameExpressionResolver.newInstance(), csBuilder.build() @@ -217,14 +205,14 @@ public void testLatestIndexMatchingBaseName_hasLater() { assertTrue(MlIndexAndAlias.has6DigitSuffix(".ml-anomalies-custom-foo-000002")); - var latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName( + var latest = MlIndexAndAlias.latestIndexMatchingBaseName( ".ml-anomalies-custom-foo", TestIndexNameExpressionResolver.newInstance(), state ); assertEquals(".ml-anomalies-custom-foo-000002", latest); - latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName( + latest = MlIndexAndAlias.latestIndexMatchingBaseName( ".ml-anomalies-custom-baz-000001", TestIndexNameExpressionResolver.newInstance(), state @@ -243,14 +231,14 @@ public void testLatestIndexMatchingBaseName_CollidingIndexNames() { csBuilder.metadata(metadata); var state = csBuilder.build(); - var latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName( + var latest = MlIndexAndAlias.latestIndexMatchingBaseName( ".ml-anomalies-custom-foo", TestIndexNameExpressionResolver.newInstance(), state ); assertEquals(".ml-anomalies-custom-foo", latest); - latest = MlAnomaliesIndexUpdate.latestIndexMatchingBaseName( + latest = MlIndexAndAlias.latestIndexMatchingBaseName( ".ml-anomalies-custom-foo-notthisone-000001", TestIndexNameExpressionResolver.newInstance(), state diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceServiceTests.java index 43f1bf5d1710b..1982ef8d45571 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceServiceTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskInfo; @@ -319,6 +320,7 @@ private void executeMaintenanceTriggers( clusterService, mlAssignmentNotifier, scheduleProvider, + TestIndexNameExpressionResolver.newInstance(), isAnomalyDetectionEnabled, isDataFrameAnalyticsEnabled, isNlpEnabled diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java index 80c957ecb7a09..1ee26d244679a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue; +import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.ml.inference.adaptiveallocations.AdaptiveAllocationsScalerService; @@ -75,6 +76,7 @@ public void testInitialize() { client, adaptiveAllocationsScalerService, mlAssignmentNotifier, + TestIndexNameExpressionResolver.newInstance(), true, true, true @@ -91,6 +93,7 @@ public void testInitialize_noMasterNode() { client, adaptiveAllocationsScalerService, mlAssignmentNotifier, + TestIndexNameExpressionResolver.newInstance(), true, true, true diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutorTests.java index 98169d1aa6f5b..97ceb5365c03d 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutorTests.java @@ -56,7 +56,7 @@ public void testVerifyIndicesPrimaryShardsAreActive() { cs, resolver, true, - ".ml-anomalies-shared", + ".ml-anomalies-shared-000001", AnomalyDetectorsIndex.jobStateIndexPattern(), MlMetaIndex.indexName(), MlConfigIndex.indexName() @@ -69,7 +69,7 @@ public void testVerifyIndicesPrimaryShardsAreActive() { resolver.concreteIndexNames( cs, IndicesOptions.lenientExpandOpen(), - ".ml-anomalies-shared", + ".ml-anomalies-shared-000001", AnomalyDetectorsIndex.jobStateIndexPattern(), MlMetaIndex.indexName(), MlConfigIndex.indexName() @@ -100,7 +100,7 @@ public void testVerifyIndicesPrimaryShardsAreActive() { csBuilder.build(), resolver, true, - ".ml-anomalies-shared", + ".ml-anomalies-shared-000001", AnomalyDetectorsIndex.jobStateIndexPattern(), MlMetaIndex.indexName(), MlConfigIndex.indexName() diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/custom_all_field.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/custom_all_field.yml index eefd9b937cbec..f072e67e266fb 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/custom_all_field.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/custom_all_field.yml @@ -75,7 +75,7 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.refresh: - index: [.ml-anomalies-shared] + index: [.ml-anomalies-shared-000001] --- "Test querying custom all field": @@ -148,7 +148,7 @@ setup: - do: search: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 expand_wildcards: all rest_total_hits_as_int: true body: { query: { bool: { must: [ diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/get_datafeed_stats.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/get_datafeed_stats.yml index afc0ee9db16bd..ab24331c2f65d 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/get_datafeed_stats.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/get_datafeed_stats.yml @@ -275,7 +275,7 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.delete: - index: ".ml-anomalies-shared" + index: ".ml-anomalies-shared-000001" - do: headers: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml index 52f70efcf4b04..ddbd4eeeb339a 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml @@ -69,7 +69,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-60_1" body: { @@ -85,7 +85,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-60_2" body: { @@ -101,7 +101,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-60_3" body: { @@ -116,7 +116,7 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-30_1" body: { @@ -132,7 +132,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-30_2" body: { @@ -148,7 +148,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-30_3" body: { @@ -164,7 +164,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-17_1" body: { @@ -180,7 +180,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-17_2" body: { @@ -196,7 +196,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-17_3" body: { @@ -212,7 +212,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "jobs-get-result-overall-buckets-17_4" body: { @@ -228,7 +228,7 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.refresh: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 --- "Test overall buckets given missing job": diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_stats.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_stats.yml index ab4c7311d8302..b0383b52f47ab 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_stats.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_stats.yml @@ -279,7 +279,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: job-stats-v54-bwc-test-data-counts body: { @@ -303,7 +303,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: job-stats-v54-bwc-test-model_size_stats body: { @@ -329,7 +329,7 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.refresh: - index: [.ml-anomalies-shared] + index: [.ml-anomalies-shared-000001] - do: ml.get_job_stats: @@ -366,7 +366,7 @@ setup: - do: indices.delete: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 - do: ml.get_job_stats: {} @@ -440,11 +440,11 @@ setup: - do: indices.close: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 # With the index closed the low level ML API reports a problem - do: - catch: /type=cluster_block_exception, reason=index \[.ml-anomalies-shared\] blocked by. \[FORBIDDEN\/.\/index closed\]/ + catch: /type=cluster_block_exception, reason=index \[.ml-anomalies-shared-000001\] blocked by. \[FORBIDDEN\/.\/index closed\]/ ml.get_job_stats: {} # But the high level X-Pack API returns what it can - we do this diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml index d157cc0531b65..ac2d374f3922b 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml @@ -24,7 +24,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: "new_doc" body: > { @@ -35,15 +35,15 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.refresh: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.get_field_mapping: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 fields: new_field - - match: {\.ml-anomalies-shared.mappings.new_field.mapping.new_field.type: keyword} + - match: {\.ml-anomalies-shared-000001.mappings.new_field.mapping.new_field.type: keyword} --- "Test _meta exists when two jobs share an index": @@ -67,14 +67,14 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.refresh: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.get_mapping: - index: .ml-anomalies-shared - - is_true: \.ml-anomalies-shared.mappings._meta.version + index: .ml-anomalies-shared-000001 + - is_true: \.ml-anomalies-shared-000001.mappings._meta.version - do: ml.put_job: @@ -95,11 +95,11 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.refresh: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.get_mapping: - index: .ml-anomalies-shared - - is_true: \.ml-anomalies-shared.mappings._meta.version + index: .ml-anomalies-shared-000001 + - is_true: \.ml-anomalies-shared-000001.mappings._meta.version diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/upgrade_job_snapshot.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/upgrade_job_snapshot.yml index e0281880f0f95..6e8495cda21dc 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/upgrade_job_snapshot.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/upgrade_job_snapshot.yml @@ -30,7 +30,7 @@ setup: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser Content-Type: application/json index: - index: .ml-anomalies-shared + index: .ml-anomalies-shared-000001 id: upgrade-model-snapshot_model_snapshot_1234567890 body: > { @@ -60,7 +60,7 @@ setup: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser indices.refresh: - index: [.ml-anomalies-shared,.ml-state-000001] + index: [.ml-anomalies-shared-000001,.ml-state-000001] --- "Test with unknown job id": diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index a83ad5b4f8da4..0797f375a5955 100644 --- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -45,8 +45,8 @@ public class MlHiddenIndicesFullClusterRestartIT extends AbstractXpackFullCluste Tuple.tuple(List.of(".ml-annotations-000001"), ".ml-annotations-read"), Tuple.tuple(List.of(".ml-annotations-000001"), ".ml-annotations-write"), Tuple.tuple(List.of(".ml-state", ".ml-state-000001"), ".ml-state-write"), - Tuple.tuple(List.of(".ml-anomalies-shared"), ".ml-anomalies-" + JOB_ID), - Tuple.tuple(List.of(".ml-anomalies-shared"), ".ml-anomalies-.write-" + JOB_ID) + Tuple.tuple(List.of(".ml-anomalies-shared-000001"), ".ml-anomalies-" + JOB_ID), + Tuple.tuple(List.of(".ml-anomalies-shared-000001"), ".ml-anomalies-.write-" + JOB_ID) ); public MlHiddenIndicesFullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) { diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml index 3738bd6657d07..1a51e5a4d9e1c 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml @@ -141,7 +141,7 @@ # killing the node - do: cluster.health: - index: [".ml-state", ".ml-anomalies-shared"] + index: [".ml-state-000001", ".ml-anomalies-shared*"] wait_for_status: green --- diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml index 0c0575e51db91..fb9608c9cb3a8 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml @@ -1,7 +1,7 @@ setup: - do: index: - index: .ml-state + index: .ml-state-000001 id: "dummy-document-to-make-index-creation-idempotent" body: > { @@ -9,7 +9,7 @@ setup: - do: cluster.health: - index: [".ml-state"] + index: [".ml-state-000001"] wait_for_status: green --- @@ -72,7 +72,7 @@ setup: # killing the node - do: cluster.health: - index: [".ml-state", ".ml-anomalies-shared"] + index: [".ml-state-000001", ".ml-anomalies*"] wait_for_status: green --- @@ -136,7 +136,7 @@ setup: # killing the node - do: cluster.health: - index: [".ml-state", ".ml-anomalies-shared"] + index: [".ml-state-000001", ".ml-anomalies*"] wait_for_status: green --- @@ -196,7 +196,7 @@ setup: # killing the node - do: cluster.health: - index: [".ml-state", ".ml-anomalies-shared"] + index: [".ml-state-000001", ".ml-anomalies*"] wait_for_status: green --- @@ -259,7 +259,7 @@ setup: # killing the node - do: cluster.health: - index: [".ml-state", ".ml-anomalies-shared"] + index: [".ml-state-000001", ".ml-anomalies*"] wait_for_status: green ---