diff --git a/docs/reference/data-streams/change-mappings-and-settings.asciidoc b/docs/reference/data-streams/change-mappings-and-settings.asciidoc index e07e079519377..8470a55668977 100644 --- a/docs/reference/data-streams/change-mappings-and-settings.asciidoc +++ b/docs/reference/data-streams/change-mappings-and-settings.asciidoc @@ -596,7 +596,8 @@ stream's oldest backing index. "allow_custom_routing": false, "replicated": false, "index_mode": "standard", - "rollover_on_write": false + "rollover_on_write": false, + "settings": {} } ] } diff --git a/docs/reference/data-streams/downsampling-manual.asciidoc b/docs/reference/data-streams/downsampling-manual.asciidoc index cb170a7596701..ae6e1d944ae7c 100644 --- a/docs/reference/data-streams/downsampling-manual.asciidoc +++ b/docs/reference/data-streams/downsampling-manual.asciidoc @@ -382,7 +382,8 @@ This returns: "end": "2023-07-26T13:26:42.000Z" } ] - } + }, + "settings": {} } ] } diff --git a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc index c56bf218708bc..ef07e00d24255 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc @@ -11,6 +11,14 @@ being managed by data stream lifecycle. As we'll see, {ilm-init} and data stream can co-manage a data stream; however, an index can only be managed by one system at a time. +[source,console] +-------------------------------------------------- +DELETE _data_stream/dsl-data-stream +DELETE _index_template/dsl-data-stream-template +DELETE _ilm/policy/pre-dsl-ilm-policy +-------------------------------------------------- +// TEARDOWN + [discrete] [[migrate-dsl-ilm-tldr]] ==== TL;DR @@ -141,7 +149,8 @@ and that the next generation index will also be managed by {ilm-init}: "allow_custom_routing": false, "replicated": false, "index_mode": "standard", - "rollover_on_write": false + "rollover_on_write": false, + "settings": {} } ] } @@ -283,7 +292,8 @@ GET _data_stream/dsl-data-stream "allow_custom_routing": false, "replicated": false, "index_mode": "standard", - "rollover_on_write": false + "rollover_on_write": false, + "settings": {} } ] } @@ -367,7 +377,8 @@ GET _data_stream/dsl-data-stream "allow_custom_routing": false, "replicated": false, "index_mode": "standard", - "rollover_on_write": false + "rollover_on_write": false, + "settings": {} } ] } @@ -469,7 +480,8 @@ GET _data_stream/dsl-data-stream "allow_custom_routing": false, "replicated": false, "index_mode": "standard", - "rollover_on_write": false + "rollover_on_write": false, + "settings": {} } ] } @@ -488,15 +500,3 @@ GET _data_stream/dsl-data-stream Had we removed the {ilm-init} policy from the index template when we <> it, the write index of the data stream will now be `Unmanaged` because the index wouldn't have the {ilm-init} policy configured to fallback onto. - -////////////////////////// -[source,console] --------------------------------------------------- -DELETE _data_stream/dsl-data-stream -DELETE _index_template/dsl-data-stream-template -DELETE _ilm/policy/pre-dsl-ilm-policy --------------------------------------------------- -// TEST[continued] - -////////////////////////// - diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc index a46d247d9a9a0..f2bffc92e7486 100644 --- a/docs/reference/indices/get-data-stream.asciidoc +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -330,7 +330,8 @@ The API returns the following response: "system": false, "allow_custom_routing": false, "replicated": false, - "rollover_on_write": false + "rollover_on_write": false, + "settings": {} }, { "name": "my-data-stream-two", @@ -361,7 +362,8 @@ The API returns the following response: "system": false, "allow_custom_routing": false, "replicated": false, - "rollover_on_write": false + "rollover_on_write": false, + "settings": {} } ] } diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamSettingsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamSettingsIT.java new file mode 100644 index 0000000000000..a54a32edd584e --- /dev/null +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamSettingsIT.java @@ -0,0 +1,351 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datastreams; + +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.admin.indices.readonly.AddIndexBlockRequest; +import org.elasticsearch.action.admin.indices.readonly.AddIndexBlockResponse; +import org.elasticsearch.action.admin.indices.rollover.RolloverAction; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; +import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.datastreams.CreateDataStreamAction; +import org.elasticsearch.action.datastreams.GetDataStreamSettingsAction; +import org.elasticsearch.action.datastreams.UpdateDataStreamSettingsAction; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.transport.MockTransportService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; + +public class DataStreamSettingsIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return List.of(DataStreamsPlugin.class, MockTransportService.TestPlugin.class, TestPlugin.class); + } + + public void testPutDataStreamSettings() throws Exception { + String dataStreamName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + putComposableIndexTemplate("my-template", List.of(dataStreamName), indexSettings(1, 0).build()); + final var createDataStreamRequest = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet()); + final int numberOfShards = randomIntBetween(2, 7); + final String newLifecycleName = randomAlphanumericOfLength(20).toLowerCase(Locale.ROOT); + { + List getSettingsResponses = client().execute( + GetDataStreamSettingsAction.INSTANCE, + new GetDataStreamSettingsAction.Request().indices(dataStreamName) + ).actionGet().getDataStreamSettingsResponses(); + assertThat(getSettingsResponses.size(), equalTo(1)); + assertThat(getSettingsResponses.get(0).settings(), equalTo(Settings.EMPTY)); + Settings dataStreamSettings = Settings.builder() + .put("index.number_of_shards", numberOfShards) + .put("index.lifecycle.name", newLifecycleName) + .build(); + UpdateDataStreamSettingsAction.Response putSettingsResponse = client().execute( + UpdateDataStreamSettingsAction.INSTANCE, + new UpdateDataStreamSettingsAction.Request(dataStreamSettings, TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).indices( + dataStreamName + ) + ).actionGet(); + List dataStreamSettingsResponses = putSettingsResponse + .getDataStreamSettingsResponses(); + assertThat(dataStreamSettingsResponses.size(), equalTo(1)); + UpdateDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse = dataStreamSettingsResponses.get(0); + assertThat(dataStreamSettingsResponse.dataStreamName(), equalTo(dataStreamName)); + assertThat(dataStreamSettingsResponse.dataStreamSucceeded(), equalTo(true)); + assertThat(dataStreamSettingsResponse.settings().get("index.number_of_shards"), equalTo(Integer.toString(numberOfShards))); + assertThat( + dataStreamSettingsResponse.effectiveSettings().get("index.number_of_shards"), + equalTo(Integer.toString(numberOfShards)) + ); + assertThat(dataStreamSettingsResponse.indicesSettingsResult().indexSettingErrors().size(), equalTo(0)); + assertThat(dataStreamSettingsResponse.indicesSettingsResult().appliedToDataStreamOnly().size(), equalTo(1)); + assertThat( + dataStreamSettingsResponse.indicesSettingsResult().appliedToDataStreamOnly().get(0), + equalTo("index.number_of_shards") + ); + assertThat(dataStreamSettingsResponse.indicesSettingsResult().appliedToDataStreamAndBackingIndices().size(), equalTo(1)); + assertThat( + dataStreamSettingsResponse.indicesSettingsResult().appliedToDataStreamAndBackingIndices().get(0), + equalTo("index.lifecycle.name") + ); + GetIndexResponse response = admin().indices().getIndex(new GetIndexRequest().indices(dataStreamName)).actionGet(); + Settings settings = response.getSettings().values().iterator().next(); + assertThat(settings.get("index.number_of_shards"), equalTo("1")); + assertThat(settings.get("index.lifecycle.name"), equalTo(newLifecycleName)); + getSettingsResponses = client().execute( + GetDataStreamSettingsAction.INSTANCE, + new GetDataStreamSettingsAction.Request().indices(dataStreamName) + ).actionGet().getDataStreamSettingsResponses(); + assertThat(getSettingsResponses.size(), equalTo(1)); + assertThat(getSettingsResponses.get(0).settings(), equalTo(dataStreamSettings)); + assertThat( + getSettingsResponses.get(0).effectiveSettings(), + equalTo(Settings.builder().put(dataStreamSettings).put("index.number_of_replicas", "0").build()) + ); + } + { + // Try to set an invalid value for a valid setting, and make sure the data stream is not updated + int invalidNumberOfShards = 2000; + UpdateDataStreamSettingsAction.Response putSettingsResponse = client().execute( + UpdateDataStreamSettingsAction.INSTANCE, + new UpdateDataStreamSettingsAction.Request( + Settings.builder().put("index.number_of_shards", invalidNumberOfShards).build(), + TimeValue.THIRTY_SECONDS, + TimeValue.THIRTY_SECONDS + ).indices(dataStreamName) + ).actionGet(); + List dataStreamSettingsResponses = putSettingsResponse + .getDataStreamSettingsResponses(); + assertThat(dataStreamSettingsResponses.size(), equalTo(1)); + UpdateDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse = dataStreamSettingsResponses.get(0); + assertThat(dataStreamSettingsResponse.dataStreamName(), equalTo(dataStreamName)); + assertThat(dataStreamSettingsResponse.dataStreamSucceeded(), equalTo(false)); + } + { + // Try to set an invalid setting, and make sure the data stream is not updated + UpdateDataStreamSettingsAction.Response putSettingsResponse = client().execute( + UpdateDataStreamSettingsAction.INSTANCE, + new UpdateDataStreamSettingsAction.Request( + Settings.builder() + .put("index.number_of_shards", numberOfShards) + .put("unknown.setting", randomAlphaOfLength(20)) + .build(), + TimeValue.THIRTY_SECONDS, + TimeValue.THIRTY_SECONDS + ).indices(dataStreamName) + ).actionGet(); + List dataStreamSettingsResponses = putSettingsResponse + .getDataStreamSettingsResponses(); + assertThat(dataStreamSettingsResponses.size(), equalTo(1)); + UpdateDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse = dataStreamSettingsResponses.get(0); + assertThat(dataStreamSettingsResponse.dataStreamName(), equalTo(dataStreamName)); + assertThat(dataStreamSettingsResponse.dataStreamSucceeded(), equalTo(false)); + } + } + + public void testPutMultipleDataStreamSettings() throws Exception { + List testDataStreamNames = new ArrayList<>(); + List ignoredDataStreamNames = new ArrayList<>(); + for (int i = 0; i < randomIntBetween(2, 5); i++) { + String dataStreamName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + putComposableIndexTemplate("my-template-" + i, List.of(dataStreamName), indexSettings(1, 0).build()); + final var createDataStreamRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + dataStreamName + ); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet()); + testDataStreamNames.add(dataStreamName); + } + for (int i = 0; i < randomIntBetween(2, 5); i++) { + String dataStreamName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + putComposableIndexTemplate("my-other-template-" + i, List.of(dataStreamName), indexSettings(1, 0).build()); + final var createDataStreamRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + dataStreamName + ); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet()); + ignoredDataStreamNames.add(dataStreamName); + } + final int numberOfShards = randomIntBetween(2, 7); + final String newLifecycleName = randomAlphanumericOfLength(20).toLowerCase(Locale.ROOT); + { + { + // First, a quick check that we fetch all data streams when no data stream names are given: + UpdateDataStreamSettingsAction.Response putSettingsResponse = client().execute( + UpdateDataStreamSettingsAction.INSTANCE, + new UpdateDataStreamSettingsAction.Request( + Settings.builder() + .put("index.number_of_shards", numberOfShards) + .put("index.lifecycle.name", newLifecycleName) + .build(), + TimeValue.THIRTY_SECONDS, + TimeValue.THIRTY_SECONDS + ) + ).actionGet(); + List dataStreamSettingsResponses = putSettingsResponse + .getDataStreamSettingsResponses(); + assertThat(dataStreamSettingsResponses.size(), equalTo(testDataStreamNames.size() + ignoredDataStreamNames.size())); + } + UpdateDataStreamSettingsAction.Response putSettingsResponse = client().execute( + UpdateDataStreamSettingsAction.INSTANCE, + new UpdateDataStreamSettingsAction.Request( + Settings.builder().put("index.number_of_shards", numberOfShards).put("index.lifecycle.name", newLifecycleName).build(), + TimeValue.THIRTY_SECONDS, + TimeValue.THIRTY_SECONDS + ).indices(testDataStreamNames.toArray(new String[0])) + ).actionGet(); + List dataStreamSettingsResponses = putSettingsResponse + .getDataStreamSettingsResponses(); + assertThat(dataStreamSettingsResponses.size(), equalTo(testDataStreamNames.size())); + for (int i = 0; i < testDataStreamNames.size(); i++) { + UpdateDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse = dataStreamSettingsResponses.get(i); + assertThat(dataStreamSettingsResponse.dataStreamSucceeded(), equalTo(true)); + assertThat(dataStreamSettingsResponse.settings().get("index.number_of_shards"), equalTo(Integer.toString(numberOfShards))); + assertThat( + dataStreamSettingsResponse.effectiveSettings().get("index.number_of_shards"), + equalTo(Integer.toString(numberOfShards)) + ); + assertThat(dataStreamSettingsResponse.indicesSettingsResult().indexSettingErrors().size(), equalTo(0)); + assertThat(dataStreamSettingsResponse.indicesSettingsResult().appliedToDataStreamOnly().size(), equalTo(1)); + assertThat( + dataStreamSettingsResponse.indicesSettingsResult().appliedToDataStreamOnly().get(0), + equalTo("index.number_of_shards") + ); + assertThat(dataStreamSettingsResponse.indicesSettingsResult().appliedToDataStreamAndBackingIndices().size(), equalTo(1)); + assertThat( + dataStreamSettingsResponse.indicesSettingsResult().appliedToDataStreamAndBackingIndices().get(0), + equalTo("index.lifecycle.name") + ); + GetIndexResponse response = admin().indices() + .getIndex(new GetIndexRequest().indices(dataStreamSettingsResponse.dataStreamName())) + .actionGet(); + Settings settings = response.getSettings().values().iterator().next(); + assertThat(settings.get("index.number_of_shards"), equalTo("1")); + assertThat(settings.get("index.lifecycle.name"), equalTo(newLifecycleName)); + } + List getSettingsResponses = client().execute( + GetDataStreamSettingsAction.INSTANCE, + new GetDataStreamSettingsAction.Request().indices(testDataStreamNames.toArray(new String[0])) + ).actionGet().getDataStreamSettingsResponses(); + assertThat(getSettingsResponses.size(), equalTo(testDataStreamNames.size())); + } + } + + public void testRolloverWithDataStreamSettings() throws Exception { + String dataStreamName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + putComposableIndexTemplate("my-template", List.of(dataStreamName), indexSettings(1, 0).build()); + final var createDataStreamRequest = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet()); + final int numberOfShards = randomIntBetween(2, 7); + final String newLifecycleName = randomAlphanumericOfLength(20).toLowerCase(Locale.ROOT); + client().execute( + UpdateDataStreamSettingsAction.INSTANCE, + new UpdateDataStreamSettingsAction.Request( + Settings.builder().put("index.number_of_shards", numberOfShards).put("index.lifecycle.name", newLifecycleName).build(), + TimeValue.THIRTY_SECONDS, + TimeValue.THIRTY_SECONDS + ).indices(dataStreamName) + ).actionGet(); + + RolloverResponse rolloverResponse = client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)) + .actionGet(); + assertThat(rolloverResponse.isRolledOver(), equalTo(true)); + String newIndexName = rolloverResponse.getNewIndex(); + GetIndexResponse response = admin().indices().getIndex(new GetIndexRequest().indices(newIndexName)).actionGet(); + Settings settings = response.getSettings().get(newIndexName); + assertThat(settings.get("index.number_of_shards"), equalTo(Integer.toString(numberOfShards))); + assertThat(settings.get("index.lifecycle.name"), equalTo(newLifecycleName)); + } + + public void testIndexBlock() throws Exception { + /* + * This tests that if there is a block on one index, the settings changes still go through on all the other + * indices. + */ + String dataStreamName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + putComposableIndexTemplate("my-template", List.of(dataStreamName), indexSettings(1, 0).build()); + final var createDataStreamRequest = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet()); + Set indexNames = new HashSet<>(); + for (int i = 0; i < randomIntBetween(1, 10); i++) { + RolloverResponse rolloverResponse = client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)) + .actionGet(); + assertThat(rolloverResponse.isRolledOver(), equalTo(true)); + indexNames.add(rolloverResponse.getOldIndex()); + } + String indexToBlock = randomFrom(indexNames); + PlainActionFuture plainActionFuture = new PlainActionFuture<>(); + indicesAdmin().addBlock(new AddIndexBlockRequest(IndexMetadata.APIBlock.METADATA, indexToBlock), plainActionFuture); + assertThat(plainActionFuture.get().isShardsAcknowledged(), equalTo(true)); + final int numberOfShards = randomIntBetween(2, 7); + final String newLifecycleName = randomAlphanumericOfLength(20).toLowerCase(Locale.ROOT); + { + UpdateDataStreamSettingsAction.Response putSettingsResponse = client().execute( + UpdateDataStreamSettingsAction.INSTANCE, + new UpdateDataStreamSettingsAction.Request( + Settings.builder().put("index.number_of_shards", numberOfShards).put("index.lifecycle.name", newLifecycleName).build(), + TimeValue.THIRTY_SECONDS, + TimeValue.THIRTY_SECONDS + ).indices(dataStreamName) + ).actionGet(); + List dataStreamSettingsResponses = putSettingsResponse + .getDataStreamSettingsResponses(); + UpdateDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse = dataStreamSettingsResponses.get(0); + assertThat(dataStreamSettingsResponse.dataStreamSucceeded(), equalTo(true)); + indicesAdmin().prepareUpdateSettings(indexToBlock) + .setSettings(Settings.builder().put("index.blocks.metadata", false).build()) + .get(); + GetIndexResponse response = admin().indices() + .getIndex(new GetIndexRequest().indices(dataStreamSettingsResponse.dataStreamName())) + .actionGet(); + for (Map.Entry indexAndsettings : response.getSettings().entrySet()) { + if (indexAndsettings.getKey().equals(indexToBlock)) { + assertThat(indexAndsettings.getValue().get("index.lifecycle.name"), equalTo(null)); + } else { + assertThat(indexAndsettings.getValue().get("index.lifecycle.name"), equalTo(newLifecycleName)); + } + } + } + } + + static void putComposableIndexTemplate(String id, List patterns, @Nullable Settings settings) throws IOException { + TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request(id); + request.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(patterns) + .template(Template.builder().settings(settings)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build() + ); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + } + + public static class TestPlugin extends Plugin { + /* + * index.lifecycle.name is one of the settings allowed by TransportUpdateDataStreamSettingsAction, but it is in the ilm plugin. We + * add it here so that it is available for testing. + */ + public static final Setting LIFECYCLE_SETTING = Setting.simpleString( + "index.lifecycle.name", + "", + Setting.Property.IndexScope + ); + + @Override + public List> getSettings() { + return List.of(LIFECYCLE_SETTING); + } + } + +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java index a6a55c348d41e..934f9b24269d1 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java @@ -33,6 +33,8 @@ public class DataStreamFeatures implements FeatureSpecification { "data_stream.downsample.default_aggregate_metric_fix" ); + public static final NodeFeature LOGS_STREAM_FEATURE = new NodeFeature("logs_stream"); + @Override public Map getHistoricalFeatures() { return Map.of(DATA_STREAM_LIFECYCLE, Version.V_8_11_0); @@ -51,6 +53,6 @@ public Set getFeatures() { @Override public Set getTestFeatures() { - return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX, DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX); + return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX, DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX, LOGS_STREAM_FEATURE); } } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java index f21237f564fb5..a423020570377 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java @@ -16,10 +16,12 @@ import org.elasticsearch.action.datastreams.DataStreamsStatsAction; import org.elasticsearch.action.datastreams.DeleteDataStreamAction; import org.elasticsearch.action.datastreams.GetDataStreamAction; +import org.elasticsearch.action.datastreams.GetDataStreamSettingsAction; import org.elasticsearch.action.datastreams.MigrateToDataStreamAction; import org.elasticsearch.action.datastreams.ModifyDataStreamsAction; import org.elasticsearch.action.datastreams.PromoteDataStreamAction; import org.elasticsearch.action.datastreams.PutDataStreamOptionsAction; +import org.elasticsearch.action.datastreams.UpdateDataStreamSettingsAction; import org.elasticsearch.action.datastreams.lifecycle.ExplainDataStreamLifecycleAction; import org.elasticsearch.action.datastreams.lifecycle.GetDataStreamLifecycleAction; import org.elasticsearch.action.datastreams.lifecycle.PutDataStreamLifecycleAction; @@ -37,10 +39,12 @@ import org.elasticsearch.datastreams.action.TransportCreateDataStreamAction; import org.elasticsearch.datastreams.action.TransportDataStreamsStatsAction; import org.elasticsearch.datastreams.action.TransportDeleteDataStreamAction; +import org.elasticsearch.datastreams.action.TransportGetDataStreamSettingsAction; import org.elasticsearch.datastreams.action.TransportGetDataStreamsAction; import org.elasticsearch.datastreams.action.TransportMigrateToDataStreamAction; import org.elasticsearch.datastreams.action.TransportModifyDataStreamsAction; import org.elasticsearch.datastreams.action.TransportPromoteDataStreamAction; +import org.elasticsearch.datastreams.action.TransportUpdateDataStreamSettingsAction; import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleErrorStore; import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService; import org.elasticsearch.datastreams.lifecycle.action.DeleteDataStreamLifecycleAction; @@ -68,10 +72,12 @@ import org.elasticsearch.datastreams.rest.RestCreateDataStreamAction; import org.elasticsearch.datastreams.rest.RestDataStreamsStatsAction; import org.elasticsearch.datastreams.rest.RestDeleteDataStreamAction; +import org.elasticsearch.datastreams.rest.RestGetDataStreamSettingsAction; import org.elasticsearch.datastreams.rest.RestGetDataStreamsAction; import org.elasticsearch.datastreams.rest.RestMigrateToDataStreamAction; import org.elasticsearch.datastreams.rest.RestModifyDataStreamsAction; import org.elasticsearch.datastreams.rest.RestPromoteDataStreamAction; +import org.elasticsearch.datastreams.rest.RestUpdateDataStreamSettingsAction; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.health.HealthIndicatorService; import org.elasticsearch.index.IndexSettingProvider; @@ -241,6 +247,8 @@ public Collection createComponents(PluginServices services) { actions.add(new ActionHandler<>(GetDataStreamOptionsAction.INSTANCE, TransportGetDataStreamOptionsAction.class)); actions.add(new ActionHandler<>(PutDataStreamOptionsAction.INSTANCE, TransportPutDataStreamOptionsAction.class)); actions.add(new ActionHandler<>(DeleteDataStreamOptionsAction.INSTANCE, TransportDeleteDataStreamOptionsAction.class)); + actions.add(new ActionHandler<>(GetDataStreamSettingsAction.INSTANCE, TransportGetDataStreamSettingsAction.class)); + actions.add(new ActionHandler<>(UpdateDataStreamSettingsAction.INSTANCE, TransportUpdateDataStreamSettingsAction.class)); return actions; } @@ -276,6 +284,8 @@ public List getRestHandlers( handlers.add(new RestGetDataStreamOptionsAction()); handlers.add(new RestPutDataStreamOptionsAction()); handlers.add(new RestDeleteDataStreamOptionsAction()); + handlers.add(new RestGetDataStreamSettingsAction()); + handlers.add(new RestUpdateDataStreamSettingsAction()); return handlers; } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamSettingsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamSettingsAction.java new file mode 100644 index 0000000000000..42a8a55c33d32 --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamSettingsAction.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datastreams.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.datastreams.GetDataStreamSettingsAction; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.TransportLocalClusterStateAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class TransportGetDataStreamSettingsAction extends TransportLocalClusterStateAction< + GetDataStreamSettingsAction.Request, + GetDataStreamSettingsAction.Response> { + private final IndexNameExpressionResolver indexNameExpressionResolver; + private final SettingsFilter settingsFilter; + + @Inject + public TransportGetDataStreamSettingsAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + SettingsFilter settingsFilter, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver + ) { + super( + GetDataStreamSettingsAction.NAME, + clusterService, + transportService, + actionFilters, + GetDataStreamSettingsAction.Request::localOnly, + threadPool.executor(ThreadPool.Names.MANAGEMENT) + ); + this.indexNameExpressionResolver = indexNameExpressionResolver; + this.settingsFilter = settingsFilter; + } + + @Override + protected ClusterBlockException checkBlock(GetDataStreamSettingsAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + @Override + protected void localClusterStateOperation( + Task task, + GetDataStreamSettingsAction.Request request, + ClusterState state, + ActionListener listener + ) throws Exception { + List dataStreamNames = indexNameExpressionResolver.dataStreamNames( + clusterService.state(), + IndicesOptions.DEFAULT, + request.indices() + ); + Map dataStreamMap = state.metadata().dataStreams(); + List responseList = new ArrayList<>(dataStreamNames.size()); + for (String dataStreamName : dataStreamNames) { + DataStream dataStream = dataStreamMap.get(dataStreamName); + Settings settings = settingsFilter.filter(dataStream.getSettings()); + Settings effectiveSettings = settingsFilter.filter(dataStream.getEffectiveSettings(state.metadata())); + responseList.add(new GetDataStreamSettingsAction.DataStreamSettingsResponse(dataStreamName, settings, effectiveSettings)); + } + listener.onResponse(new GetDataStreamSettingsAction.Response(responseList)); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java index 0759feef71053..e7279d47a156a 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java @@ -238,7 +238,7 @@ static GetDataStreamAction.Response innerOperation( } else { indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false); if (indexTemplate != null) { - Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate); + Settings settings = dataStream.getEffectiveSettings(state.metadata()); ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME); if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) { indexMode = resolveMode( @@ -246,7 +246,7 @@ static GetDataStreamAction.Response innerOperation( indexSettingProviders, dataStream, settings, - state.metadata().templatesV2().get(indexTemplate) + dataStream.getEffectiveIndexTemplate(state.metadata()) ); } indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java new file mode 100644 index 0000000000000..3e6aaf105bb7e --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java @@ -0,0 +1,334 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsClusterStateUpdateRequest; +import org.elasticsearch.action.datastreams.UpdateDataStreamSettingsAction; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.CountDownActionListener; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetadataDataStreamsService; +import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.common.settings.Settings.EMPTY; +import static org.elasticsearch.common.settings.Settings.builder; + +public class TransportUpdateDataStreamSettingsAction extends TransportMasterNodeAction< + UpdateDataStreamSettingsAction.Request, + UpdateDataStreamSettingsAction.Response> { + private static final Logger logger = LogManager.getLogger(TransportUpdateDataStreamSettingsAction.class); + private static final Set APPLY_TO_BACKING_INDICES = Set.of("index.lifecycle.name", IndexSettings.PREFER_ILM); + private static final Set APPLY_TO_DATA_STREAM_ONLY = Set.of("index.number_of_shards"); + private final MetadataDataStreamsService metadataDataStreamsService; + private final MetadataUpdateSettingsService updateSettingsService; + private final IndexNameExpressionResolver indexNameExpressionResolver; + private final SystemIndices systemIndices; + private final SettingsFilter settingsFilter; + + @Inject + public TransportUpdateDataStreamSettingsAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + MetadataDataStreamsService metadataDataStreamsService, + MetadataUpdateSettingsService updateSettingsService, + IndexNameExpressionResolver indexNameExpressionResolver, + SystemIndices systemIndices, + SettingsFilter settingsFilter + ) { + super( + UpdateDataStreamSettingsAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + UpdateDataStreamSettingsAction.Request::new, + UpdateDataStreamSettingsAction.Response::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.metadataDataStreamsService = metadataDataStreamsService; + this.updateSettingsService = updateSettingsService; + this.indexNameExpressionResolver = indexNameExpressionResolver; + this.systemIndices = systemIndices; + this.settingsFilter = settingsFilter; + } + + @Override + protected void masterOperation( + Task task, + UpdateDataStreamSettingsAction.Request request, + ClusterState state, + ActionListener listener + ) throws Exception { + List dataStreamNames = indexNameExpressionResolver.dataStreamNames( + clusterService.state(), + IndicesOptions.DEFAULT, + request.indices() + ); + List dataStreamSettingsResponse = new ArrayList<>(); + CountDownActionListener countDownListener = new CountDownActionListener( + dataStreamNames.size() + 1, + listener.delegateFailure( + (responseActionListener, unused) -> responseActionListener.onResponse( + new UpdateDataStreamSettingsAction.Response(dataStreamSettingsResponse) + ) + ) + ); + countDownListener.onResponse(null); + for (String dataStreamName : dataStreamNames) { + updateSingleDataStream( + dataStreamName, + request.getSettings(), + request.isDryRun(), + request.masterNodeTimeout(), + request.ackTimeout(), + ActionListener.wrap(dataStreamResponse -> { + dataStreamSettingsResponse.add(dataStreamResponse); + countDownListener.onResponse(null); + }, e -> { + dataStreamSettingsResponse.add( + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( + dataStreamName, + false, + e.getMessage(), + EMPTY, + EMPTY, + UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult.EMPTY + ) + ); + countDownListener.onResponse(null); + }) + ); + } + } + + private void updateSingleDataStream( + String dataStreamName, + Settings settingsOverrides, + boolean dryRun, + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + ActionListener listener + ) { + logger.debug("updating settings for {}", dataStreamName); + Set settingsToReject = new HashSet<>(); + for (String settingName : settingsOverrides.keySet()) { + if (APPLY_TO_BACKING_INDICES.contains(settingName) == false && APPLY_TO_DATA_STREAM_ONLY.contains(settingName) == false) { + settingsToReject.add(settingName); + } + } + if (settingsToReject.isEmpty() == false) { + listener.onResponse( + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( + dataStreamName, + false, + Strings.format("Cannot set the following settings on a data stream: [%s]", String.join(",", settingsToReject)), + EMPTY, + EMPTY, + UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult.EMPTY + ) + ); + return; + } + + if (systemIndices.isSystemDataStream(dataStreamName)) { + listener.onResponse( + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( + dataStreamName, + false, + "Cannot update a system data stream", + EMPTY, + EMPTY, + UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult.EMPTY + ) + ); + return; + } + metadataDataStreamsService.updateSettings( + masterNodeTimeout, + ackTimeout, + dataStreamName, + settingsOverrides, + dryRun, + listener.delegateFailure((dataStreamSettingsResponseActionListener, dataStream) -> { + if (dataStream != null) { + updateSettingsOnIndices(dataStream, settingsOverrides, dryRun, masterNodeTimeout, ackTimeout, listener); + } else { + dataStreamSettingsResponseActionListener.onResponse( + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( + dataStreamName, + false, + "Updating settings not accepted for unknown reasons", + EMPTY, + EMPTY, + UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult.EMPTY + ) + ); + } + }) + ); + } + + private void updateSettingsOnIndices( + DataStream dataStream, + Settings requestSettings, + boolean dryRun, + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + ActionListener listener + ) { + Map settingsToApply = new HashMap<>(); + List appliedToDataStreamOnly = new ArrayList<>(); + List appliedToDataStreamAndBackingIndices = new ArrayList<>(); + for (String settingName : requestSettings.keySet()) { + if (APPLY_TO_BACKING_INDICES.contains(settingName)) { + settingsToApply.put(settingName, requestSettings.get(settingName)); + appliedToDataStreamAndBackingIndices.add(settingName); + } else if (APPLY_TO_DATA_STREAM_ONLY.contains(settingName)) { + appliedToDataStreamOnly.add(settingName); + } + } + final List concreteIndices = dataStream.getIndices(); + final List indexSettingErrors = new ArrayList<>(); + + CountDownActionListener indexCountDownListener = new CountDownActionListener( + concreteIndices.size() + 1, + listener.delegateFailure( + (dataStreamSettingsResponseActionListener, unused) -> dataStreamSettingsResponseActionListener.onResponse( + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( + dataStream.getName(), + true, + null, + settingsFilter.filter(dataStream.getSettings()), + settingsFilter.filter(dataStream.getEffectiveSettings(clusterService.state().metadata())), + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult( + appliedToDataStreamOnly, + appliedToDataStreamAndBackingIndices, + indexSettingErrors + ) + ) + ) + ) + ); + + indexCountDownListener.onResponse(null); // handles the case where there were zero indices + Settings applyToIndexSettings = builder().loadFromMap(settingsToApply).build(); + for (Index index : concreteIndices) { + updateSettingsOnSingleIndex( + index, + applyToIndexSettings, + dryRun, + masterNodeTimeout, + ackTimeout, + indexCountDownListener.delegateFailure((listener1, indexSettingError) -> { + if (indexSettingError != null) { + indexSettingErrors.add(indexSettingError); + } + listener1.onResponse(null); + }) + ); + } + } + + private void updateSettingsOnSingleIndex( + Index index, + Settings requestSettings, + boolean dryRun, + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + ActionListener listener + ) { + if (requestSettings.isEmpty()) { + listener.onResponse(null); + } else { + ClusterBlockException blockException = clusterService.state() + .blocks() + .indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, new String[] { index.getName() }); + if (blockException != null) { + listener.onResponse( + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndexSettingError( + index.getName(), + blockException.getMessage() + ) + ); + return; + } + if (dryRun) { + /* + * This is as far as we go with dry run mode. We get the benefit of having checked that all the indices that will be touced + * are not blocked, but there is no value in going beyond this. So just respond to the listener and move on. + */ + listener.onResponse(null); + } else { + updateSettingsService.updateSettings( + new UpdateSettingsClusterStateUpdateRequest( + masterNodeTimeout, + ackTimeout, + requestSettings, + UpdateSettingsClusterStateUpdateRequest.OnExisting.OVERWRITE, + UpdateSettingsClusterStateUpdateRequest.OnStaticSetting.REOPEN_INDICES, + index + ), + ActionListener.wrap(response -> { + UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndexSettingError error; + if (response.isAcknowledged() == false) { + error = new UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndexSettingError( + index.getName(), + "Updating settings not acknowledged for unknown reason" + ); + } else { + error = null; + } + listener.onResponse(error); + }, + e -> listener.onResponse( + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndexSettingError(index.getName(), e.getMessage()) + ) + ) + ); + } + } + + } + + @Override + protected ClusterBlockException checkBlock(UpdateDataStreamSettingsAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamSettingsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamSettingsAction.java new file mode 100644 index 0000000000000..5752c77e19e6f --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamSettingsAction.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datastreams.rest; + +import org.elasticsearch.action.datastreams.GetDataStreamSettingsAction; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestCancellableNodeClient; +import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +@ServerlessScope(Scope.PUBLIC) +public class RestGetDataStreamSettingsAction extends BaseRestHandler { + @Override + public String getName() { + return "get_data_stream_settings_action"; + } + + @Override + public List routes() { + return List.of(new Route(GET, "/_data_stream/{name}/_settings")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + GetDataStreamSettingsAction.Request getDataStreamRequest = new GetDataStreamSettingsAction.Request().indices( + Strings.splitStringByCommaToArray(request.param("name")) + ); + return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute( + GetDataStreamSettingsAction.INSTANCE, + getDataStreamRequest, + new RestRefCountedChunkedToXContentListener<>(channel) + ); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestUpdateDataStreamSettingsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestUpdateDataStreamSettingsAction.java new file mode 100644 index 0000000000000..d21efb247a231 --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestUpdateDataStreamSettingsAction.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.rest; + +import org.elasticsearch.action.datastreams.UpdateDataStreamSettingsAction; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestUtils; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestCancellableNodeClient; +import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.PUT; + +@ServerlessScope(Scope.PUBLIC) +public class RestUpdateDataStreamSettingsAction extends BaseRestHandler { + + @Override + public String getName() { + return "update_data_stream_settings_action"; + } + + @Override + public List routes() { + return List.of(new Route(PUT, "/_data_stream/{name}/_settings")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + Settings settings; + try (XContentParser parser = request.contentParser()) { + settings = Settings.fromXContent(parser); + } + boolean dryRun = request.paramAsBoolean("dry_run", false); + UpdateDataStreamSettingsAction.Request putDataStreamRequest = new UpdateDataStreamSettingsAction.Request( + settings, + dryRun, + RestUtils.getMasterNodeTimeout(request), + RestUtils.getAckTimeout(request) + ).indices(Strings.splitStringByCommaToArray(request.param("name"))); + return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute( + UpdateDataStreamSettingsAction.INSTANCE, + putDataStreamRequest, + new RestRefCountedChunkedToXContentListener<>(channel) + ); + } +} diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java index 5da05e3e0fb68..2435cf1ced8ae 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java @@ -248,6 +248,7 @@ public void testUpdateTimeSeriesTemporalOneBadDataStream() { ds2Indices, 2, ds2.getMetadata(), + ds2.getSettings(), ds2.isHidden(), ds2.isReplicated(), ds2.isSystem(), diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java index bcb3236d19c87..dbf8a31747b3d 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java @@ -11,13 +11,17 @@ import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings; import org.elasticsearch.cluster.metadata.DataStreamTestHelper; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; @@ -26,15 +30,21 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettingProviders; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESTestCase; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; +import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName; +import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createIndexMetadata; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.getClusterStateWithDataStreams; import static org.elasticsearch.test.LambdaMatchers.transformedItemsMatch; import static org.elasticsearch.test.LambdaMatchers.transformedMatch; @@ -312,9 +322,9 @@ public void testGetTimeSeriesMixedDataStream() { null ); - var name1 = DataStream.getDefaultBackingIndexName("ds-1", 1, instant.toEpochMilli()); - var name2 = DataStream.getDefaultBackingIndexName("ds-1", 2, instant.toEpochMilli()); - var name3 = DataStream.getDefaultBackingIndexName("ds-1", 3, twoHoursAgo.toEpochMilli()); + var name1 = getDefaultBackingIndexName("ds-1", 1, instant.toEpochMilli()); + var name2 = getDefaultBackingIndexName("ds-1", 2, instant.toEpochMilli()); + var name3 = getDefaultBackingIndexName("ds-1", 3, twoHoursAgo.toEpochMilli()); assertThat( response.getDataStreams(), contains( @@ -534,4 +544,159 @@ public void testProvidersAffectMode() { equalTo("standard") ); } + + public void testGetEffectiveSettingsTemplateOnlySettings() { + // Set a lifecycle only in the template, and make sure that is in the response: + GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); + final String templatePolicy = "templatePolicy"; + final String templateIndexMode = IndexMode.LOOKUP.getName(); + + ClusterState state = getClusterStateWithDataStreamWithSettings( + Settings.builder() + .put(IndexMetadata.LIFECYCLE_NAME, templatePolicy) + .put(IndexSettings.MODE.getKey(), templateIndexMode) + .build(), + Settings.EMPTY, + Settings.EMPTY + ); + + GetDataStreamAction.Response response = TransportGetDataStreamsAction.innerOperation( + state, + req, + resolver, + systemIndices, + ClusterSettings.createBuiltInClusterSettings(), + dataStreamGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), + null + ); + assertNotNull(response.getDataStreams()); + assertThat(response.getDataStreams().size(), equalTo(1)); + assertThat(response.getDataStreams().get(0).getIlmPolicy(), equalTo(templatePolicy)); + assertThat(response.getDataStreams().get(0).getIndexModeName(), equalTo(templateIndexMode)); + } + + public void testGetEffectiveSettingsComponentTemplateOnlySettings() { + // Set a lifecycle only in the template, and make sure that is in the response: + GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); + final String templatePolicy = "templatePolicy"; + final String templateIndexMode = IndexMode.LOOKUP.getName(); + + ClusterState state = getClusterStateWithDataStreamWithSettings( + Settings.EMPTY, + Settings.builder() + .put(IndexMetadata.LIFECYCLE_NAME, templatePolicy) + .put(IndexSettings.MODE.getKey(), templateIndexMode) + .build(), + Settings.EMPTY + ); + + GetDataStreamAction.Response response = TransportGetDataStreamsAction.innerOperation( + state, + req, + resolver, + systemIndices, + ClusterSettings.createBuiltInClusterSettings(), + dataStreamGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), + null + ); + assertNotNull(response.getDataStreams()); + assertThat(response.getDataStreams().size(), equalTo(1)); + assertThat(response.getDataStreams().get(0).getIlmPolicy(), equalTo(templatePolicy)); + assertThat(response.getDataStreams().get(0).getIndexModeName(), equalTo(templateIndexMode)); + } + + public void testGetEffectiveSettings() { + GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); + final String templatePolicy = "templatePolicy"; + final String templateIndexMode = IndexMode.LOOKUP.getName(); + final String dataStreamPolicy = "dataStreamPolicy"; + final String dataStreamIndexMode = IndexMode.LOGSDB.getName(); + // Now set a lifecycle in both the template and the data stream, and make sure the response has the data stream one: + ClusterState state = getClusterStateWithDataStreamWithSettings( + Settings.builder() + .put(IndexMetadata.LIFECYCLE_NAME, templatePolicy) + .put(IndexSettings.MODE.getKey(), templateIndexMode) + .build(), + Settings.builder() + .put(IndexMetadata.LIFECYCLE_NAME, templatePolicy) + .put(IndexSettings.MODE.getKey(), templateIndexMode) + .build(), + Settings.builder() + .put(IndexMetadata.LIFECYCLE_NAME, dataStreamPolicy) + .put(IndexSettings.MODE.getKey(), dataStreamIndexMode) + .build() + ); + GetDataStreamAction.Response response = TransportGetDataStreamsAction.innerOperation( + state, + req, + resolver, + systemIndices, + ClusterSettings.createBuiltInClusterSettings(), + dataStreamGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), + null + ); + assertNotNull(response.getDataStreams()); + assertThat(response.getDataStreams().size(), equalTo(1)); + assertThat(response.getDataStreams().get(0).getIlmPolicy(), equalTo(dataStreamPolicy)); + assertThat(response.getDataStreams().get(0).getIndexModeName(), equalTo(dataStreamIndexMode)); + } + + private static ClusterState getClusterStateWithDataStreamWithSettings( + Settings templateSettings, + Settings componentTemplateSettings, + Settings dataStreamSettings + ) { + String dataStreamName = "data-stream-1"; + int numberOfBackingIndices = randomIntBetween(1, 5); + long currentTime = System.currentTimeMillis(); + int replicas = 0; + boolean replicated = false; + Metadata.Builder builder = Metadata.builder(); + builder.put( + "template_1", + ComposableIndexTemplate.builder() + .indexPatterns(List.of("*")) + .template(Template.builder().settings(templateSettings)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .componentTemplates(List.of("component_template_1")) + .build() + ); + ComponentTemplate componentTemplate = new ComponentTemplate( + Template.builder().settings(componentTemplateSettings).build(), + null, + null, + null + ); + builder.componentTemplates(Map.of("component_template_1", componentTemplate)); + + List backingIndices = new ArrayList<>(); + for (int backingIndexNumber = 1; backingIndexNumber <= numberOfBackingIndices; backingIndexNumber++) { + backingIndices.add( + createIndexMetadata( + getDefaultBackingIndexName(dataStreamName, backingIndexNumber, currentTime), + true, + templateSettings, + replicas + ) + ); + } + List allIndices = new ArrayList<>(backingIndices); + + DataStream ds = DataStream.builder( + dataStreamName, + backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()) + ).setGeneration(numberOfBackingIndices).setSettings(dataStreamSettings).setReplicated(replicated).build(); + builder.put(ds); + + for (IndexMetadata index : allIndices) { + builder.put(index, false); + } + return ClusterState.builder(new ClusterName("_name")).metadata(builder.build()).build(); + } } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml new file mode 100644 index 0000000000000..8dd15628fa130 --- /dev/null +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml @@ -0,0 +1,301 @@ +setup: + - skip: + features: allowed_warnings + +--- +"Test single data stream": + - requires: + cluster_features: [ "logs_stream" ] + reason: requires setting 'logs_stream' to get or set data stream settings + - do: + allowed_warnings: + - "index template [my-template] has index patterns [my-data-stream-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [ my-data-stream-* ] + data_stream: { } + template: + settings: + number_of_replicas: 0 + lifecycle.name: my-policy + + - do: + indices.create_data_stream: + name: my-data-stream-1 + + - do: + cluster.health: + index: "my-data-stream-1" + wait_for_status: green + + - do: + indices.get_data_stream_settings: + name: my-data-stream-1 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.settings: {} } + - match: { data_streams.0.effective_settings.index.number_of_shards: null } + - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" } + - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-policy" } + + - do: + indices.get_data_stream: + name: my-data-stream-1 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.settings: {} } + - match: { data_streams.0.effective_settings: null } + + - do: + indices.put_data_stream_settings: + name: my-data-stream-1 + body: + index: + number_of_shards: 2 + lifecycle: + name: my-new-policy + prefer_ilm: true + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.applied_to_data_stream: true } + - match: { data_streams.0.index_settings_results.applied_to_data_stream_only: [index.number_of_shards]} + - length: { data_streams.0.index_settings_results.applied_to_data_stream_and_backing_indices: 2 } + - match: { data_streams.0.settings.index.number_of_shards: "2" } + - match: { data_streams.0.settings.index.lifecycle.name: "my-new-policy" } + - match: { data_streams.0.settings.index.lifecycle.prefer_ilm: "true" } + - match: { data_streams.0.effective_settings.index.number_of_shards: "2" } + - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" } + - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-new-policy" } + - match: { data_streams.0.effective_settings.index.lifecycle.prefer_ilm: "true" } + + - do: + indices.rollover: + alias: "my-data-stream-1" + + - do: + cluster.health: + index: "my-data-stream-1" + wait_for_status: green + + - do: + indices.get_data_stream_settings: + name: my-data-stream-1 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.settings.index.number_of_shards: "2" } + - match: { data_streams.0.effective_settings.index.number_of_shards: "2" } + - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" } + - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-new-policy" } + - match: { data_streams.0.effective_settings.index.lifecycle.prefer_ilm: "true" } + + - do: + indices.get_data_stream: + name: my-data-stream-1 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.settings.index.number_of_shards: "2" } + - match: { data_streams.0.settings.index.lifecycle.name: "my-new-policy" } + - match: { data_streams.0.settings.index.lifecycle.prefer_ilm: "true" } + - match: { data_streams.0.effective_settings: null } + + - do: + indices.get_data_stream: + name: my-data-stream-1 + - set: { data_streams.0.indices.0.index_name: idx0name } + - set: { data_streams.0.indices.1.index_name: idx1name } + + - do: + indices.get_settings: + index: my-data-stream-1 + - match: { .$idx0name.settings.index.number_of_shards: "1" } + - match: { .$idx0name.settings.index.lifecycle.name: "my-new-policy" } + - match: { .$idx1name.settings.index.number_of_shards: "2" } + - match: { .$idx1name.settings.index.lifecycle.name: "my-new-policy" } + - match: { .$idx1name.settings.index.lifecycle.prefer_ilm: "true" } + +--- +"Test multiple data streams": + - requires: + cluster_features: [ "logs_stream" ] + reason: requires setting 'logs_stream' to get or set data stream settings + - do: + allowed_warnings: + - "index template [my-template] has index patterns [my-data-stream-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [ my-data-stream-* ] + data_stream: { } + template: + settings: + number_of_replicas: 0 + lifecycle.name: my-policy + + - do: + indices.create_data_stream: + name: my-data-stream-1 + + - do: + indices.create_data_stream: + name: my-data-stream-2 + + - do: + indices.create_data_stream: + name: my-data-stream-3 + + - do: + cluster.health: + index: "my-data-stream-1,my-data-stream-2,my-data-stream-3" + wait_for_status: green + + - do: + indices.get_data_stream_settings: + name: "*" + - length: { data_streams: 3 } + + - do: + indices.get_data_stream_settings: + name: "my-data-stream-1,my-data-stream-2" + - length: { data_streams: 2 } + + - do: + indices.put_data_stream_settings: + name: my-data-stream-1,my-data-stream-2 + body: + index: + number_of_shards: 2 + lifecycle.name: my-new-policy + - length: { data_streams: 2 } + +--- +"Test invalid settings": + - requires: + cluster_features: [ "logs_stream" ] + reason: requires setting 'logs_stream' to get or set data stream settings + - do: + allowed_warnings: + - "index template [my-template] has index patterns [my-data-stream-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [ my-data-stream-* ] + data_stream: { } + template: + settings: + number_of_replicas: 0 + lifecycle.name: my-policy + + - do: + indices.create_data_stream: + name: my-data-stream-1 + + - do: + cluster.health: + index: "my-data-stream-1" + wait_for_status: green + + - do: + indices.put_data_stream_settings: + name: my-data-stream-1 + body: + index: + fake_setting: 1234 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.applied_to_data_stream: false } + - match: { data_streams.0.error: "Cannot set the following settings on a data stream: [index.fake_setting]" } + +--- +"Test dry run": + - requires: + cluster_features: [ "logs_stream" ] + reason: requires setting 'logs_stream' to get or set data stream settings + - do: + allowed_warnings: + - "index template [my-template] has index patterns [my-data-stream-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [ my-data-stream-* ] + data_stream: { } + template: + settings: + number_of_replicas: 0 + lifecycle.name: my-policy + + - do: + indices.create_data_stream: + name: my-data-stream-1 + + - do: + cluster.health: + index: "my-data-stream-1" + wait_for_status: green + + - do: + indices.get_data_stream_settings: + name: my-data-stream-1 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.settings: {} } + - match: { data_streams.0.effective_settings.index.number_of_shards: null } + - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" } + - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-policy" } + + - do: + indices.get_data_stream: + name: my-data-stream-1 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.settings: {} } + - match: { data_streams.0.effective_settings: null } + + - do: + indices.put_data_stream_settings: + name: my-data-stream-1 + dry_run: true + body: + index: + number_of_shards: 2 + lifecycle.name: my-new-policy + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.applied_to_data_stream: true } + - match: { data_streams.0.index_settings_results.applied_to_data_stream_only: [index.number_of_shards]} + - match: { data_streams.0.index_settings_results.applied_to_data_stream_and_backing_indices: [index.lifecycle.name] } + - match: { data_streams.0.settings.index.number_of_shards: "2" } + - match: { data_streams.0.settings.index.lifecycle.name: "my-new-policy" } + - match: { data_streams.0.effective_settings.index.number_of_shards: "2" } + - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" } + - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-new-policy" } + + - do: + indices.rollover: + alias: "my-data-stream-1" + + - do: + cluster.health: + index: "my-data-stream-1" + wait_for_status: green + + - do: + indices.get_data_stream_settings: + name: my-data-stream-1 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.settings: {} } + - match: { data_streams.0.effective_settings.index.number_of_shards: null } + - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" } + - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-policy" } + + - do: + indices.get_data_stream: + name: my-data-stream-1 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.settings: {} } + - match: { data_streams.0.effective_settings: null } + + - do: + indices.get_data_stream: + name: my-data-stream-1 + - set: { data_streams.0.indices.0.index_name: idx0name } + - set: { data_streams.0.indices.1.index_name: idx1name } + + - do: + indices.get_settings: + index: my-data-stream-1 + - match: { .$idx0name.settings.index.number_of_shards: "1" } + - match: { .$idx0name.settings.index.lifecycle.name: "my-policy" } + - match: { .$idx1name.settings.index.number_of_shards: "1" } + - match: { .$idx1name.settings.index.lifecycle.name: "my-policy" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream_settings.json new file mode 100644 index 0000000000000..15271890a3b79 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream_settings.json @@ -0,0 +1,35 @@ +{ + "indices.get_data_stream_settings":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html", + "description":"Gets a data stream's settings" + }, + "stability":"stable", + "visibility": "public", + "headers":{ + "accept": [ "application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_data_stream/{name}/_settings", + "methods":[ + "GET" + ], + "parts":{ + "name":{ + "type":"string", + "description":"Comma-separated list of data streams or data stream patterns" + } + } + } + ] + }, + "params":{ + "master_timeout":{ + "type":"time", + "description":"Period to wait for a connection to the master node" + } + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_stream_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_stream_settings.json new file mode 100644 index 0000000000000..b358c2d3c864f --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_stream_settings.json @@ -0,0 +1,48 @@ +{ + "indices.put_data_stream_settings":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html", + "description":"Updates a data stream's settings" + }, + "stability":"stable", + "visibility": "public", + "headers":{ + "accept": [ "application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_data_stream/{name}/_settings", + "methods":[ + "PUT" + ], + "parts":{ + "name":{ + "type":"string", + "description":"Comma-separated list of data streams or data stream patterns" + } + } + } + ] + }, + "params":{ + "dry_run":{ + "type":"boolean", + "description":"Whether this request should only be a dry run rather than actually applying settings", + "default":false + }, + "timeout":{ + "type":"time", + "description":"Period to wait for a response" + }, + "master_timeout":{ + "type":"time", + "description":"Period to wait for a connection to the master node" + } + }, + "body":{ + "description":"The data stream settings to be updated", + "required":true + } + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceIT.java new file mode 100644 index 0000000000000..adbd764c00adc --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceIT.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.support.master.ShardsAcknowledgedResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.Locale; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.equalTo; + +public class MetadataCreateIndexServiceIT extends ESIntegTestCase { + + public void testRequestTemplateIsRespected() throws InterruptedException { + /* + * This test passes a template in the CreateIndexClusterStateUpdateRequest, and makes sure that the settings from that template + * are used when creating the index. + */ + MetadataCreateIndexService metadataCreateIndexService = internalCluster().getCurrentMasterNodeInstance( + MetadataCreateIndexService.class + ); + final String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + final int numberOfReplicas = randomIntBetween(1, 7); + CreateIndexClusterStateUpdateRequest request = new CreateIndexClusterStateUpdateRequest( + "testRequestTemplateIsRespected", + indexName, + randomAlphaOfLength(20) + ); + request.setMatchingTemplate( + ComposableIndexTemplate.builder() + .template(Template.builder().settings(Settings.builder().put("index.number_of_replicas", numberOfReplicas))) + .build() + ); + final CountDownLatch listenerCalledLatch = new CountDownLatch(1); + ActionListener listener = new ActionListener<>() { + @Override + public void onResponse(ShardsAcknowledgedResponse shardsAcknowledgedResponse) { + listenerCalledLatch.countDown(); + } + + @Override + public void onFailure(Exception e) { + logger.error(e); + listenerCalledLatch.countDown(); + } + }; + + metadataCreateIndexService.createIndex( + TimeValue.THIRTY_SECONDS, + TimeValue.THIRTY_SECONDS, + TimeValue.THIRTY_SECONDS, + request, + listener + ); + listenerCalledLatch.await(10, TimeUnit.SECONDS); + GetIndexResponse response = admin().indices().getIndex(new GetIndexRequest().indices(indexName)).actionGet(); + Settings settings = response.getSettings().get(indexName); + assertThat(settings.get("index.number_of_replicas"), equalTo(Integer.toString(numberOfReplicas))); + } +} diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 174723c890d6f..5cd3420b581c8 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -241,6 +241,7 @@ static TransportVersion def(int id) { public static final TransportVersion ML_INFERENCE_ELASTIC_RERANK_ADDED_8_19 = def(8_841_0_48); public static final TransportVersion NONE_CHUNKING_STRATEGY_8_19 = def(8_841_0_49); public static final TransportVersion IDP_CUSTOM_SAML_ATTRIBUTES_ALLOW_LIST_8_19 = def(8_841_0_50); + public static final TransportVersion SETTINGS_IN_DATA_STREAMS_8_19 = def(8_841_0_51); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index b5e30701c75a7..07acefae6e9d3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -63,7 +63,6 @@ import static org.elasticsearch.cluster.metadata.IndexAbstraction.Type.ALIAS; import static org.elasticsearch.cluster.metadata.IndexAbstraction.Type.DATA_STREAM; -import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.lookupTemplateForDataStream; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template; import static org.elasticsearch.cluster.routing.allocation.allocator.AllocationActionListener.rerouteCompletionIsNotRequired; @@ -312,7 +311,7 @@ private RolloverResult rolloverDataStream( final SystemDataStreamDescriptor systemDataStreamDescriptor; if (dataStream.isSystem() == false) { systemDataStreamDescriptor = null; - templateV2 = lookupTemplateForDataStream(dataStreamName, metadata); + templateV2 = dataStream.getEffectiveIndexTemplate(currentState.metadata()); } else { systemDataStreamDescriptor = systemIndices.findMatchingDataStreamDescriptor(dataStreamName); if (systemDataStreamDescriptor == null) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 3906969403cf9..f50fd1932dc62 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -261,8 +261,8 @@ public static Template resolveTemplate( : indexName; List mappings = MetadataCreateIndexService.collectV2Mappings( null, // empty request mapping as the user can't specify any explicit mappings via the simulate api - simulatedState, - matchingTemplate, + simulatedState.metadata(), + template, xContentRegistry, simulatedIndexName ); diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index 172903727ffb5..efcce1f9d5bfb 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -212,6 +212,7 @@ public static class DataStreamInfo implements SimpleDiffable, To public static final ParseField STATUS_FIELD = new ParseField("status"); public static final ParseField INDEX_TEMPLATE_FIELD = new ParseField("template"); + public static final ParseField SETTINGS_FIELD = new ParseField("settings"); public static final ParseField PREFER_ILM = new ParseField("prefer_ilm"); public static final ParseField MANAGED_BY = new ParseField("managed_by"); public static final ParseField NEXT_GENERATION_INDEX_MANAGED_BY = new ParseField("next_generation_managed_by"); @@ -434,6 +435,9 @@ public XContentBuilder toXContent( builder.endArray(); builder.endObject(); } + builder.startObject(SETTINGS_FIELD.getPreferredName()); + dataStream.getSettings().toXContent(builder, params); + builder.endObject(); builder.startObject(DataStream.FAILURE_STORE_FIELD.getPreferredName()); builder.field(FAILURE_STORE_ENABLED.getPreferredName(), failureStoreEffectivelyEnabled); diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamSettingsAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamSettingsAction.java new file mode 100644 index 0000000000000..84e74c3ecab84 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamSettingsAction.java @@ -0,0 +1,143 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.datastreams; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ChunkedToXContentObject; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +public class GetDataStreamSettingsAction extends ActionType { + public static final String NAME = "indices:monitor/data_stream/settings/get"; + public static final GetDataStreamSettingsAction INSTANCE = new GetDataStreamSettingsAction(); + + public GetDataStreamSettingsAction() { + super(NAME); + } + + public static class Request extends ActionRequest implements IndicesRequest.Replaceable { + private String[] dataStreamNames; + + public Request() { + super(); + } + + public static Request localOnly(StreamInput ignored) { + return TransportAction.localOnly(); + } + + @Override + public GetDataStreamSettingsAction.Request indices(String... dataStreamNames) { + this.dataStreamNames = dataStreamNames; + return this; + } + + @Override + public boolean includeDataStreams() { + return true; + } + + @Override + public String[] indices() { + return dataStreamNames; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED; + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, "", parentTaskId, headers); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GetDataStreamSettingsAction.Request request = (GetDataStreamSettingsAction.Request) o; + return Arrays.equals(dataStreamNames, request.dataStreamNames); + } + + @Override + public int hashCode() { + return Arrays.hashCode(dataStreamNames); + } + } + + public static class Response extends ActionResponse implements ChunkedToXContentObject { + private final List dataStreamSettingsResponses; + + public Response(List dataStreamSettingsResponses) { + this.dataStreamSettingsResponses = dataStreamSettingsResponses; + } + + public List getDataStreamSettingsResponses() { + return dataStreamSettingsResponses; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + assert false : "This ought to never be called because this action only runs locally"; + } + + @Override + public Iterator toXContentChunked(ToXContent.Params params) { + return Iterators.concat( + Iterators.single((builder, params1) -> builder.startObject().startArray("data_streams")), + dataStreamSettingsResponses.stream().map(dataStreamSettingsResponse -> (ToXContent) dataStreamSettingsResponse).iterator(), + Iterators.single((builder, params1) -> builder.endArray().endObject()) + ); + } + } + + public record DataStreamSettingsResponse(String dataStreamName, Settings settings, Settings effectiveSettings) implements ToXContent { + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("name", dataStreamName); + builder.startObject("settings"); + settings.toXContent(builder, params); + builder.endObject(); + builder.startObject("effective_settings"); + effectiveSettings.toXContent(builder, params); + builder.endObject(); + builder.endObject(); + return builder; + } + } + +} diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsAction.java new file mode 100644 index 0000000000000..a21ba63c7ed8b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsAction.java @@ -0,0 +1,275 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.datastreams; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ChunkedToXContentObject; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class UpdateDataStreamSettingsAction extends ActionType { + + public static final String NAME = "indices:admin/data_stream/settings/update"; + public static final UpdateDataStreamSettingsAction INSTANCE = new UpdateDataStreamSettingsAction(); + + public UpdateDataStreamSettingsAction() { + super(NAME); + } + + public static class Request extends AcknowledgedRequest implements IndicesRequest.Replaceable { + private final Settings settings; + private String[] dataStreamNames = Strings.EMPTY_ARRAY; + private final boolean dryRun; + + public Request(Settings settings, TimeValue masterNodeTimeout, TimeValue ackTimeout) { + this(settings, false, masterNodeTimeout, ackTimeout); + } + + public Request(Settings settings, boolean dryRun, TimeValue masterNodeTimeout, TimeValue ackTimeout) { + super(masterNodeTimeout, ackTimeout); + this.settings = settings; + this.dryRun = dryRun; + } + + @Override + public Request indices(String... dataStreamNames) { + this.dataStreamNames = dataStreamNames; + return this; + } + + public Settings getSettings() { + return settings; + } + + public boolean isDryRun() { + return dryRun; + } + + @Override + public boolean includeDataStreams() { + return true; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.dataStreamNames = in.readStringArray(); + this.settings = Settings.readSettingsFromStream(in); + this.dryRun = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeStringArray(dataStreamNames); + settings.writeTo(out); + out.writeBoolean(dryRun); + } + + @Override + public String[] indices() { + return dataStreamNames; + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED; + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, "", parentTaskId, headers); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Arrays.equals(dataStreamNames, request.dataStreamNames) + && settings.equals(request.settings) + && dryRun == request.dryRun + && Objects.equals(masterNodeTimeout(), request.masterNodeTimeout()) + && Objects.equals(ackTimeout(), request.ackTimeout()); + } + + @Override + public int hashCode() { + return Objects.hash(Arrays.hashCode(dataStreamNames), settings, dryRun, masterNodeTimeout(), ackTimeout()); + } + + } + + public static class Response extends ActionResponse implements ChunkedToXContentObject { + private final List dataStreamSettingsResponses; + + public Response(List dataStreamSettingsResponses) { + this.dataStreamSettingsResponses = dataStreamSettingsResponses; + } + + public Response(StreamInput in) throws IOException { + this(in.readCollectionAsList(DataStreamSettingsResponse::new)); + } + + public List getDataStreamSettingsResponses() { + return dataStreamSettingsResponses; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(dataStreamSettingsResponses, (out1, value) -> value.writeTo(out1)); + } + + @Override + public Iterator toXContentChunked(ToXContent.Params params) { + return Iterators.concat( + Iterators.single((builder, params1) -> builder.startObject().startArray("data_streams")), + dataStreamSettingsResponses.stream().map(dataStreamSettingsResponse -> (ToXContent) dataStreamSettingsResponse).iterator(), + Iterators.single((builder, params1) -> builder.endArray().endObject()) + ); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return Objects.equals(dataStreamSettingsResponses, response.dataStreamSettingsResponses); + } + + @Override + public int hashCode() { + return Objects.hash(dataStreamSettingsResponses); + } + } + + public record DataStreamSettingsResponse( + String dataStreamName, + boolean dataStreamSucceeded, + String dataStreamErrorMessage, + Settings settings, + Settings effectiveSettings, + IndicesSettingsResult indicesSettingsResult + ) implements ToXContent, Writeable { + + public DataStreamSettingsResponse(StreamInput in) throws IOException { + this( + in.readString(), + in.readBoolean(), + in.readOptionalString(), + Settings.readSettingsFromStream(in), + Settings.readSettingsFromStream(in), + new IndicesSettingsResult(in) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(dataStreamName); + out.writeBoolean(dataStreamSucceeded); + out.writeOptionalString(dataStreamErrorMessage); + settings.writeTo(out); + effectiveSettings.writeTo(out); + indicesSettingsResult.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("name", dataStreamName); + builder.field("applied_to_data_stream", dataStreamSucceeded); + if (dataStreamErrorMessage != null) { + builder.field("error", dataStreamErrorMessage); + } + builder.startObject("settings"); + settings.toXContent(builder, params); + builder.endObject(); + builder.startObject("effective_settings"); + effectiveSettings.toXContent(builder, params); + builder.endObject(); + builder.startObject("index_settings_results"); + indicesSettingsResult.toXContent(builder, params); + builder.endObject(); + builder.endObject(); + return builder; + } + + public record IndicesSettingsResult( + List appliedToDataStreamOnly, + List appliedToDataStreamAndBackingIndices, + List indexSettingErrors + ) implements ToXContent, Writeable { + + public static final IndicesSettingsResult EMPTY = new IndicesSettingsResult(List.of(), List.of(), List.of()); + + public IndicesSettingsResult(StreamInput in) throws IOException { + this(in.readStringCollectionAsList(), in.readStringCollectionAsList(), in.readCollectionAsList(IndexSettingError::new)); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("applied_to_data_stream_only", appliedToDataStreamOnly); + builder.field("applied_to_data_stream_and_backing_indices", appliedToDataStreamAndBackingIndices); + if (indexSettingErrors.isEmpty() == false) { + builder.field("errors", indexSettingErrors); + } + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeStringCollection(appliedToDataStreamOnly); + out.writeStringCollection(appliedToDataStreamAndBackingIndices); + out.writeCollection(indexSettingErrors, (out1, value) -> value.writeTo(out1)); + } + } + + public record IndexSettingError(String indexName, String errorMessage) implements ToXContent, Writeable { + public IndexSettingError(StreamInput in) throws IOException { + this(in.readString(), in.readString()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(indexName); + out.writeString(errorMessage); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("index", indexName); + builder.field("error", errorMessage); + builder.endObject(); + return builder; + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index c82af6cf0dd8d..3c29b70720887 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.MapperService; @@ -309,6 +310,34 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla return builder; } + /* + * Merges the given settings into the settings in this ComposableIndexTemplate. Any null values in the + * given settings are removed from the settings in the returned ComposableIndexTemplate. If this + * ComposableIndexTemplate has no settings, the given settings are the only ones in the returned template + * (with any null values removed). If this ComposableIndexTemplate has no template, an empty template with + * those settings is created. If the given settings are empty, this ComposableIndexTemplate is just + * returned unchanged. This method never changes this object. + */ + public ComposableIndexTemplate mergeSettings(Settings settings) { + Objects.requireNonNull(settings); + if (Settings.EMPTY.equals(settings)) { + return this; + } + ComposableIndexTemplate.Builder mergedIndexTemplateBuilder = this.toBuilder(); + Template.Builder mergedTemplateBuilder; + Settings templateSettings; + if (this.template() == null) { + mergedTemplateBuilder = Template.builder(); + templateSettings = null; + } else { + mergedTemplateBuilder = Template.builder(this.template()); + templateSettings = this.template().settings(); + } + mergedTemplateBuilder.settings(templateSettings == null ? settings : templateSettings.merge(settings)); + mergedIndexTemplateBuilder.template(mergedTemplateBuilder); + return mergedIndexTemplateBuilder.build(); + } + @Override public int hashCode() { return Objects.hash( diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 774df9f710f7b..e905bddcbd20e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -28,8 +28,10 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateFormatters; +import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; @@ -64,6 +66,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.lookupTemplateForDataStream; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; import static org.elasticsearch.index.IndexSettings.LIFECYCLE_ORIGINATION_DATE; import static org.elasticsearch.index.IndexSettings.PREFER_ILM_SETTING; @@ -73,6 +76,7 @@ public final class DataStream implements SimpleDiffable, ToXContentO private static final Logger LOGGER = LogManager.getLogger(DataStream.class); public static final NodeFeature DATA_STREAM_FAILURE_STORE_FEATURE = new NodeFeature("data_stream.failure_store"); + public static final boolean LOGS_STREAM_FEATURE_FLAG = new FeatureFlag("logs_stream").isEnabled(); public static final TransportVersion ADDED_FAILURE_STORE_TRANSPORT_VERSION = TransportVersions.V_8_12_0; public static final TransportVersion ADDED_AUTO_SHARDING_EVENT_VERSION = TransportVersions.V_8_14_0; public static final TransportVersion ADD_DATA_STREAM_OPTIONS_VERSION = TransportVersions.V_8_16_0; @@ -105,6 +109,7 @@ public final class DataStream implements SimpleDiffable, ToXContentO private final long generation; @Nullable private final Map metadata; + private final Settings settings; private final boolean hidden; private final boolean replicated; private final boolean system; @@ -137,8 +142,46 @@ public DataStream( ) { this( name, + indices, generation, metadata, + Settings.EMPTY, + hidden, + replicated, + system, + allowCustomRouting, + indexMode, + lifecycle, + dataStreamOptions, + failureIndices, + rolloverOnWrite, + autoShardingEvent + ); + } + + // visible for testing + public DataStream( + String name, + List indices, + long generation, + Map metadata, + Settings settings, + boolean hidden, + boolean replicated, + boolean system, + boolean allowCustomRouting, + IndexMode indexMode, + DataStreamLifecycle lifecycle, + @Nullable DataStreamOptions dataStreamOptions, + List failureIndices, + boolean rolloverOnWrite, + @Nullable DataStreamAutoShardingEvent autoShardingEvent + ) { + this( + name, + generation, + metadata, + settings, hidden, replicated, system, @@ -156,6 +199,7 @@ public DataStream( String name, long generation, Map metadata, + Settings settings, boolean hidden, boolean replicated, boolean system, @@ -170,6 +214,7 @@ public DataStream( this.name = name; this.generation = generation; this.metadata = metadata; + this.settings = Objects.requireNonNull(settings); // The following assert is commented out, because system data streams created before 8.1 are not hidden, // but should be updated to hidden by 8.18/8.19 (SystemIndexMetadataUpgradeService) // assert system == false || hidden; // system indices must be hidden @@ -226,10 +271,17 @@ public static DataStream read(StreamInput in) throws IOException { // is still behind a feature flag in previous version we use the default value instead of explicitly disabling it. dataStreamOptions = failureStoreEnabled ? DataStreamOptions.FAILURE_STORE_ENABLED : null; } + final Settings settings; + if (in.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS_8_19)) { + settings = Settings.readSettingsFromStream(in); + } else { + settings = Settings.EMPTY; + } return new DataStream( name, generation, metadata, + settings, hidden, replicated, system, @@ -320,6 +372,20 @@ public boolean rolloverOnWrite() { return backingIndices.rolloverOnWrite; } + public ComposableIndexTemplate getEffectiveIndexTemplate(Metadata metadata) { + return getMatchingIndexTemplate(metadata).mergeSettings(settings); + } + + public Settings getEffectiveSettings(Metadata metadata) { + ComposableIndexTemplate template = getMatchingIndexTemplate(metadata); + Settings templateSettings = MetadataIndexTemplateService.resolveSettings(template, metadata.componentTemplates()); + return templateSettings.merge(settings); + } + + private ComposableIndexTemplate getMatchingIndexTemplate(Metadata metadata) { + return lookupTemplateForDataStream(name, metadata); + } + /** * We define that a data stream is considered internal either if it is a system index or if * its name starts with a dot. @@ -431,6 +497,10 @@ public Map getMetadata() { return metadata; } + public Settings getSettings() { + return settings; + } + @Override public boolean isHidden() { return hidden; @@ -1250,6 +1320,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(DataStream.ADD_DATA_STREAM_OPTIONS_VERSION)) { out.writeOptionalWriteable(dataStreamOptions.isEmpty() ? null : dataStreamOptions); } + if (out.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS_8_19)) { + settings.writeTo(out); + } } public static final ParseField NAME_FIELD = new ParseField("name"); @@ -1271,6 +1344,7 @@ public void writeTo(StreamOutput out) throws IOException { public static final ParseField FAILURE_ROLLOVER_ON_WRITE_FIELD = new ParseField("failure_rollover_on_write"); public static final ParseField FAILURE_AUTO_SHARDING_FIELD = new ParseField("failure_auto_sharding"); public static final ParseField DATA_STREAM_OPTIONS_FIELD = new ParseField("options"); + public static final ParseField SETTINGS_FIELD = new ParseField("settings"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -1279,6 +1353,7 @@ public void writeTo(StreamOutput out) throws IOException { (String) args[0], (Long) args[2], (Map) args[3], + args[17] == null ? Settings.EMPTY : (Settings) args[17], args[4] != null && (boolean) args[4], args[5] != null && (boolean) args[5], args[6] != null && (boolean) args[6], @@ -1349,6 +1424,7 @@ public void writeTo(StreamOutput out) throws IOException { (p, c) -> DataStreamOptions.fromXContent(p), DATA_STREAM_OPTIONS_FIELD ); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> Settings.fromXContent(p), SETTINGS_FIELD); } public static DataStream fromXContent(XContentParser parser) throws IOException { @@ -1410,6 +1486,9 @@ public XContentBuilder toXContent( backingIndices.autoShardingEvent.toXContent(builder, params); builder.endObject(); } + builder.startObject(SETTINGS_FIELD.getPreferredName()); + this.settings.toXContent(builder, params); + builder.endObject(); builder.endObject(); return builder; } @@ -1422,6 +1501,7 @@ public boolean equals(Object o) { return name.equals(that.name) && generation == that.generation && Objects.equals(metadata, that.metadata) + && Objects.equals(settings, that.settings) && hidden == that.hidden && system == that.system && replicated == that.replicated @@ -1439,6 +1519,7 @@ public int hashCode() { name, generation, metadata, + settings, hidden, system, replicated, @@ -1751,6 +1832,7 @@ public static class Builder { private long generation = 1; @Nullable private Map metadata = null; + private Settings settings = Settings.EMPTY; private boolean hidden = false; private boolean replicated = false; private boolean system = false; @@ -1778,6 +1860,7 @@ private Builder(DataStream dataStream) { name = dataStream.name; generation = dataStream.generation; metadata = dataStream.metadata; + settings = dataStream.settings; hidden = dataStream.hidden; replicated = dataStream.replicated; system = dataStream.system; @@ -1809,6 +1892,11 @@ public Builder setMetadata(Map metadata) { return this; } + public Builder setSettings(Settings settings) { + this.settings = settings; + return this; + } + public Builder setHidden(boolean hidden) { this.hidden = hidden; return this; @@ -1869,6 +1957,7 @@ public DataStream build() { name, generation, metadata, + settings, hidden, replicated, system, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 175992aa90737..196593ccb55cf 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -321,6 +321,7 @@ static ClusterState createDataStream( dataStreamName, initialGeneration, template.metadata() != null ? Map.copyOf(template.metadata()) : null, + Settings.EMPTY, hidden, false, isSystem, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 5c8f4451ecf4a..ebe4ae300e59e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -410,6 +410,18 @@ public ClusterState applyCreateIndexRequest( ? IndexMetadata.INDEX_HIDDEN_SETTING.get(request.settings()) : null; + ComposableIndexTemplate templateFromRequest = request.matchingTemplate(); + if (templateFromRequest != null) { + return applyCreateIndexRequestWithV2Template( + currentState, + request, + silent, + templateFromRequest, + metadataTransformer, + rerouteListener + ); + } + // Check to see if a v2 template matched final String v2Template = MetadataIndexTemplateService.findV2Template( currentState.metadata(), @@ -657,7 +669,8 @@ private ClusterState applyCreateIndexRequestWithV2Template( ) throws Exception { logger.debug("applying create index request using composable template [{}]", templateName); - ComposableIndexTemplate template = currentState.getMetadata().templatesV2().get(templateName); + final Metadata metadata = currentState.getMetadata(); + ComposableIndexTemplate template = metadata.templatesV2().get(templateName); final boolean isDataStream = template.getDataStreamTemplate() != null; if (isDataStream && request.dataStreamName() == null) { throw new IllegalArgumentException( @@ -669,18 +682,34 @@ private ClusterState applyCreateIndexRequestWithV2Template( + "use create data stream api instead" ); } + return applyCreateIndexRequestWithV2Template(currentState, request, silent, template, metadataTransformer, rerouteListener); + } + + private ClusterState applyCreateIndexRequestWithV2Template( + final ClusterState currentState, + final CreateIndexClusterStateUpdateRequest request, + final boolean silent, + final ComposableIndexTemplate template, + final BiConsumer metadataTransformer, + final ActionListener rerouteListener + ) throws Exception { + + final Metadata metadata = currentState.getMetadata(); + final RoutingTable routingTable = currentState.routingTable(); + + final boolean isDataStream = template.getDataStreamTemplate() != null; final List mappings = collectV2Mappings( request.mappings(), - currentState, - templateName, + metadata, + template, xContentRegistry, request.index() ); final Settings aggregatedIndexSettings = aggregateIndexSettings( currentState, request, - resolveSettings(currentState.metadata(), templateName), + resolveSettings(template, metadata.componentTemplates()), mappings, null, settings, @@ -702,7 +731,7 @@ private ClusterState applyCreateIndexRequestWithV2Template( request.index(), // data stream aliases are created separately in MetadataCreateDataStreamService::createDataStream isDataStream ? Set.of() : request.aliases(), - isDataStream ? List.of() : MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName), + isDataStream ? List.of() : MetadataIndexTemplateService.resolveAliases(currentState.metadata(), template), currentState.metadata(), xContentRegistry, // the context is used ony for validation so it's fine to pass fake values for the shard id and the current timestamp @@ -710,7 +739,7 @@ private ClusterState applyCreateIndexRequestWithV2Template( IndexService.dateMathExpressionResolverAt(request.getNameResolvedAt()), systemIndices::isSystemName ), - Collections.singletonList(templateName), + Collections.singletonList("provided in request"), metadataTransformer, rerouteListener ); @@ -837,17 +866,6 @@ private static List collectSystemV2Mappings( return collectV2Mappings(null, templateMappings, xContentRegistry); } - public static List collectV2Mappings( - @Nullable final String requestMappings, - final ClusterState currentState, - final String templateName, - final NamedXContentRegistry xContentRegistry, - final String indexName - ) throws Exception { - List templateMappings = MetadataIndexTemplateService.collectMappings(currentState, templateName, indexName); - return collectV2Mappings(requestMappings, templateMappings, xContentRegistry); - } - private static List collectV2Mappings( @Nullable final String requestMappings, final List templateMappings, @@ -864,6 +882,21 @@ private static List collectV2Mappings( return result; } + public static List collectV2Mappings( + @Nullable final String requestMappings, + final Metadata metadata, + final ComposableIndexTemplate template, + final NamedXContentRegistry xContentRegistry, + final String indexName + ) throws Exception { + List templateMappings = MetadataIndexTemplateService.collectMappings( + template, + metadata.componentTemplates(), + indexName + ); + return collectV2Mappings(requestMappings, templateMappings, xContentRegistry); + } + private ClusterState applyCreateIndexRequestWithExistingMetadata( final ClusterState currentState, final CreateIndexClusterStateUpdateRequest request, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java index 7d103b667f2bb..55275c0a26daf 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java @@ -9,6 +9,7 @@ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.datastreams.ModifyDataStreamsAction; @@ -39,10 +40,13 @@ import java.io.IOException; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.lookupTemplateForDataStream; + /** * Handles data stream modification requests. */ @@ -54,6 +58,7 @@ public class MetadataDataStreamsService { private final MasterServiceTaskQueue updateLifecycleTaskQueue; private final MasterServiceTaskQueue setRolloverOnWriteTaskQueue; private final MasterServiceTaskQueue updateOptionsTaskQueue; + private final MasterServiceTaskQueue updateSettingsTaskQueue; public MetadataDataStreamsService( ClusterService clusterService, @@ -116,6 +121,31 @@ public Tuple executeTask( } }; this.updateOptionsTaskQueue = clusterService.createTaskQueue("modify-data-stream-options", Priority.NORMAL, updateOptionsExecutor); + ClusterStateTaskExecutor updateSettingsExecutor = new SimpleBatchedAckListenerTaskExecutor<>() { + + @Override + public Tuple executeTask( + UpdateSettingsTask updateSettingsTask, + ClusterState clusterState + ) throws Exception { + DataStream dataStream = createDataStreamForUpdatedDataStreamSettings( + updateSettingsTask.dataStreamName, + updateSettingsTask.settingsOverrides, + clusterState + ); + Metadata projectMetadata = clusterState.metadata(); + Metadata.Builder builder = Metadata.builder(projectMetadata); + builder.removeDataStream(updateSettingsTask.dataStreamName); + builder.put(dataStream); + ClusterState updatedClusterState = ClusterState.builder(clusterState).metadata(builder).build(); + return new Tuple<>(updatedClusterState, updateSettingsTask); + } + }; + this.updateSettingsTaskQueue = clusterService.createTaskQueue( + "update-data-stream-settings", + Priority.NORMAL, + updateSettingsExecutor + ); } public void modifyDataStream(final ModifyDataStreamsAction.Request request, final ActionListener listener) { @@ -339,6 +369,71 @@ public static ClusterState setRolloverOnWrite( return ClusterState.builder(currentState).metadata(builder.build()).build(); } + public void updateSettings( + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + String dataStreamName, + Settings settingsOverrides, + boolean dryRun, + ActionListener listener + ) { + if (dryRun) { + /* + * If this is a dry run, we'll do the settings validation and apply the changes to the data stream locally, but we won't run + * the task that actually updates the cluster state. + */ + try { + DataStream updatedDataStream = createDataStreamForUpdatedDataStreamSettings( + dataStreamName, + settingsOverrides, + clusterService.state() + ); + listener.onResponse(updatedDataStream); + } catch (Exception e) { + listener.onFailure(e); + } + } else { + UpdateSettingsTask updateSettingsTask = new UpdateSettingsTask( + dataStreamName, + settingsOverrides, + clusterService, + ackTimeout, + listener + ); + updateSettingsTaskQueue.submitTask("updating settings on data stream", updateSettingsTask, masterNodeTimeout); + } + } + + /* + * This method validates that the settings won't cause any validation problems with existing templates. If successful, a copy of the + * data stream is returned with the new settings applied. + */ + private DataStream createDataStreamForUpdatedDataStreamSettings( + String dataStreamName, + Settings settingsOverrides, + ClusterState clusterState + ) throws Exception { + Metadata metadata = clusterState.metadata(); + Map dataStreamMap = metadata.dataStreams(); + DataStream dataStream = dataStreamMap.get(dataStreamName); + Settings existingSettings = dataStream.getSettings(); + + Template.Builder templateBuilder = Template.builder(); + Settings.Builder mergedSettingsBuilder = Settings.builder().put(existingSettings).put(settingsOverrides); + Settings mergedSettings = mergedSettingsBuilder.build(); + + final ComposableIndexTemplate template = lookupTemplateForDataStream(dataStreamName, metadata); + ComposableIndexTemplate mergedTemplate = template.mergeSettings(mergedSettings); + MetadataIndexTemplateService.validateTemplate( + mergedTemplate.template().settings(), + mergedTemplate.template().mappings(), + indicesService + ); + + templateBuilder.settings(mergedSettingsBuilder); + return dataStream.copy().setSettings(mergedSettings).build(); + } + private static void addBackingIndex( Metadata metadata, Metadata.Builder builder, @@ -563,4 +658,27 @@ public boolean targetFailureStore() { return targetFailureStore; } } + + static class UpdateSettingsTask extends AckedBatchedClusterStateUpdateTask { + private final String dataStreamName; + private final Settings settingsOverrides; + + UpdateSettingsTask( + String dataStreamName, + Settings settingsOverrides, + ClusterService clusterService, + TimeValue ackTimeout, + ActionListener listener + ) { + super(ackTimeout, listener.safeMap(response -> { + if (response.isAcknowledged()) { + return clusterService.state().metadata().dataStreams().get(dataStreamName); + } else { + throw new ElasticsearchException("Updating settings not accepted for unknown reasons"); + } + })); + this.dataStreamName = dataStreamName; + this.settingsOverrides = settingsOverrides; + } + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 1afa350658396..42082be4169b9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1854,8 +1854,7 @@ private static void validateCompositeTemplate( }); } - private static void validateTemplate(Settings validateSettings, CompressedXContent mappings, IndicesService indicesService) - throws Exception { + static void validateTemplate(Settings validateSettings, CompressedXContent mappings, IndicesService indicesService) throws Exception { // Hard to validate settings if they're non-existent, so used empty ones if none were provided Settings settings = validateSettings; if (settings == null) { diff --git a/server/src/main/java/org/elasticsearch/common/settings/Settings.java b/server/src/main/java/org/elasticsearch/common/settings/Settings.java index 784fb5ebc63d3..4153702bf1af8 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -876,6 +876,27 @@ public Set keySet() { return newKeySet; } + /* + * This method merges the given newSettings into this Settings, returning either a new Settings object or this if the newSettings are + * empty. If any values are null in newSettings, those keys are removed from the returned object. + */ + public Settings merge(Settings newSettings) { + Objects.requireNonNull(newSettings); + if (Settings.EMPTY.equals(newSettings)) { + return this; + } + Settings.Builder builder = Settings.builder().put(this); + for (String key : newSettings.keySet()) { + String rawValue = newSettings.get(key); + if (rawValue == null) { + builder.remove(key); + } else { + builder.put(key, rawValue); + } + } + return builder.build(); + } + /** * A builder allowing to put different settings and then {@link #build()} an immutable * settings implementation. Use {@link Settings#builder()} in order to diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 729b1734fea22..6550d18fb1adc 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -63,6 +63,8 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -638,7 +640,14 @@ public void testRolloverClusterStateForDataStream() throws Exception { false ); long after = testThreadPool.absoluteTimeInMillis(); - + Settings rolledOverIndexSettings = rolloverResult.clusterState() + .metadata() + .index(rolloverResult.rolloverIndexName()) + .getSettings(); + Set rolledOverIndexSettingNames = rolledOverIndexSettings.keySet(); + for (String settingName : dataStream.getEffectiveSettings(clusterState.metadata()).keySet()) { + assertTrue(rolledOverIndexSettingNames.contains(settingName)); + } String newIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1); assertEquals(sourceIndexName, rolloverResult.sourceIndexName()); assertEquals(newIndexName, rolloverResult.rolloverIndexName()); @@ -708,7 +717,14 @@ public void testRolloverClusterStateForDataStreamFailureStore() throws Exception true ); long after = testThreadPool.absoluteTimeInMillis(); - + Settings rolledOverIndexSettings = rolloverResult.clusterState() + .metadata() + .index(rolloverResult.rolloverIndexName()) + .getSettings(); + Set rolledOverIndexSettingNames = rolledOverIndexSettings.keySet(); + for (String settingName : dataStream.getSettings().keySet()) { + assertFalse(rolledOverIndexSettingNames.contains(settingName)); + } var epochMillis = System.currentTimeMillis(); String sourceIndexName = DataStream.getDefaultFailureStoreName(dataStream.getName(), dataStream.getGeneration(), epochMillis); String newIndexName = DataStream.getDefaultFailureStoreName(dataStream.getName(), dataStream.getGeneration() + 1, epochMillis); diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java index 0614356943d3a..9aa1a2f2a2c07 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; @@ -27,6 +28,7 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.cluster.metadata.ComponentTemplateTests.randomSettings; import static org.hamcrest.Matchers.equalTo; public class GetDataStreamActionTests extends ESTestCase { @@ -60,6 +62,8 @@ public void testDataStreamInfoToXContent() throws IOException { assertThat(lifecycleResult.get("data_retention"), equalTo(configuredRetention.getStringRep())); assertThat(lifecycleResult.get("effective_retention"), equalTo(globalMaxRetention.getStringRep())); assertThat(lifecycleResult.get("retention_determined_by"), equalTo("max_global_retention")); + Map> settingsMap = (Map>) resultMap.get("settings"); + assertThat(Settings.builder().loadFromMap(settingsMap).build(), equalTo(dataStreamInfo.getDataStream().getSettings())); } } @@ -100,6 +104,7 @@ private static GetDataStreamAction.Response.DataStreamInfo newDataStreamInfo(boo private static DataStream newDataStreamInstance(boolean isSystem, TimeValue retention) { List indices = List.of(new Index(randomAlphaOfLength(10), randomAlphaOfLength(10))); DataStreamLifecycle lifecycle = DataStreamLifecycle.createDataLifecycle(true, retention, null); + Settings settings = randomSettings(); return DataStream.builder(randomAlphaOfLength(50), indices) .setGeneration(randomLongBetween(1, 1000)) .setMetadata(Map.of()) @@ -107,6 +112,7 @@ private static DataStream newDataStreamInstance(boolean isSystem, TimeValue rete .setHidden(isSystem) .setReplicated(randomBoolean()) .setLifecycle(lifecycle) + .setSettings(settings) .build(); } } diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamSettingsActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamSettingsActionTests.java new file mode 100644 index 0000000000000..c6b2b699be00b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamSettingsActionTests.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.datastreams; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; +import static org.hamcrest.Matchers.equalTo; + +public class GetDataStreamSettingsActionTests extends ESTestCase { + + public void testResponseToXContentEmpty() throws IOException { + List responseList = new ArrayList<>(); + GetDataStreamSettingsAction.Response response = new GetDataStreamSettingsAction.Response(responseList); + try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { + builder.humanReadable(true); + response.toXContentChunked(ToXContent.EMPTY_PARAMS).forEachRemaining(xcontent -> { + try { + xcontent.toXContent(builder, EMPTY_PARAMS); + } catch (IOException e) { + fail(e); + } + }); + Map xContentMap = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + assertThat(xContentMap, equalTo(Map.of("data_streams", List.of()))); + } + } + + public void testResponseToXContent() throws IOException { + Map dataStream1Settings = Map.of("setting1", "value1", "setting2", "value2"); + Map dataStream1EffectiveSettings = Map.of("setting1", "value1", "setting2", "value2", "setting3", "value3"); + Map dataStream2Settings = Map.of("setting4", "value4", "setting5", "value5"); + Map dataStream2EffectiveSettings = Map.of("setting4", "value4", "setting5", "value5", "settings6", "value6"); + GetDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse1 = + new GetDataStreamSettingsAction.DataStreamSettingsResponse( + "dataStream1", + Settings.builder().loadFromMap(dataStream1Settings).build(), + Settings.builder().loadFromMap(dataStream1EffectiveSettings).build() + ); + GetDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse2 = + new GetDataStreamSettingsAction.DataStreamSettingsResponse( + "dataStream2", + Settings.builder().loadFromMap(dataStream2Settings).build(), + Settings.builder().loadFromMap(dataStream2EffectiveSettings).build() + ); + List responseList = List.of( + dataStreamSettingsResponse1, + dataStreamSettingsResponse2 + ); + GetDataStreamSettingsAction.Response response = new GetDataStreamSettingsAction.Response(responseList); + try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { + builder.humanReadable(true); + response.toXContentChunked(ToXContent.EMPTY_PARAMS).forEachRemaining(xcontent -> { + try { + xcontent.toXContent(builder, EMPTY_PARAMS); + } catch (IOException e) { + fail(e); + } + }); + Map xContentMap = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + assertThat( + xContentMap, + equalTo( + Map.of( + "data_streams", + List.of( + Map.of( + "name", + "dataStream1", + "settings", + dataStream1Settings, + "effective_settings", + dataStream1EffectiveSettings + ), + Map.of( + "name", + "dataStream2", + "settings", + dataStream2Settings, + "effective_settings", + dataStream2EffectiveSettings + ) + ) + ) + ) + ); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsActionRequestTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsActionRequestTests.java new file mode 100644 index 0000000000000..e3f7e4063a4e3 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsActionRequestTests.java @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.datastreams; + +import org.elasticsearch.cluster.metadata.ComponentTemplateTests; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Arrays; +import java.util.function.Supplier; + +import static org.elasticsearch.cluster.metadata.ComponentTemplateTests.randomSettings; + +public class UpdateDataStreamSettingsActionRequestTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return UpdateDataStreamSettingsAction.Request::new; + } + + @Override + protected UpdateDataStreamSettingsAction.Request createTestInstance() { + UpdateDataStreamSettingsAction.Request request = new UpdateDataStreamSettingsAction.Request( + randomSettings(), + randomBoolean(), + randomTimeValue(), + randomTimeValue() + ); + request.indices(randomIndices()); + return request; + } + + @Override + protected UpdateDataStreamSettingsAction.Request mutateInstance(UpdateDataStreamSettingsAction.Request instance) throws IOException { + String[] indices = instance.indices(); + Settings settings = instance.getSettings(); + boolean dryRun = instance.isDryRun(); + TimeValue masterNodeTimeout = instance.masterNodeTimeout(); + TimeValue ackTimeout = instance.ackTimeout(); + switch (between(0, 4)) { + case 0 -> { + indices = randomArrayValueOtherThan(indices, this::randomIndices); + } + case 1 -> { + settings = randomValueOtherThan(settings, ComponentTemplateTests::randomSettings); + } + case 2 -> { + dryRun = dryRun == false; + } + case 3 -> { + masterNodeTimeout = randomValueOtherThan(masterNodeTimeout, ESTestCase::randomTimeValue); + } + case 4 -> { + ackTimeout = randomValueOtherThan(ackTimeout, ESTestCase::randomTimeValue); + } + default -> throw new AssertionError("Should not be here"); + } + return new UpdateDataStreamSettingsAction.Request(settings, dryRun, masterNodeTimeout, ackTimeout).indices(indices); + } + + private String[] randomIndices() { + return randomList(10, () -> randomAlphaOfLength(20)).toArray(new String[0]); + } + + public static T[] randomArrayValueOtherThan(T[] input, Supplier randomSupplier) { + return randomValueOtherThanMany(v -> Arrays.equals(input, v), randomSupplier); + } +} diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsActionResponseTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsActionResponseTests.java new file mode 100644 index 0000000000000..a249703beb30b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsActionResponseTests.java @@ -0,0 +1,213 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.datastreams; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.action.datastreams.UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndexSettingError; +import static org.elasticsearch.cluster.metadata.ComponentTemplateTests.randomSettings; +import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; +import static org.hamcrest.Matchers.equalTo; + +public class UpdateDataStreamSettingsActionResponseTests extends AbstractWireSerializingTestCase { + @Override + protected Writeable.Reader instanceReader() { + return UpdateDataStreamSettingsAction.Response::new; + } + + public void testToXContent() throws IOException { + Map dataStream1Settings = Map.of("setting1", "value1", "setting2", "value2"); + Map dataStream1EffectiveSettings = Map.of("setting1", "value1", "setting2", "value2", "setting3", "value3"); + List dataStream1AppliedToDataStreamOnly = randomList(10, () -> randomAlphanumericOfLength(10)); + List dataStream1AppliedToBackingIndices = randomList(10, () -> randomAlphanumericOfLength(10)); + List dataStream1IndexErrors = randomList( + 10, + () -> new IndexSettingError(randomAlphanumericOfLength(10), randomAlphaOfLength(10)) + ); + Map dataStream2Settings = Map.of("setting4", "value4", "setting5", "value5"); + Map dataStream2EffectiveSettings = Map.of("setting4", "value4", "setting5", "value5", "settings6", "value6"); + List dataStream2AppliedToDataStreamOnly = randomList(10, () -> randomAlphanumericOfLength(10)); + List dataStream2AppliedToBackingIndices = randomList(10, () -> randomAlphanumericOfLength(10)); + List dataStream2IndexErrors = randomList( + 10, + () -> new IndexSettingError(randomAlphanumericOfLength(10), randomAlphaOfLength(10)) + ); + boolean dataStream1Succeeded = randomBoolean(); + String dataStream1Error = randomBoolean() ? null : randomAlphaOfLength(20); + boolean dataStream2Succeeded = randomBoolean(); + String dataStream2Error = randomBoolean() ? null : randomAlphaOfLength(20); + UpdateDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse1 = + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( + "dataStream1", + dataStream1Succeeded, + dataStream1Error, + Settings.builder().loadFromMap(dataStream1Settings).build(), + Settings.builder().loadFromMap(dataStream1EffectiveSettings).build(), + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult( + dataStream1AppliedToDataStreamOnly, + dataStream1AppliedToBackingIndices, + dataStream1IndexErrors + ) + ); + UpdateDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse2 = + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( + "dataStream2", + dataStream2Succeeded, + dataStream2Error, + Settings.builder().loadFromMap(dataStream2Settings).build(), + Settings.builder().loadFromMap(dataStream2EffectiveSettings).build(), + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult( + dataStream2AppliedToDataStreamOnly, + dataStream2AppliedToBackingIndices, + dataStream2IndexErrors + ) + ); + List responseList = List.of( + dataStreamSettingsResponse1, + dataStreamSettingsResponse2 + ); + UpdateDataStreamSettingsAction.Response response = new UpdateDataStreamSettingsAction.Response(responseList); + try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { + builder.humanReadable(true); + response.toXContentChunked(ToXContent.EMPTY_PARAMS).forEachRemaining(xcontent -> { + try { + xcontent.toXContent(builder, EMPTY_PARAMS); + } catch (IOException e) { + fail(e); + } + }); + Map xContentMap = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + assertThat( + xContentMap, + equalTo( + Map.of( + "data_streams", + List.of( + buildExpectedMap( + "dataStream1", + dataStream1Succeeded, + dataStream1Error, + dataStream1Settings, + dataStream1EffectiveSettings, + dataStream1AppliedToDataStreamOnly, + dataStream1AppliedToBackingIndices, + dataStream1IndexErrors + ), + buildExpectedMap( + "dataStream2", + dataStream2Succeeded, + dataStream2Error, + dataStream2Settings, + dataStream2EffectiveSettings, + dataStream2AppliedToDataStreamOnly, + dataStream2AppliedToBackingIndices, + dataStream2IndexErrors + ) + ) + ) + ) + ); + } + } + + private Map buildExpectedMap( + String name, + boolean succeeded, + String error, + Map settings, + Map effectiveSettings, + List appliedToDataStreamOnly, + List appliedToIndices, + List indexErrors + ) { + Map result = new HashMap<>(); + result.put("name", name); + result.put("applied_to_data_stream", succeeded); + if (error != null) { + result.put("error", error); + } + result.put("settings", settings); + result.put("effective_settings", effectiveSettings); + Map indexSettingsResults = new HashMap<>(); + indexSettingsResults.put("applied_to_data_stream_only", appliedToDataStreamOnly); + indexSettingsResults.put("applied_to_data_stream_and_backing_indices", appliedToIndices); + if (indexErrors.isEmpty() == false) { + indexSettingsResults.put( + "errors", + indexErrors.stream() + .map(indexSettingError -> Map.of("index", indexSettingError.indexName(), "error", indexSettingError.errorMessage())) + .toList() + ); + } + result.put("index_settings_results", indexSettingsResults); + return result; + } + + @Override + protected UpdateDataStreamSettingsAction.Response createTestInstance() { + return new UpdateDataStreamSettingsAction.Response(randomList(10, this::randomDataStreamSettingsResponse)); + } + + private UpdateDataStreamSettingsAction.DataStreamSettingsResponse randomDataStreamSettingsResponse() { + return new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( + "dataStream1", + randomBoolean(), + randomBoolean() ? null : randomAlphaOfLength(20), + randomSettings(), + randomSettings(), + randomIndicesSettingsResult() + ); + } + + private UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult randomIndicesSettingsResult() { + return new UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult( + randomList(10, () -> randomAlphanumericOfLength(20)), + randomList(10, () -> randomAlphanumericOfLength(20)), + randomList(10, this::randomIndexSettingError) + ); + } + + private IndexSettingError randomIndexSettingError() { + return new IndexSettingError(randomAlphanumericOfLength(20), randomAlphanumericOfLength(20)); + } + + @Override + protected UpdateDataStreamSettingsAction.Response mutateInstance(UpdateDataStreamSettingsAction.Response instance) throws IOException { + List responseList = instance.getDataStreamSettingsResponses(); + List mutatedResponseList = new ArrayList<>(responseList); + switch (between(0, 1)) { + case 0 -> { + if (responseList.isEmpty()) { + mutatedResponseList.add(randomDataStreamSettingsResponse()); + } else { + mutatedResponseList.remove(randomInt(responseList.size() - 1)); + } + } + case 1 -> { + mutatedResponseList.add(randomDataStreamSettingsResponse()); + } + default -> throw new AssertionError("Should not be here"); + } + return new UpdateDataStreamSettingsAction.Response(mutatedResponseList); + } +} diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java index b0978d7449884..e75d8dd9fbcab 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; import static org.elasticsearch.cluster.metadata.DataStream.TIMESTAMP_FIELD_NAME; @@ -280,4 +281,68 @@ public void testBuilderRoundtrip() { assertEquals(template.template(), Template.builder(template.template()).build()); } } + + public void testMergeEmptySettingsIntoTemplateWithNonEmptySettings() { + // We only have settings from the template, so the effective template will just be the original template + Settings templateSettings = randomSettings(); + Template.Builder templateBuilder = Template.builder().settings(templateSettings).mappings(randomMappings(null)); + ComposableIndexTemplate indexTemplate = randomInstance(); + expectThrows(NullPointerException.class, () -> indexTemplate.mergeSettings(null)); + assertThat(indexTemplate.mergeSettings(Settings.EMPTY), equalTo(indexTemplate)); + } + + public void testMergeNonEmptySettingsIntoTemplateWithEmptySettings() { + // We only have settings from the data stream, so we expect to get only those back in the effective template + Settings dataStreamSettings = randomSettings(); + String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + Settings templateSettings = Settings.EMPTY; + CompressedXContent templateMappings = randomMappings(randomDataStreamTemplate()); + Template.Builder templateBuilder = Template.builder().settings(templateSettings).mappings(templateMappings); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(templateBuilder) + .build(); + Template.Builder expectedTemplateBuilder = Template.builder().settings(dataStreamSettings).mappings(templateMappings); + ComposableIndexTemplate expectedEffectiveTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(expectedTemplateBuilder) + .build(); + assertThat(indexTemplate.mergeSettings(dataStreamSettings), equalTo(expectedEffectiveTemplate)); + } + + public void testMergeSettings() { + // Here we have settings from both the template and the data stream, so we expect the data stream settings to take precedence + Settings dataStreamSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .put("index.setting3", (String) null) // This one gets removed from the effective settings + .build(); + String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + Settings templateSettings = Settings.builder() + .put("index.setting1", "templateValue") + .put("index.setting3", "templateValue") + .put("index.setting4", "templateValue") + .build(); + CompressedXContent templateMappings = randomMappings(randomDataStreamTemplate()); + Template.Builder templateBuilder = Template.builder().settings(templateSettings).mappings(templateMappings); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(templateBuilder) + .build(); + Settings mergedSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .put("index.setting4", "templateValue") + .build(); + Template.Builder expectedTemplateBuilder = Template.builder().settings(mergedSettings).mappings(templateMappings); + ComposableIndexTemplate expectedEffectiveTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(expectedTemplateBuilder) + .build(); + assertThat(indexTemplate.mergeSettings(dataStreamSettings), equalTo(expectedEffectiveTemplate)); + } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index 8205d4f079b75..be4116e86b894 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; @@ -52,11 +53,13 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.elasticsearch.cluster.metadata.ComponentTemplateTests.randomMappings; import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName; import static org.elasticsearch.cluster.metadata.DataStream.getDefaultFailureStoreName; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.randomIndexInstances; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.randomNonEmptyIndexInstances; +import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.randomSettings; import static org.elasticsearch.index.IndexSettings.LIFECYCLE_ORIGINATION_DATE; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -68,6 +71,7 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -94,6 +98,7 @@ protected DataStream mutateInstance(DataStream instance) { var indices = instance.getIndices(); var generation = instance.getGeneration(); var metadata = instance.getMetadata(); + var settings = instance.getSettings(); var isHidden = instance.isHidden(); var isReplicated = instance.isReplicated(); var isSystem = instance.isSystem(); @@ -106,7 +111,7 @@ protected DataStream mutateInstance(DataStream instance) { var autoShardingEvent = instance.getAutoShardingEvent(); var failureRolloverOnWrite = instance.getFailureComponent().isRolloverOnWrite(); var failureAutoShardingEvent = instance.getDataComponent().getAutoShardingEvent(); - switch (between(0, 15)) { + switch (between(0, 16)) { case 0 -> name = randomAlphaOfLength(10); case 1 -> indices = randomNonEmptyIndexInstances(); case 2 -> generation = instance.getGeneration() + randomIntBetween(1, 10); @@ -178,12 +183,14 @@ protected DataStream mutateInstance(DataStream instance) { randomIntBetween(1, 10), randomMillisUpToYear9999() ); + case 16 -> settings = randomValueOtherThan(settings, DataStreamTestHelper::randomSettings); } return new DataStream( name, generation, metadata, + settings, isHidden, isReplicated, isSystem, @@ -1916,6 +1923,7 @@ public void testXContentSerializationWithRolloverAndEffectiveRetention() throws indices, generation, metadata, + randomSettings(), isSystem, randomBoolean(), isSystem, @@ -2102,6 +2110,7 @@ public void testWriteFailureIndex() { randomNonEmptyIndexInstances(), randomNonNegativeInt(), null, + randomSettings(), hidden, replicated, system, @@ -2120,6 +2129,7 @@ public void testWriteFailureIndex() { randomNonEmptyIndexInstances(), randomNonNegativeInt(), null, + randomSettings(), hidden, replicated, system, @@ -2145,6 +2155,7 @@ public void testWriteFailureIndex() { randomNonEmptyIndexInstances(), randomNonNegativeInt(), null, + randomSettings(), hidden, replicated, system, @@ -2169,6 +2180,7 @@ public void testIsFailureIndex() { backingIndices, randomNonNegativeInt(), null, + randomSettings(), hidden, replicated, system, @@ -2191,6 +2203,7 @@ public void testIsFailureIndex() { backingIndices, randomNonNegativeInt(), null, + randomSettings(), hidden, replicated, system, @@ -2222,6 +2235,7 @@ public void testIsFailureIndex() { backingIndices, randomNonNegativeInt(), null, + randomSettings(), hidden, replicated, system, @@ -2409,6 +2423,173 @@ public void testIsFailureStoreEffectivelyEnabled_staticHelperMethod() { ); } + public void testGetEffectiveSettingsNoMatchingTemplate() { + // No matching template, so we expect an IllegalArgumentException + DataStream dataStream = createTestInstance(); + Metadata.Builder metadataBuilder = Metadata.builder(); + assertThrows(IllegalArgumentException.class, () -> dataStream.getEffectiveSettings(metadataBuilder.build())); + } + + public void testGetEffectiveSettingsTemplateSettingsOnly() { + // We only have settings from the template, so we expect to get those back + DataStream dataStream = createDataStream(Settings.EMPTY); + Settings templateSettings = randomSettings(); + Template.Builder templateBuilder = Template.builder().settings(templateSettings); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStream.getName())) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(templateBuilder) + .build(); + Metadata.Builder metadataBuilder = Metadata.builder().indexTemplates(Map.of(dataStream.getName(), indexTemplate)); + assertThat(dataStream.getEffectiveSettings(metadataBuilder.build()), equalTo(templateSettings)); + } + + public void testGetEffectiveSettingsComponentTemplateSettingsOnly() { + // We only have settings from a component template, so we expect to get those back + DataStream dataStream = createDataStream(Settings.EMPTY); + Settings templateSettings = Settings.EMPTY; + Template.Builder indexTemplateBuilder = Template.builder().settings(templateSettings); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStream.getName())) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(indexTemplateBuilder) + .componentTemplates(List.of("component-template-1")) + .build(); + Settings componentSettings = randomSettings(); + Template.Builder componentTemplateBuilder = Template.builder().settings(componentSettings); + ComponentTemplate componentTemplate1 = new ComponentTemplate(componentTemplateBuilder.build(), null, null, null); + Metadata.Builder projectMetadataBuilder = Metadata.builder() + .indexTemplates(Map.of(dataStream.getName(), indexTemplate)) + .componentTemplates(Map.of("component-template-1", componentTemplate1)); + assertThat(dataStream.getEffectiveSettings(projectMetadataBuilder.build()), equalTo(componentSettings)); + } + + public void testGetEffectiveSettingsDataStreamSettingsOnly() { + // We only have settings from the data stream, so we expect to get those back + Settings dataStreamSettings = randomSettings(); + DataStream dataStream = createDataStream(dataStreamSettings); + Settings templateSettings = Settings.EMPTY; + Template.Builder templateBuilder = Template.builder().settings(templateSettings); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStream.getName())) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(templateBuilder) + .build(); + Metadata.Builder metadataBuilder = Metadata.builder().indexTemplates(Map.of(dataStream.getName(), indexTemplate)); + assertThat(dataStream.getEffectiveSettings(metadataBuilder.build()), equalTo(dataStreamSettings)); + } + + public void testGetEffectiveSettings() { + // Here we have settings from both the template and the data stream, so we expect the data stream settings to take precedence + Settings dataStreamSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .put("index.setting3", (String) null) // This one gets removed from the effective settings + .build(); + DataStream dataStream = createDataStream(dataStreamSettings); + Settings templateSettings = Settings.builder() + .put("index.setting1", "templateValue") + .put("index.setting3", "templateValue") + .put("index.setting4", "templateValue") + .build(); + Template.Builder templateBuilder = Template.builder().settings(templateSettings); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStream.getName())) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(templateBuilder) + .build(); + Metadata.Builder metadataBuilder = Metadata.builder().indexTemplates(Map.of(dataStream.getName(), indexTemplate)); + Settings mergedSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .put("index.setting4", "templateValue") + .build(); + assertThat(dataStream.getEffectiveSettings(metadataBuilder.build()), equalTo(mergedSettings)); + } + + public void testGetEffectiveIndexTemplateNoMatchingTemplate() { + // No matching template, so we expect an IllegalArgumentException + DataStream dataStream = createTestInstance(); + Metadata.Builder metadataBuilder = Metadata.builder(); + assertThrows(IllegalArgumentException.class, () -> dataStream.getEffectiveIndexTemplate(metadataBuilder.build())); + } + + public void testGetEffectiveIndexTemplateTemplateSettingsOnly() { + // We only have settings from the template, so the effective template will just be the original template + DataStream dataStream = createDataStream(Settings.EMPTY); + Settings templateSettings = randomSettings(); + Template.Builder templateBuilder = Template.builder().settings(templateSettings).mappings(randomMappings()); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStream.getName())) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(templateBuilder) + .build(); + Metadata.Builder metadataBuilder = Metadata.builder().indexTemplates(Map.of(dataStream.getName(), indexTemplate)); + assertThat(dataStream.getEffectiveIndexTemplate(metadataBuilder.build()), equalTo(indexTemplate)); + } + + public void testGetEffectiveIndexTemplateDataStreamSettingsOnly() { + // We only have settings from the data stream, so we expect to get only those back in the effective template + Settings dataStreamSettings = randomSettings(); + DataStream dataStream = createDataStream(dataStreamSettings); + Settings templateSettings = Settings.EMPTY; + CompressedXContent templateMappings = randomMappings(); + Template.Builder templateBuilder = Template.builder().settings(templateSettings).mappings(templateMappings); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStream.getName())) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(templateBuilder) + .build(); + Metadata.Builder metadataBuilder = Metadata.builder().indexTemplates(Map.of(dataStream.getName(), indexTemplate)); + Template.Builder expectedTemplateBuilder = Template.builder().settings(dataStreamSettings).mappings(templateMappings); + ComposableIndexTemplate expectedEffectiveTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStream.getName())) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(expectedTemplateBuilder) + .build(); + assertThat(dataStream.getEffectiveIndexTemplate(metadataBuilder.build()), equalTo(expectedEffectiveTemplate)); + } + + public void testGetEffectiveIndexTemplate() { + // Here we have settings from both the template and the data stream, so we expect the data stream settings to take precedence + Settings dataStreamSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .put("index.setting3", (String) null) // This one gets removed from the effective settings + .build(); + DataStream dataStream = createDataStream(dataStreamSettings); + Settings templateSettings = Settings.builder() + .put("index.setting1", "templateValue") + .put("index.setting3", "templateValue") + .put("index.setting4", "templateValue") + .build(); + CompressedXContent templateMappings = randomMappings(); + Template.Builder templateBuilder = Template.builder().settings(templateSettings).mappings(templateMappings); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStream.getName())) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(templateBuilder) + .build(); + Metadata.Builder metadataBuilder = Metadata.builder().indexTemplates(Map.of(dataStream.getName(), indexTemplate)); + Settings mergedSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .put("index.setting4", "templateValue") + .build(); + Template.Builder expectedTemplateBuilder = Template.builder().settings(mergedSettings).mappings(templateMappings); + ComposableIndexTemplate expectedEffectiveTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStream.getName())) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(expectedTemplateBuilder) + .build(); + assertThat(dataStream.getEffectiveIndexTemplate(metadataBuilder.build()), equalTo(expectedEffectiveTemplate)); + } + + private DataStream createDataStream(Settings settings) { + DataStream dataStream = createTestInstance(); + return dataStream.copy().setSettings(settings).build(); + } + private record DataStreamMetadata(Long creationTimeInMillis, Long rolloverTimeInMillis, Long originationTimeInMillis) { public static DataStreamMetadata dataStreamMetadata(Long creationTimeInMillis, Long rolloverTimeInMillis) { return new DataStreamMetadata(creationTimeInMillis, rolloverTimeInMillis, null); diff --git a/server/src/test/java/org/elasticsearch/common/settings/SettingsTests.java b/server/src/test/java/org/elasticsearch/common/settings/SettingsTests.java index d608136fa564e..69928c29b977b 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/SettingsTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/SettingsTests.java @@ -729,4 +729,59 @@ public void testGlobValues() throws IOException { assertThat(values, contains("1")); } + public void testMergeNullOrEmptySettingsIntoEmptySettings() { + expectThrows(NullPointerException.class, () -> Settings.EMPTY.merge(null)); + assertThat(Settings.EMPTY.merge(Settings.EMPTY), equalTo(Settings.EMPTY)); + } + + public void testMergeEmptySettings() { + Settings.Builder builder = Settings.builder(); + for (int i = 1; i < randomInt(100); i++) { + builder.put(randomAlphanumericOfLength(20), randomAlphanumericOfLength(50)); + } + Settings settings = builder.build(); + assertThat(settings.merge(Settings.EMPTY), equalTo(settings)); + } + + public void testMergeNonEmptySettingsIntoEmptySettings() { + Settings.Builder builder = Settings.builder(); + for (int i = 1; i < randomInt(100); i++) { + builder.put(randomAlphanumericOfLength(20), randomAlphanumericOfLength(50)); + } + Settings newSettings = builder.build(); + assertThat(Settings.EMPTY.merge(newSettings), equalTo(newSettings)); + } + + public void testMergeNonEmptySettingsIntoNonEmptySettings() { + Settings settings = Settings.builder() + .put("index.setting1", "templateValue") + .put("index.setting3", "templateValue") + .put("index.setting4", "templateValue") + .build(); + Settings newSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .put("index.setting3", (String) null) // This one gets removed from the effective settings + .build(); + Settings mergedSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .put("index.setting4", "templateValue") + .build(); + assertThat(settings.merge(newSettings), equalTo(mergedSettings)); + } + + public void testMergeNonEmptySettingsWithNullIntoEmptySettings() { + Settings newSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .put("index.setting3", (String) null) // This one gets removed from the effective settings + .build(); + Settings mergedSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .build(); + assertThat(Settings.EMPTY.merge(newSettings), equalTo(mergedSettings)); + } + } diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java index d2fa30b743c76..7bc1b7b667d45 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java @@ -76,8 +76,10 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; import static org.elasticsearch.test.ESTestCase.generateRandomStringArray; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; +import static org.elasticsearch.test.ESTestCase.randomAlphanumericOfLength; import static org.elasticsearch.test.ESTestCase.randomBoolean; import static org.elasticsearch.test.ESTestCase.randomFrom; +import static org.elasticsearch.test.ESTestCase.randomInt; import static org.elasticsearch.test.ESTestCase.randomIntBetween; import static org.elasticsearch.test.ESTestCase.randomMap; import static org.elasticsearch.test.ESTestCase.randomMillisUpToYear9999; @@ -370,6 +372,7 @@ public static DataStream randomInstance(String dataStreamName, LongSupplier time dataStreamName, generation, metadata, + randomSettings(), system ? true : randomBoolean(), replicated, system, @@ -804,4 +807,15 @@ public static DataStreamOptions.Template createDataStreamOptionsTemplate(Boolean } return new DataStreamOptions.Template(DataStreamFailureStore.builder().enabled(failureStoreEnabled).buildTemplate()); } + + static Settings randomSettings() { + Settings.Builder builder = Settings.builder(); + if (randomBoolean()) { + return Settings.EMPTY; + } + for (int i = 1; i < randomInt(100); i++) { + builder.put(randomAlphanumericOfLength(20), randomAlphanumericOfLength(50)); + } + return builder.build(); + } } diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index c06cfb488b761..ad222a1d47d73 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -610,6 +610,7 @@ public class Constants { "indices:data/write/reindex", "indices:data/write/update", "indices:data/write/update/byquery", + "indices:monitor/data_stream/settings/get", "indices:monitor/data_stream/stats", "indices:monitor/field_usage_stats", "indices:monitor/fleet/global_checkpoints[s]", @@ -641,6 +642,7 @@ public class Constants { "indices:admin/data_stream/index/reindex", "indices:admin/data_stream/reindex", "indices:admin/data_stream/reindex_cancel", + "indices:admin/data_stream/settings/update", "indices:admin/index/create_from_source", "indices:admin/index/copy_lifecycle_index_metadata", "internal:admin/repository/verify",