From a3fe59f9eaff4da611c1e16ae206df68456bfe8b Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 23 Apr 2025 13:27:40 -0500 Subject: [PATCH 01/17] Adding settings to data streams (#126947) --- .../UpdateTimeSeriesRangeServiceTests.java | 1 + .../org/elasticsearch/TransportVersions.java | 1 + .../metadata/ComposableIndexTemplate.java | 29 ++++ .../cluster/metadata/DataStream.java | 87 ++++++++++ .../MetadataCreateDataStreamService.java | 1 + .../common/settings/Settings.java | 21 +++ .../ComposableIndexTemplateTests.java | 65 +++++++ .../cluster/metadata/DataStreamTests.java | 163 +++++++++++++++++- .../common/settings/SettingsTests.java | 55 ++++++ .../metadata/DataStreamTestHelper.java | 14 ++ 10 files changed, 436 insertions(+), 1 deletion(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java index 5da05e3e0fb68..2435cf1ced8ae 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java @@ -248,6 +248,7 @@ public void testUpdateTimeSeriesTemporalOneBadDataStream() { ds2Indices, 2, ds2.getMetadata(), + ds2.getSettings(), ds2.isHidden(), ds2.isReplicated(), ds2.isSystem(), diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 4f4570ddcbb8c..4d0ee63c800ce 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -239,6 +239,7 @@ static TransportVersion def(int id) { public static final TransportVersion SEARCH_SOURCE_EXCLUDE_VECTORS_PARAM_8_19 = def(8_841_0_46); public static final TransportVersion ML_INFERENCE_MISTRAL_CHAT_COMPLETION_ADDED_8_19 = def(8_841_0_47); public static final TransportVersion ML_INFERENCE_ELASTIC_RERANK_ADDED_8_19 = def(8_841_0_48); + public static final TransportVersion SETTINGS_IN_DATA_STREAMS = def(8_841_0_49); // TODO NO NO NO NO NO, also rename this /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index c82af6cf0dd8d..3c29b70720887 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.MapperService; @@ -309,6 +310,34 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla return builder; } + /* + * Merges the given settings into the settings in this ComposableIndexTemplate. Any null values in the + * given settings are removed from the settings in the returned ComposableIndexTemplate. If this + * ComposableIndexTemplate has no settings, the given settings are the only ones in the returned template + * (with any null values removed). If this ComposableIndexTemplate has no template, an empty template with + * those settings is created. If the given settings are empty, this ComposableIndexTemplate is just + * returned unchanged. This method never changes this object. + */ + public ComposableIndexTemplate mergeSettings(Settings settings) { + Objects.requireNonNull(settings); + if (Settings.EMPTY.equals(settings)) { + return this; + } + ComposableIndexTemplate.Builder mergedIndexTemplateBuilder = this.toBuilder(); + Template.Builder mergedTemplateBuilder; + Settings templateSettings; + if (this.template() == null) { + mergedTemplateBuilder = Template.builder(); + templateSettings = null; + } else { + mergedTemplateBuilder = Template.builder(this.template()); + templateSettings = this.template().settings(); + } + mergedTemplateBuilder.settings(templateSettings == null ? settings : templateSettings.merge(settings)); + mergedIndexTemplateBuilder.template(mergedTemplateBuilder); + return mergedIndexTemplateBuilder.build(); + } + @Override public int hashCode() { return Objects.hash( diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 774df9f710f7b..5c095a302b228 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateFormatters; import org.elasticsearch.common.xcontent.XContentHelper; @@ -64,6 +65,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.lookupTemplateForDataStream; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; import static org.elasticsearch.index.IndexSettings.LIFECYCLE_ORIGINATION_DATE; import static org.elasticsearch.index.IndexSettings.PREFER_ILM_SETTING; @@ -105,6 +107,7 @@ public final class DataStream implements SimpleDiffable, ToXContentO private final long generation; @Nullable private final Map metadata; + private final Settings settings; private final boolean hidden; private final boolean replicated; private final boolean system; @@ -137,8 +140,46 @@ public DataStream( ) { this( name, + indices, generation, metadata, + Settings.EMPTY, + hidden, + replicated, + system, + allowCustomRouting, + indexMode, + lifecycle, + dataStreamOptions, + failureIndices, + rolloverOnWrite, + autoShardingEvent + ); + } + + // visible for testing + public DataStream( + String name, + List indices, + long generation, + Map metadata, + Settings settings, + boolean hidden, + boolean replicated, + boolean system, + boolean allowCustomRouting, + IndexMode indexMode, + DataStreamLifecycle lifecycle, + @Nullable DataStreamOptions dataStreamOptions, + List failureIndices, + boolean rolloverOnWrite, + @Nullable DataStreamAutoShardingEvent autoShardingEvent + ) { + this( + name, + generation, + metadata, + settings, hidden, replicated, system, @@ -156,6 +197,7 @@ public DataStream( String name, long generation, Map metadata, + Settings settings, boolean hidden, boolean replicated, boolean system, @@ -170,6 +212,7 @@ public DataStream( this.name = name; this.generation = generation; this.metadata = metadata; + this.settings = Objects.requireNonNull(settings); // The following assert is commented out, because system data streams created before 8.1 are not hidden, // but should be updated to hidden by 8.18/8.19 (SystemIndexMetadataUpgradeService) // assert system == false || hidden; // system indices must be hidden @@ -226,10 +269,17 @@ public static DataStream read(StreamInput in) throws IOException { // is still behind a feature flag in previous version we use the default value instead of explicitly disabling it. dataStreamOptions = failureStoreEnabled ? DataStreamOptions.FAILURE_STORE_ENABLED : null; } + final Settings settings; + if (in.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS)) { + settings = Settings.readSettingsFromStream(in); + } else { + settings = Settings.EMPTY; + } return new DataStream( name, generation, metadata, + settings, hidden, replicated, system, @@ -320,6 +370,20 @@ public boolean rolloverOnWrite() { return backingIndices.rolloverOnWrite; } + public ComposableIndexTemplate getEffectiveIndexTemplate(Metadata metadata) { + return getMatchingIndexTemplate(metadata).mergeSettings(settings); + } + + public Settings getEffectiveSettings(Metadata metadata) { + ComposableIndexTemplate template = getMatchingIndexTemplate(metadata); + Settings templateSettings = template.template() == null ? Settings.EMPTY : template.template().settings(); + return templateSettings.merge(settings); + } + + private ComposableIndexTemplate getMatchingIndexTemplate(Metadata metadata) { + return lookupTemplateForDataStream(name, metadata); + } + /** * We define that a data stream is considered internal either if it is a system index or if * its name starts with a dot. @@ -431,6 +495,10 @@ public Map getMetadata() { return metadata; } + public Settings getSettings() { + return settings; + } + @Override public boolean isHidden() { return hidden; @@ -1250,6 +1318,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(DataStream.ADD_DATA_STREAM_OPTIONS_VERSION)) { out.writeOptionalWriteable(dataStreamOptions.isEmpty() ? null : dataStreamOptions); } + if (out.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS)) { + settings.writeTo(out); + } } public static final ParseField NAME_FIELD = new ParseField("name"); @@ -1271,6 +1342,7 @@ public void writeTo(StreamOutput out) throws IOException { public static final ParseField FAILURE_ROLLOVER_ON_WRITE_FIELD = new ParseField("failure_rollover_on_write"); public static final ParseField FAILURE_AUTO_SHARDING_FIELD = new ParseField("failure_auto_sharding"); public static final ParseField DATA_STREAM_OPTIONS_FIELD = new ParseField("options"); + public static final ParseField SETTINGS_FIELD = new ParseField("settings"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -1279,6 +1351,7 @@ public void writeTo(StreamOutput out) throws IOException { (String) args[0], (Long) args[2], (Map) args[3], + args[17] == null ? Settings.EMPTY : (Settings) args[17], args[4] != null && (boolean) args[4], args[5] != null && (boolean) args[5], args[6] != null && (boolean) args[6], @@ -1349,6 +1422,7 @@ public void writeTo(StreamOutput out) throws IOException { (p, c) -> DataStreamOptions.fromXContent(p), DATA_STREAM_OPTIONS_FIELD ); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> Settings.fromXContent(p), SETTINGS_FIELD); } public static DataStream fromXContent(XContentParser parser) throws IOException { @@ -1410,6 +1484,9 @@ public XContentBuilder toXContent( backingIndices.autoShardingEvent.toXContent(builder, params); builder.endObject(); } + builder.startObject(SETTINGS_FIELD.getPreferredName()); + this.settings.toXContent(builder, params); + builder.endObject(); builder.endObject(); return builder; } @@ -1422,6 +1499,7 @@ public boolean equals(Object o) { return name.equals(that.name) && generation == that.generation && Objects.equals(metadata, that.metadata) + && Objects.equals(settings, that.settings) && hidden == that.hidden && system == that.system && replicated == that.replicated @@ -1439,6 +1517,7 @@ public int hashCode() { name, generation, metadata, + settings, hidden, system, replicated, @@ -1751,6 +1830,7 @@ public static class Builder { private long generation = 1; @Nullable private Map metadata = null; + private Settings settings = Settings.EMPTY; private boolean hidden = false; private boolean replicated = false; private boolean system = false; @@ -1778,6 +1858,7 @@ private Builder(DataStream dataStream) { name = dataStream.name; generation = dataStream.generation; metadata = dataStream.metadata; + settings = dataStream.settings; hidden = dataStream.hidden; replicated = dataStream.replicated; system = dataStream.system; @@ -1809,6 +1890,11 @@ public Builder setMetadata(Map metadata) { return this; } + public Builder setSettings(Settings settings) { + this.settings = settings; + return this; + } + public Builder setHidden(boolean hidden) { this.hidden = hidden; return this; @@ -1869,6 +1955,7 @@ public DataStream build() { name, generation, metadata, + settings, hidden, replicated, system, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 175992aa90737..196593ccb55cf 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -321,6 +321,7 @@ static ClusterState createDataStream( dataStreamName, initialGeneration, template.metadata() != null ? Map.copyOf(template.metadata()) : null, + Settings.EMPTY, hidden, false, isSystem, diff --git a/server/src/main/java/org/elasticsearch/common/settings/Settings.java b/server/src/main/java/org/elasticsearch/common/settings/Settings.java index 784fb5ebc63d3..4153702bf1af8 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Settings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Settings.java @@ -876,6 +876,27 @@ public Set keySet() { return newKeySet; } + /* + * This method merges the given newSettings into this Settings, returning either a new Settings object or this if the newSettings are + * empty. If any values are null in newSettings, those keys are removed from the returned object. + */ + public Settings merge(Settings newSettings) { + Objects.requireNonNull(newSettings); + if (Settings.EMPTY.equals(newSettings)) { + return this; + } + Settings.Builder builder = Settings.builder().put(this); + for (String key : newSettings.keySet()) { + String rawValue = newSettings.get(key); + if (rawValue == null) { + builder.remove(key); + } else { + builder.put(key, rawValue); + } + } + return builder.build(); + } + /** * A builder allowing to put different settings and then {@link #build()} an immutable * settings implementation. Use {@link Settings#builder()} in order to diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java index b0978d7449884..e75d8dd9fbcab 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; import static org.elasticsearch.cluster.metadata.DataStream.TIMESTAMP_FIELD_NAME; @@ -280,4 +281,68 @@ public void testBuilderRoundtrip() { assertEquals(template.template(), Template.builder(template.template()).build()); } } + + public void testMergeEmptySettingsIntoTemplateWithNonEmptySettings() { + // We only have settings from the template, so the effective template will just be the original template + Settings templateSettings = randomSettings(); + Template.Builder templateBuilder = Template.builder().settings(templateSettings).mappings(randomMappings(null)); + ComposableIndexTemplate indexTemplate = randomInstance(); + expectThrows(NullPointerException.class, () -> indexTemplate.mergeSettings(null)); + assertThat(indexTemplate.mergeSettings(Settings.EMPTY), equalTo(indexTemplate)); + } + + public void testMergeNonEmptySettingsIntoTemplateWithEmptySettings() { + // We only have settings from the data stream, so we expect to get only those back in the effective template + Settings dataStreamSettings = randomSettings(); + String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + Settings templateSettings = Settings.EMPTY; + CompressedXContent templateMappings = randomMappings(randomDataStreamTemplate()); + Template.Builder templateBuilder = Template.builder().settings(templateSettings).mappings(templateMappings); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(templateBuilder) + .build(); + Template.Builder expectedTemplateBuilder = Template.builder().settings(dataStreamSettings).mappings(templateMappings); + ComposableIndexTemplate expectedEffectiveTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(expectedTemplateBuilder) + .build(); + assertThat(indexTemplate.mergeSettings(dataStreamSettings), equalTo(expectedEffectiveTemplate)); + } + + public void testMergeSettings() { + // Here we have settings from both the template and the data stream, so we expect the data stream settings to take precedence + Settings dataStreamSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .put("index.setting3", (String) null) // This one gets removed from the effective settings + .build(); + String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + Settings templateSettings = Settings.builder() + .put("index.setting1", "templateValue") + .put("index.setting3", "templateValue") + .put("index.setting4", "templateValue") + .build(); + CompressedXContent templateMappings = randomMappings(randomDataStreamTemplate()); + Template.Builder templateBuilder = Template.builder().settings(templateSettings).mappings(templateMappings); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(templateBuilder) + .build(); + Settings mergedSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .put("index.setting4", "templateValue") + .build(); + Template.Builder expectedTemplateBuilder = Template.builder().settings(mergedSettings).mappings(templateMappings); + ComposableIndexTemplate expectedEffectiveTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(expectedTemplateBuilder) + .build(); + assertThat(indexTemplate.mergeSettings(dataStreamSettings), equalTo(expectedEffectiveTemplate)); + } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index 8205d4f079b75..6c51f736eeb7b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; @@ -52,11 +53,13 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.elasticsearch.cluster.metadata.ComponentTemplateTests.randomMappings; import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName; import static org.elasticsearch.cluster.metadata.DataStream.getDefaultFailureStoreName; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.randomIndexInstances; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.randomNonEmptyIndexInstances; +import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.randomSettings; import static org.elasticsearch.index.IndexSettings.LIFECYCLE_ORIGINATION_DATE; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -68,6 +71,7 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -94,6 +98,7 @@ protected DataStream mutateInstance(DataStream instance) { var indices = instance.getIndices(); var generation = instance.getGeneration(); var metadata = instance.getMetadata(); + var settings = instance.getSettings(); var isHidden = instance.isHidden(); var isReplicated = instance.isReplicated(); var isSystem = instance.isSystem(); @@ -106,7 +111,7 @@ protected DataStream mutateInstance(DataStream instance) { var autoShardingEvent = instance.getAutoShardingEvent(); var failureRolloverOnWrite = instance.getFailureComponent().isRolloverOnWrite(); var failureAutoShardingEvent = instance.getDataComponent().getAutoShardingEvent(); - switch (between(0, 15)) { + switch (between(0, 16)) { case 0 -> name = randomAlphaOfLength(10); case 1 -> indices = randomNonEmptyIndexInstances(); case 2 -> generation = instance.getGeneration() + randomIntBetween(1, 10); @@ -178,12 +183,14 @@ protected DataStream mutateInstance(DataStream instance) { randomIntBetween(1, 10), randomMillisUpToYear9999() ); + case 16 -> settings = randomValueOtherThan(settings, DataStreamTestHelper::randomSettings); } return new DataStream( name, generation, metadata, + settings, isHidden, isReplicated, isSystem, @@ -1916,6 +1923,7 @@ public void testXContentSerializationWithRolloverAndEffectiveRetention() throws indices, generation, metadata, + randomSettings(), isSystem, randomBoolean(), isSystem, @@ -2102,6 +2110,7 @@ public void testWriteFailureIndex() { randomNonEmptyIndexInstances(), randomNonNegativeInt(), null, + randomSettings(), hidden, replicated, system, @@ -2120,6 +2129,7 @@ public void testWriteFailureIndex() { randomNonEmptyIndexInstances(), randomNonNegativeInt(), null, + randomSettings(), hidden, replicated, system, @@ -2145,6 +2155,7 @@ public void testWriteFailureIndex() { randomNonEmptyIndexInstances(), randomNonNegativeInt(), null, + randomSettings(), hidden, replicated, system, @@ -2169,6 +2180,7 @@ public void testIsFailureIndex() { backingIndices, randomNonNegativeInt(), null, + randomSettings(), hidden, replicated, system, @@ -2191,6 +2203,7 @@ public void testIsFailureIndex() { backingIndices, randomNonNegativeInt(), null, + randomSettings(), hidden, replicated, system, @@ -2222,6 +2235,7 @@ public void testIsFailureIndex() { backingIndices, randomNonNegativeInt(), null, + randomSettings(), hidden, replicated, system, @@ -2409,6 +2423,153 @@ public void testIsFailureStoreEffectivelyEnabled_staticHelperMethod() { ); } + public void testGetEffectiveSettingsNoMatchingTemplate() { + // No matching template, so we expect an IllegalArgumentException + DataStream dataStream = createTestInstance(); + Metadata.Builder metadataBuilder = Metadata.builder(); + assertThrows(IllegalArgumentException.class, () -> dataStream.getEffectiveSettings(metadataBuilder.build())); + } + + public void testGetEffectiveSettingsTemplateSettingsOnly() { + // We only have settings from the template, so we expect to get those back + DataStream dataStream = createDataStream(Settings.EMPTY); + Settings templateSettings = randomSettings(); + Template.Builder templateBuilder = Template.builder().settings(templateSettings); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStream.getName())) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(templateBuilder) + .build(); + Metadata.Builder metadataBuilder = Metadata.builder().indexTemplates(Map.of(dataStream.getName(), indexTemplate)); + assertThat(dataStream.getEffectiveSettings(metadataBuilder.build()), equalTo(templateSettings)); + } + + public void testGetEffectiveSettingsDataStreamSettingsOnly() { + // We only have settings from the data stream, so we expect to get those back + Settings dataStreamSettings = randomSettings(); + DataStream dataStream = createDataStream(dataStreamSettings); + Settings templateSettings = Settings.EMPTY; + Template.Builder templateBuilder = Template.builder().settings(templateSettings); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStream.getName())) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(templateBuilder) + .build(); + Metadata.Builder metadataBuilder = Metadata.builder().indexTemplates(Map.of(dataStream.getName(), indexTemplate)); + assertThat(dataStream.getEffectiveSettings(metadataBuilder.build()), equalTo(dataStreamSettings)); + } + + public void testGetEffectiveSettings() { + // Here we have settings from both the template and the data stream, so we expect the data stream settings to take precedence + Settings dataStreamSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .put("index.setting3", (String) null) // This one gets removed from the effective settings + .build(); + DataStream dataStream = createDataStream(dataStreamSettings); + Settings templateSettings = Settings.builder() + .put("index.setting1", "templateValue") + .put("index.setting3", "templateValue") + .put("index.setting4", "templateValue") + .build(); + Template.Builder templateBuilder = Template.builder().settings(templateSettings); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStream.getName())) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(templateBuilder) + .build(); + Metadata.Builder metadataBuilder = Metadata.builder().indexTemplates(Map.of(dataStream.getName(), indexTemplate)); + Settings mergedSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .put("index.setting4", "templateValue") + .build(); + assertThat(dataStream.getEffectiveSettings(metadataBuilder.build()), equalTo(mergedSettings)); + } + + public void testGetEffectiveIndexTemplateNoMatchingTemplate() { + // No matching template, so we expect an IllegalArgumentException + DataStream dataStream = createTestInstance(); + Metadata.Builder metadataBuilder = Metadata.builder(); + assertThrows(IllegalArgumentException.class, () -> dataStream.getEffectiveIndexTemplate(metadataBuilder.build())); + } + + public void testGetEffectiveIndexTemplateTemplateSettingsOnly() { + // We only have settings from the template, so the effective template will just be the original template + DataStream dataStream = createDataStream(Settings.EMPTY); + Settings templateSettings = randomSettings(); + Template.Builder templateBuilder = Template.builder().settings(templateSettings).mappings(randomMappings()); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStream.getName())) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(templateBuilder) + .build(); + Metadata.Builder metadataBuilder = Metadata.builder().indexTemplates(Map.of(dataStream.getName(), indexTemplate)); + assertThat(dataStream.getEffectiveIndexTemplate(metadataBuilder.build()), equalTo(indexTemplate)); + } + + public void testGetEffectiveIndexTemplateDataStreamSettingsOnly() { + // We only have settings from the data stream, so we expect to get only those back in the effective template + Settings dataStreamSettings = randomSettings(); + DataStream dataStream = createDataStream(dataStreamSettings); + Settings templateSettings = Settings.EMPTY; + CompressedXContent templateMappings = randomMappings(); + Template.Builder templateBuilder = Template.builder().settings(templateSettings).mappings(templateMappings); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStream.getName())) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(templateBuilder) + .build(); + Metadata.Builder metadataBuilder = Metadata.builder().indexTemplates(Map.of(dataStream.getName(), indexTemplate)); + Template.Builder expectedTemplateBuilder = Template.builder().settings(dataStreamSettings).mappings(templateMappings); + ComposableIndexTemplate expectedEffectiveTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStream.getName())) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(expectedTemplateBuilder) + .build(); + assertThat(dataStream.getEffectiveIndexTemplate(metadataBuilder.build()), equalTo(expectedEffectiveTemplate)); + } + + public void testGetEffectiveIndexTemplate() { + // Here we have settings from both the template and the data stream, so we expect the data stream settings to take precedence + Settings dataStreamSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .put("index.setting3", (String) null) // This one gets removed from the effective settings + .build(); + DataStream dataStream = createDataStream(dataStreamSettings); + Settings templateSettings = Settings.builder() + .put("index.setting1", "templateValue") + .put("index.setting3", "templateValue") + .put("index.setting4", "templateValue") + .build(); + CompressedXContent templateMappings = randomMappings(); + Template.Builder templateBuilder = Template.builder().settings(templateSettings).mappings(templateMappings); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStream.getName())) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(templateBuilder) + .build(); + Metadata.Builder metadataBuilder = Metadata.builder().indexTemplates(Map.of(dataStream.getName(), indexTemplate)); + Settings mergedSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .put("index.setting4", "templateValue") + .build(); + Template.Builder expectedTemplateBuilder = Template.builder().settings(mergedSettings).mappings(templateMappings); + ComposableIndexTemplate expectedEffectiveTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStream.getName())) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(expectedTemplateBuilder) + .build(); + assertThat(dataStream.getEffectiveIndexTemplate(metadataBuilder.build()), equalTo(expectedEffectiveTemplate)); + } + + private DataStream createDataStream(Settings settings) { + DataStream dataStream = createTestInstance(); + return dataStream.copy().setSettings(settings).build(); + } + private record DataStreamMetadata(Long creationTimeInMillis, Long rolloverTimeInMillis, Long originationTimeInMillis) { public static DataStreamMetadata dataStreamMetadata(Long creationTimeInMillis, Long rolloverTimeInMillis) { return new DataStreamMetadata(creationTimeInMillis, rolloverTimeInMillis, null); diff --git a/server/src/test/java/org/elasticsearch/common/settings/SettingsTests.java b/server/src/test/java/org/elasticsearch/common/settings/SettingsTests.java index d608136fa564e..69928c29b977b 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/SettingsTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/SettingsTests.java @@ -729,4 +729,59 @@ public void testGlobValues() throws IOException { assertThat(values, contains("1")); } + public void testMergeNullOrEmptySettingsIntoEmptySettings() { + expectThrows(NullPointerException.class, () -> Settings.EMPTY.merge(null)); + assertThat(Settings.EMPTY.merge(Settings.EMPTY), equalTo(Settings.EMPTY)); + } + + public void testMergeEmptySettings() { + Settings.Builder builder = Settings.builder(); + for (int i = 1; i < randomInt(100); i++) { + builder.put(randomAlphanumericOfLength(20), randomAlphanumericOfLength(50)); + } + Settings settings = builder.build(); + assertThat(settings.merge(Settings.EMPTY), equalTo(settings)); + } + + public void testMergeNonEmptySettingsIntoEmptySettings() { + Settings.Builder builder = Settings.builder(); + for (int i = 1; i < randomInt(100); i++) { + builder.put(randomAlphanumericOfLength(20), randomAlphanumericOfLength(50)); + } + Settings newSettings = builder.build(); + assertThat(Settings.EMPTY.merge(newSettings), equalTo(newSettings)); + } + + public void testMergeNonEmptySettingsIntoNonEmptySettings() { + Settings settings = Settings.builder() + .put("index.setting1", "templateValue") + .put("index.setting3", "templateValue") + .put("index.setting4", "templateValue") + .build(); + Settings newSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .put("index.setting3", (String) null) // This one gets removed from the effective settings + .build(); + Settings mergedSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .put("index.setting4", "templateValue") + .build(); + assertThat(settings.merge(newSettings), equalTo(mergedSettings)); + } + + public void testMergeNonEmptySettingsWithNullIntoEmptySettings() { + Settings newSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .put("index.setting3", (String) null) // This one gets removed from the effective settings + .build(); + Settings mergedSettings = Settings.builder() + .put("index.setting1", "dataStreamValue") + .put("index.setting2", "dataStreamValue") + .build(); + assertThat(Settings.EMPTY.merge(newSettings), equalTo(mergedSettings)); + } + } diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java index d2fa30b743c76..7bc1b7b667d45 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java @@ -76,8 +76,10 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; import static org.elasticsearch.test.ESTestCase.generateRandomStringArray; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; +import static org.elasticsearch.test.ESTestCase.randomAlphanumericOfLength; import static org.elasticsearch.test.ESTestCase.randomBoolean; import static org.elasticsearch.test.ESTestCase.randomFrom; +import static org.elasticsearch.test.ESTestCase.randomInt; import static org.elasticsearch.test.ESTestCase.randomIntBetween; import static org.elasticsearch.test.ESTestCase.randomMap; import static org.elasticsearch.test.ESTestCase.randomMillisUpToYear9999; @@ -370,6 +372,7 @@ public static DataStream randomInstance(String dataStreamName, LongSupplier time dataStreamName, generation, metadata, + randomSettings(), system ? true : randomBoolean(), replicated, system, @@ -804,4 +807,15 @@ public static DataStreamOptions.Template createDataStreamOptionsTemplate(Boolean } return new DataStreamOptions.Template(DataStreamFailureStore.builder().enabled(failureStoreEnabled).buildTemplate()); } + + static Settings randomSettings() { + Settings.Builder builder = Settings.builder(); + if (randomBoolean()) { + return Settings.EMPTY; + } + for (int i = 1; i < randomInt(100); i++) { + builder.put(randomAlphanumericOfLength(20), randomAlphanumericOfLength(50)); + } + return builder.build(); + } } From ad77f8bdfb8a0c9c746066e6f0c2cfe2401c870b Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 25 Apr 2025 14:40:37 -0500 Subject: [PATCH 02/17] Using DataStream::getEffectiveSettings (#127282) --- .../action/TransportGetDataStreamsAction.java | 4 +- .../TransportGetDataStreamsActionTests.java | 122 +++++++++++++++++- .../MetadataCreateIndexServiceIT.java | 75 +++++++++++ .../rollover/MetadataRolloverService.java | 3 +- .../TransportSimulateIndexTemplateAction.java | 4 +- .../cluster/metadata/DataStream.java | 7 +- .../metadata/MetadataCreateIndexService.java | 67 +++++++--- .../MetadataRolloverServiceTests.java | 20 ++- 8 files changed, 273 insertions(+), 29 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceIT.java diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java index 0759feef71053..e7279d47a156a 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java @@ -238,7 +238,7 @@ static GetDataStreamAction.Response innerOperation( } else { indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false); if (indexTemplate != null) { - Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate); + Settings settings = dataStream.getEffectiveSettings(state.metadata()); ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME); if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) { indexMode = resolveMode( @@ -246,7 +246,7 @@ static GetDataStreamAction.Response innerOperation( indexSettingProviders, dataStream, settings, - state.metadata().templatesV2().get(indexTemplate) + dataStream.getEffectiveIndexTemplate(state.metadata()) ); } indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java index bcb3236d19c87..5d97986658bc2 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java @@ -11,13 +11,16 @@ import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings; import org.elasticsearch.cluster.metadata.DataStreamTestHelper; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; @@ -26,15 +29,20 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettingProviders; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESTestCase; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; +import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName; +import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createIndexMetadata; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.getClusterStateWithDataStreams; import static org.elasticsearch.test.LambdaMatchers.transformedItemsMatch; import static org.elasticsearch.test.LambdaMatchers.transformedMatch; @@ -312,9 +320,9 @@ public void testGetTimeSeriesMixedDataStream() { null ); - var name1 = DataStream.getDefaultBackingIndexName("ds-1", 1, instant.toEpochMilli()); - var name2 = DataStream.getDefaultBackingIndexName("ds-1", 2, instant.toEpochMilli()); - var name3 = DataStream.getDefaultBackingIndexName("ds-1", 3, twoHoursAgo.toEpochMilli()); + var name1 = getDefaultBackingIndexName("ds-1", 1, instant.toEpochMilli()); + var name2 = getDefaultBackingIndexName("ds-1", 2, instant.toEpochMilli()); + var name3 = getDefaultBackingIndexName("ds-1", 3, twoHoursAgo.toEpochMilli()); assertThat( response.getDataStreams(), contains( @@ -534,4 +542,112 @@ public void testProvidersAffectMode() { equalTo("standard") ); } + + public void testGetEffectiveSettingsTemplateOnlySettings() { + // Set a lifecycle only in the template, and make sure that is in the response: + GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); + final String templatePolicy = "templatePolicy"; + final String templateIndexMode = IndexMode.LOOKUP.getName(); + final String dataStreamPolicy = "dataStreamPolicy"; + final String dataStreamIndexMode = IndexMode.LOGSDB.getName(); + + ClusterState state = getClusterStateWithDataStreamWithSettings( + Settings.builder() + .put(IndexMetadata.LIFECYCLE_NAME, templatePolicy) + .put(IndexSettings.MODE.getKey(), templateIndexMode) + .build(), + Settings.EMPTY + ); + + GetDataStreamAction.Response response = TransportGetDataStreamsAction.innerOperation( + state, + req, + resolver, + systemIndices, + ClusterSettings.createBuiltInClusterSettings(), + dataStreamGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), + null + ); + assertNotNull(response.getDataStreams()); + assertThat(response.getDataStreams().size(), equalTo(1)); + assertThat(response.getDataStreams().get(0).getIlmPolicy(), equalTo(templatePolicy)); + assertThat(response.getDataStreams().get(0).getIndexModeName(), equalTo(templateIndexMode)); + } + + public void testGetEffectiveSettings() { + GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); + final String templatePolicy = "templatePolicy"; + final String templateIndexMode = IndexMode.LOOKUP.getName(); + final String dataStreamPolicy = "dataStreamPolicy"; + final String dataStreamIndexMode = IndexMode.LOGSDB.getName(); + // Now set a lifecycle in both the template and the data stream, and make sure the response has the data stream one: + ClusterState state = getClusterStateWithDataStreamWithSettings( + Settings.builder() + .put(IndexMetadata.LIFECYCLE_NAME, templatePolicy) + .put(IndexSettings.MODE.getKey(), templateIndexMode) + .build(), + Settings.builder() + .put(IndexMetadata.LIFECYCLE_NAME, dataStreamPolicy) + .put(IndexSettings.MODE.getKey(), dataStreamIndexMode) + .build() + ); + GetDataStreamAction.Response response = TransportGetDataStreamsAction.innerOperation( + state, + req, + resolver, + systemIndices, + ClusterSettings.createBuiltInClusterSettings(), + dataStreamGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), + null + ); + assertNotNull(response.getDataStreams()); + assertThat(response.getDataStreams().size(), equalTo(1)); + assertThat(response.getDataStreams().get(0).getIlmPolicy(), equalTo(dataStreamPolicy)); + assertThat(response.getDataStreams().get(0).getIndexModeName(), equalTo(dataStreamIndexMode)); + } + + private static ClusterState getClusterStateWithDataStreamWithSettings(Settings templateSettings, Settings dataStreamSettings) { + String dataStreamName = "data-stream-1"; + int numberOfBackingIndices = randomIntBetween(1, 5); + long currentTime = System.currentTimeMillis(); + int replicas = 0; + boolean replicated = false; + Metadata.Builder builder = Metadata.builder(); + builder.put( + "template_1", + ComposableIndexTemplate.builder() + .indexPatterns(List.of("*")) + .template(Template.builder().settings(templateSettings)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build() + ); + + List backingIndices = new ArrayList<>(); + for (int backingIndexNumber = 1; backingIndexNumber <= numberOfBackingIndices; backingIndexNumber++) { + backingIndices.add( + createIndexMetadata( + getDefaultBackingIndexName(dataStreamName, backingIndexNumber, currentTime), + true, + templateSettings, + replicas + ) + ); + } + List allIndices = new ArrayList<>(backingIndices); + + DataStream ds = DataStream.builder( + dataStreamName, + backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()) + ).setGeneration(numberOfBackingIndices).setSettings(dataStreamSettings).setReplicated(replicated).build(); + builder.put(ds); + + for (IndexMetadata index : allIndices) { + builder.put(index, false); + } + return ClusterState.builder(new ClusterName("_name")).metadata(builder.build()).build(); + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceIT.java new file mode 100644 index 0000000000000..adbd764c00adc --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceIT.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.support.master.ShardsAcknowledgedResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.Locale; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.equalTo; + +public class MetadataCreateIndexServiceIT extends ESIntegTestCase { + + public void testRequestTemplateIsRespected() throws InterruptedException { + /* + * This test passes a template in the CreateIndexClusterStateUpdateRequest, and makes sure that the settings from that template + * are used when creating the index. + */ + MetadataCreateIndexService metadataCreateIndexService = internalCluster().getCurrentMasterNodeInstance( + MetadataCreateIndexService.class + ); + final String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + final int numberOfReplicas = randomIntBetween(1, 7); + CreateIndexClusterStateUpdateRequest request = new CreateIndexClusterStateUpdateRequest( + "testRequestTemplateIsRespected", + indexName, + randomAlphaOfLength(20) + ); + request.setMatchingTemplate( + ComposableIndexTemplate.builder() + .template(Template.builder().settings(Settings.builder().put("index.number_of_replicas", numberOfReplicas))) + .build() + ); + final CountDownLatch listenerCalledLatch = new CountDownLatch(1); + ActionListener listener = new ActionListener<>() { + @Override + public void onResponse(ShardsAcknowledgedResponse shardsAcknowledgedResponse) { + listenerCalledLatch.countDown(); + } + + @Override + public void onFailure(Exception e) { + logger.error(e); + listenerCalledLatch.countDown(); + } + }; + + metadataCreateIndexService.createIndex( + TimeValue.THIRTY_SECONDS, + TimeValue.THIRTY_SECONDS, + TimeValue.THIRTY_SECONDS, + request, + listener + ); + listenerCalledLatch.await(10, TimeUnit.SECONDS); + GetIndexResponse response = admin().indices().getIndex(new GetIndexRequest().indices(indexName)).actionGet(); + Settings settings = response.getSettings().get(indexName); + assertThat(settings.get("index.number_of_replicas"), equalTo(Integer.toString(numberOfReplicas))); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index b5e30701c75a7..07acefae6e9d3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -63,7 +63,6 @@ import static org.elasticsearch.cluster.metadata.IndexAbstraction.Type.ALIAS; import static org.elasticsearch.cluster.metadata.IndexAbstraction.Type.DATA_STREAM; -import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.lookupTemplateForDataStream; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template; import static org.elasticsearch.cluster.routing.allocation.allocator.AllocationActionListener.rerouteCompletionIsNotRequired; @@ -312,7 +311,7 @@ private RolloverResult rolloverDataStream( final SystemDataStreamDescriptor systemDataStreamDescriptor; if (dataStream.isSystem() == false) { systemDataStreamDescriptor = null; - templateV2 = lookupTemplateForDataStream(dataStreamName, metadata); + templateV2 = dataStream.getEffectiveIndexTemplate(currentState.metadata()); } else { systemDataStreamDescriptor = systemIndices.findMatchingDataStreamDescriptor(dataStreamName); if (systemDataStreamDescriptor == null) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 3906969403cf9..f50fd1932dc62 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -261,8 +261,8 @@ public static Template resolveTemplate( : indexName; List mappings = MetadataCreateIndexService.collectV2Mappings( null, // empty request mapping as the user can't specify any explicit mappings via the simulate api - simulatedState, - matchingTemplate, + simulatedState.metadata(), + template, xContentRegistry, simulatedIndexName ); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 5c095a302b228..18d2ebb7e5313 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -376,7 +376,12 @@ public ComposableIndexTemplate getEffectiveIndexTemplate(Metadata metadata) { public Settings getEffectiveSettings(Metadata metadata) { ComposableIndexTemplate template = getMatchingIndexTemplate(metadata); - Settings templateSettings = template.template() == null ? Settings.EMPTY : template.template().settings(); + final Settings templateSettings; + if (template.template() == null || template.template().settings() == null) { + templateSettings = Settings.EMPTY; + } else { + templateSettings = template.template().settings(); + } return templateSettings.merge(settings); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 5c8f4451ecf4a..ebe4ae300e59e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -410,6 +410,18 @@ public ClusterState applyCreateIndexRequest( ? IndexMetadata.INDEX_HIDDEN_SETTING.get(request.settings()) : null; + ComposableIndexTemplate templateFromRequest = request.matchingTemplate(); + if (templateFromRequest != null) { + return applyCreateIndexRequestWithV2Template( + currentState, + request, + silent, + templateFromRequest, + metadataTransformer, + rerouteListener + ); + } + // Check to see if a v2 template matched final String v2Template = MetadataIndexTemplateService.findV2Template( currentState.metadata(), @@ -657,7 +669,8 @@ private ClusterState applyCreateIndexRequestWithV2Template( ) throws Exception { logger.debug("applying create index request using composable template [{}]", templateName); - ComposableIndexTemplate template = currentState.getMetadata().templatesV2().get(templateName); + final Metadata metadata = currentState.getMetadata(); + ComposableIndexTemplate template = metadata.templatesV2().get(templateName); final boolean isDataStream = template.getDataStreamTemplate() != null; if (isDataStream && request.dataStreamName() == null) { throw new IllegalArgumentException( @@ -669,18 +682,34 @@ private ClusterState applyCreateIndexRequestWithV2Template( + "use create data stream api instead" ); } + return applyCreateIndexRequestWithV2Template(currentState, request, silent, template, metadataTransformer, rerouteListener); + } + + private ClusterState applyCreateIndexRequestWithV2Template( + final ClusterState currentState, + final CreateIndexClusterStateUpdateRequest request, + final boolean silent, + final ComposableIndexTemplate template, + final BiConsumer metadataTransformer, + final ActionListener rerouteListener + ) throws Exception { + + final Metadata metadata = currentState.getMetadata(); + final RoutingTable routingTable = currentState.routingTable(); + + final boolean isDataStream = template.getDataStreamTemplate() != null; final List mappings = collectV2Mappings( request.mappings(), - currentState, - templateName, + metadata, + template, xContentRegistry, request.index() ); final Settings aggregatedIndexSettings = aggregateIndexSettings( currentState, request, - resolveSettings(currentState.metadata(), templateName), + resolveSettings(template, metadata.componentTemplates()), mappings, null, settings, @@ -702,7 +731,7 @@ private ClusterState applyCreateIndexRequestWithV2Template( request.index(), // data stream aliases are created separately in MetadataCreateDataStreamService::createDataStream isDataStream ? Set.of() : request.aliases(), - isDataStream ? List.of() : MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName), + isDataStream ? List.of() : MetadataIndexTemplateService.resolveAliases(currentState.metadata(), template), currentState.metadata(), xContentRegistry, // the context is used ony for validation so it's fine to pass fake values for the shard id and the current timestamp @@ -710,7 +739,7 @@ private ClusterState applyCreateIndexRequestWithV2Template( IndexService.dateMathExpressionResolverAt(request.getNameResolvedAt()), systemIndices::isSystemName ), - Collections.singletonList(templateName), + Collections.singletonList("provided in request"), metadataTransformer, rerouteListener ); @@ -837,17 +866,6 @@ private static List collectSystemV2Mappings( return collectV2Mappings(null, templateMappings, xContentRegistry); } - public static List collectV2Mappings( - @Nullable final String requestMappings, - final ClusterState currentState, - final String templateName, - final NamedXContentRegistry xContentRegistry, - final String indexName - ) throws Exception { - List templateMappings = MetadataIndexTemplateService.collectMappings(currentState, templateName, indexName); - return collectV2Mappings(requestMappings, templateMappings, xContentRegistry); - } - private static List collectV2Mappings( @Nullable final String requestMappings, final List templateMappings, @@ -864,6 +882,21 @@ private static List collectV2Mappings( return result; } + public static List collectV2Mappings( + @Nullable final String requestMappings, + final Metadata metadata, + final ComposableIndexTemplate template, + final NamedXContentRegistry xContentRegistry, + final String indexName + ) throws Exception { + List templateMappings = MetadataIndexTemplateService.collectMappings( + template, + metadata.componentTemplates(), + indexName + ); + return collectV2Mappings(requestMappings, templateMappings, xContentRegistry); + } + private ClusterState applyCreateIndexRequestWithExistingMetadata( final ClusterState currentState, final CreateIndexClusterStateUpdateRequest request, diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 729b1734fea22..6550d18fb1adc 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -63,6 +63,8 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -638,7 +640,14 @@ public void testRolloverClusterStateForDataStream() throws Exception { false ); long after = testThreadPool.absoluteTimeInMillis(); - + Settings rolledOverIndexSettings = rolloverResult.clusterState() + .metadata() + .index(rolloverResult.rolloverIndexName()) + .getSettings(); + Set rolledOverIndexSettingNames = rolledOverIndexSettings.keySet(); + for (String settingName : dataStream.getEffectiveSettings(clusterState.metadata()).keySet()) { + assertTrue(rolledOverIndexSettingNames.contains(settingName)); + } String newIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1); assertEquals(sourceIndexName, rolloverResult.sourceIndexName()); assertEquals(newIndexName, rolloverResult.rolloverIndexName()); @@ -708,7 +717,14 @@ public void testRolloverClusterStateForDataStreamFailureStore() throws Exception true ); long after = testThreadPool.absoluteTimeInMillis(); - + Settings rolledOverIndexSettings = rolloverResult.clusterState() + .metadata() + .index(rolloverResult.rolloverIndexName()) + .getSettings(); + Set rolledOverIndexSettingNames = rolledOverIndexSettings.keySet(); + for (String settingName : dataStream.getSettings().keySet()) { + assertFalse(rolledOverIndexSettingNames.contains(settingName)); + } var epochMillis = System.currentTimeMillis(); String sourceIndexName = DataStream.getDefaultFailureStoreName(dataStream.getName(), dataStream.getGeneration(), epochMillis); String newIndexName = DataStream.getDefaultFailureStoreName(dataStream.getName(), dataStream.getGeneration() + 1, epochMillis); From 8bc833776545f5afd691da5ad7a033bbba28476f Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Mon, 28 Apr 2025 10:46:20 -0500 Subject: [PATCH 03/17] Adding transport actions for getting and updating data stream settings (#127417) --- .../datastreams/DataStreamSettingsIT.java | 351 ++++++++++++++++++ .../datastreams/DataStreamsPlugin.java | 6 + .../TransportGetDataStreamSettingsAction.java | 90 +++++ ...ansportUpdateDataStreamSettingsAction.java | 345 +++++++++++++++++ .../GetDataStreamSettingsAction.java | 149 ++++++++ .../UpdateDataStreamSettingsAction.java | 262 +++++++++++++ .../metadata/MetadataDataStreamsService.java | 73 ++++ .../MetadataIndexTemplateService.java | 3 +- .../GetDataStreamSettingsActionTests.java | 107 ++++++ ...eDataStreamSettingsActionRequestTests.java | 74 ++++ ...DataStreamSettingsActionResponseTests.java | 213 +++++++++++ .../xpack/security/operator/Constants.java | 2 + 12 files changed, 1673 insertions(+), 2 deletions(-) create mode 100644 modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamSettingsIT.java create mode 100644 modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamSettingsAction.java create mode 100644 modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java create mode 100644 server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamSettingsAction.java create mode 100644 server/src/main/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsAction.java create mode 100644 server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamSettingsActionTests.java create mode 100644 server/src/test/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsActionRequestTests.java create mode 100644 server/src/test/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsActionResponseTests.java diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamSettingsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamSettingsIT.java new file mode 100644 index 0000000000000..33ff3f4ffb1b6 --- /dev/null +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamSettingsIT.java @@ -0,0 +1,351 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datastreams; + +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.admin.indices.readonly.AddIndexBlockRequest; +import org.elasticsearch.action.admin.indices.readonly.AddIndexBlockResponse; +import org.elasticsearch.action.admin.indices.rollover.RolloverAction; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; +import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.datastreams.CreateDataStreamAction; +import org.elasticsearch.action.datastreams.GetDataStreamSettingsAction; +import org.elasticsearch.action.datastreams.UpdateDataStreamSettingsAction; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.transport.MockTransportService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; + +public class DataStreamSettingsIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return List.of(DataStreamsPlugin.class, MockTransportService.TestPlugin.class, TestPlugin.class); + } + + public void testPutDataStreamSettings() throws Exception { + String dataStreamName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + putComposableIndexTemplate("my-template", List.of(dataStreamName), indexSettings(1, 0).build()); + final var createDataStreamRequest = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet()); + final int numberOfShards = randomIntBetween(2, 7); + final String newLifecycleName = randomAlphanumericOfLength(20).toLowerCase(Locale.ROOT); + { + List getSettingsResponses = client().execute( + GetDataStreamSettingsAction.INSTANCE, + new GetDataStreamSettingsAction.Request(TimeValue.THIRTY_SECONDS).indices(dataStreamName) + ).actionGet().getDataStreamSettingsResponses(); + assertThat(getSettingsResponses.size(), equalTo(1)); + assertThat(getSettingsResponses.get(0).settings(), equalTo(Settings.EMPTY)); + Settings dataStreamSettings = Settings.builder() + .put("index.number_of_shards", numberOfShards) + .put("index.lifecycle.name", newLifecycleName) + .build(); + UpdateDataStreamSettingsAction.Response putSettingsResponse = client().execute( + UpdateDataStreamSettingsAction.INSTANCE, + new UpdateDataStreamSettingsAction.Request(dataStreamSettings, TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS).indices( + dataStreamName + ) + ).actionGet(); + List dataStreamSettingsResponses = putSettingsResponse + .getDataStreamSettingsResponses(); + assertThat(dataStreamSettingsResponses.size(), equalTo(1)); + UpdateDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse = dataStreamSettingsResponses.get(0); + assertThat(dataStreamSettingsResponse.dataStreamName(), equalTo(dataStreamName)); + assertThat(dataStreamSettingsResponse.dataStreamSucceeded(), equalTo(true)); + assertThat(dataStreamSettingsResponse.settings().get("index.number_of_shards"), equalTo(Integer.toString(numberOfShards))); + assertThat( + dataStreamSettingsResponse.effectiveSettings().get("index.number_of_shards"), + equalTo(Integer.toString(numberOfShards)) + ); + assertThat(dataStreamSettingsResponse.indicesSettingsResult().indexSettingErrors().size(), equalTo(0)); + assertThat(dataStreamSettingsResponse.indicesSettingsResult().appliedToDataStreamOnly().size(), equalTo(1)); + assertThat( + dataStreamSettingsResponse.indicesSettingsResult().appliedToDataStreamOnly().get(0), + equalTo("index.number_of_shards") + ); + assertThat(dataStreamSettingsResponse.indicesSettingsResult().appliedToDataStreamAndBackingIndices().size(), equalTo(1)); + assertThat( + dataStreamSettingsResponse.indicesSettingsResult().appliedToDataStreamAndBackingIndices().get(0), + equalTo("index.lifecycle.name") + ); + GetIndexResponse response = admin().indices().getIndex(new GetIndexRequest().indices(dataStreamName)).actionGet(); + Settings settings = response.getSettings().values().iterator().next(); + assertThat(settings.get("index.number_of_shards"), equalTo("1")); + assertThat(settings.get("index.lifecycle.name"), equalTo(newLifecycleName)); + getSettingsResponses = client().execute( + GetDataStreamSettingsAction.INSTANCE, + new GetDataStreamSettingsAction.Request(TimeValue.THIRTY_SECONDS).indices(dataStreamName) + ).actionGet().getDataStreamSettingsResponses(); + assertThat(getSettingsResponses.size(), equalTo(1)); + assertThat(getSettingsResponses.get(0).settings(), equalTo(dataStreamSettings)); + assertThat( + getSettingsResponses.get(0).effectiveSettings(), + equalTo(Settings.builder().put(dataStreamSettings).put("index.number_of_replicas", "0").build()) + ); + } + { + // Try to set an invalid value for a valid setting, and make sure the data stream is not updated + int invalidNumberOfShards = 2000; + UpdateDataStreamSettingsAction.Response putSettingsResponse = client().execute( + UpdateDataStreamSettingsAction.INSTANCE, + new UpdateDataStreamSettingsAction.Request( + Settings.builder().put("index.number_of_shards", invalidNumberOfShards).build(), + TimeValue.THIRTY_SECONDS, + TimeValue.THIRTY_SECONDS + ).indices(dataStreamName) + ).actionGet(); + List dataStreamSettingsResponses = putSettingsResponse + .getDataStreamSettingsResponses(); + assertThat(dataStreamSettingsResponses.size(), equalTo(1)); + UpdateDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse = dataStreamSettingsResponses.get(0); + assertThat(dataStreamSettingsResponse.dataStreamName(), equalTo(dataStreamName)); + assertThat(dataStreamSettingsResponse.dataStreamSucceeded(), equalTo(false)); + } + { + // Try to set an invalid setting, and make sure the data stream is not updated + UpdateDataStreamSettingsAction.Response putSettingsResponse = client().execute( + UpdateDataStreamSettingsAction.INSTANCE, + new UpdateDataStreamSettingsAction.Request( + Settings.builder() + .put("index.number_of_shards", numberOfShards) + .put("unknown.setting", randomAlphaOfLength(20)) + .build(), + TimeValue.THIRTY_SECONDS, + TimeValue.THIRTY_SECONDS + ).indices(dataStreamName) + ).actionGet(); + List dataStreamSettingsResponses = putSettingsResponse + .getDataStreamSettingsResponses(); + assertThat(dataStreamSettingsResponses.size(), equalTo(1)); + UpdateDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse = dataStreamSettingsResponses.get(0); + assertThat(dataStreamSettingsResponse.dataStreamName(), equalTo(dataStreamName)); + assertThat(dataStreamSettingsResponse.dataStreamSucceeded(), equalTo(false)); + } + } + + public void testPutMultipleDataStreamSettings() throws Exception { + List testDataStreamNames = new ArrayList<>(); + List ignoredDataStreamNames = new ArrayList<>(); + for (int i = 0; i < randomIntBetween(2, 5); i++) { + String dataStreamName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + putComposableIndexTemplate("my-template-" + i, List.of(dataStreamName), indexSettings(1, 0).build()); + final var createDataStreamRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + dataStreamName + ); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet()); + testDataStreamNames.add(dataStreamName); + } + for (int i = 0; i < randomIntBetween(2, 5); i++) { + String dataStreamName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + putComposableIndexTemplate("my-other-template-" + i, List.of(dataStreamName), indexSettings(1, 0).build()); + final var createDataStreamRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + dataStreamName + ); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet()); + ignoredDataStreamNames.add(dataStreamName); + } + final int numberOfShards = randomIntBetween(2, 7); + final String newLifecycleName = randomAlphanumericOfLength(20).toLowerCase(Locale.ROOT); + { + { + // First, a quick check that we fetch all data streams when no data stream names are given: + UpdateDataStreamSettingsAction.Response putSettingsResponse = client().execute( + UpdateDataStreamSettingsAction.INSTANCE, + new UpdateDataStreamSettingsAction.Request( + Settings.builder() + .put("index.number_of_shards", numberOfShards) + .put("index.lifecycle.name", newLifecycleName) + .build(), + TimeValue.THIRTY_SECONDS, + TimeValue.THIRTY_SECONDS + ) + ).actionGet(); + List dataStreamSettingsResponses = putSettingsResponse + .getDataStreamSettingsResponses(); + assertThat(dataStreamSettingsResponses.size(), equalTo(testDataStreamNames.size() + ignoredDataStreamNames.size())); + } + UpdateDataStreamSettingsAction.Response putSettingsResponse = client().execute( + UpdateDataStreamSettingsAction.INSTANCE, + new UpdateDataStreamSettingsAction.Request( + Settings.builder().put("index.number_of_shards", numberOfShards).put("index.lifecycle.name", newLifecycleName).build(), + TimeValue.THIRTY_SECONDS, + TimeValue.THIRTY_SECONDS + ).indices(testDataStreamNames.toArray(new String[0])) + ).actionGet(); + List dataStreamSettingsResponses = putSettingsResponse + .getDataStreamSettingsResponses(); + assertThat(dataStreamSettingsResponses.size(), equalTo(testDataStreamNames.size())); + for (int i = 0; i < testDataStreamNames.size(); i++) { + UpdateDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse = dataStreamSettingsResponses.get(0); + assertThat(dataStreamSettingsResponse.dataStreamSucceeded(), equalTo(true)); + assertThat(dataStreamSettingsResponse.settings().get("index.number_of_shards"), equalTo(Integer.toString(numberOfShards))); + assertThat( + dataStreamSettingsResponse.effectiveSettings().get("index.number_of_shards"), + equalTo(Integer.toString(numberOfShards)) + ); + assertThat(dataStreamSettingsResponse.indicesSettingsResult().indexSettingErrors().size(), equalTo(0)); + assertThat(dataStreamSettingsResponse.indicesSettingsResult().appliedToDataStreamOnly().size(), equalTo(1)); + assertThat( + dataStreamSettingsResponse.indicesSettingsResult().appliedToDataStreamOnly().get(0), + equalTo("index.number_of_shards") + ); + assertThat(dataStreamSettingsResponse.indicesSettingsResult().appliedToDataStreamAndBackingIndices().size(), equalTo(1)); + assertThat( + dataStreamSettingsResponse.indicesSettingsResult().appliedToDataStreamAndBackingIndices().get(0), + equalTo("index.lifecycle.name") + ); + GetIndexResponse response = admin().indices() + .getIndex(new GetIndexRequest().indices(dataStreamSettingsResponse.dataStreamName())) + .actionGet(); + Settings settings = response.getSettings().values().iterator().next(); + assertThat(settings.get("index.number_of_shards"), equalTo("1")); + assertThat(settings.get("index.lifecycle.name"), equalTo(newLifecycleName)); + } + List getSettingsResponses = client().execute( + GetDataStreamSettingsAction.INSTANCE, + new GetDataStreamSettingsAction.Request(TimeValue.THIRTY_SECONDS).indices(testDataStreamNames.toArray(new String[0])) + ).actionGet().getDataStreamSettingsResponses(); + assertThat(getSettingsResponses.size(), equalTo(testDataStreamNames.size())); + } + } + + public void testRolloverWithDataStreamSettings() throws Exception { + String dataStreamName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + putComposableIndexTemplate("my-template", List.of(dataStreamName), indexSettings(1, 0).build()); + final var createDataStreamRequest = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet()); + final int numberOfShards = randomIntBetween(2, 7); + final String newLifecycleName = randomAlphanumericOfLength(20).toLowerCase(Locale.ROOT); + client().execute( + UpdateDataStreamSettingsAction.INSTANCE, + new UpdateDataStreamSettingsAction.Request( + Settings.builder().put("index.number_of_shards", numberOfShards).put("index.lifecycle.name", newLifecycleName).build(), + TimeValue.THIRTY_SECONDS, + TimeValue.THIRTY_SECONDS + ).indices(dataStreamName) + ).actionGet(); + + RolloverResponse rolloverResponse = client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)) + .actionGet(); + assertThat(rolloverResponse.isRolledOver(), equalTo(true)); + String newIndexName = rolloverResponse.getNewIndex(); + GetIndexResponse response = admin().indices().getIndex(new GetIndexRequest().indices(newIndexName)).actionGet(); + Settings settings = response.getSettings().get(newIndexName); + assertThat(settings.get("index.number_of_shards"), equalTo(Integer.toString(numberOfShards))); + assertThat(settings.get("index.lifecycle.name"), equalTo(newLifecycleName)); + } + + public void testIndexBlock() throws Exception { + /* + * This tests that if there is a block on one index, the settings changes still go through on all the other + * indices. + */ + String dataStreamName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + putComposableIndexTemplate("my-template", List.of(dataStreamName), indexSettings(1, 0).build()); + final var createDataStreamRequest = new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet()); + Set indexNames = new HashSet<>(); + for (int i = 0; i < randomIntBetween(1, 10); i++) { + RolloverResponse rolloverResponse = client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)) + .actionGet(); + assertThat(rolloverResponse.isRolledOver(), equalTo(true)); + indexNames.add(rolloverResponse.getOldIndex()); + } + String indexToBlock = randomFrom(indexNames); + PlainActionFuture plainActionFuture = new PlainActionFuture<>(); + indicesAdmin().addBlock(new AddIndexBlockRequest(IndexMetadata.APIBlock.METADATA, indexToBlock), plainActionFuture); + assertThat(plainActionFuture.get().isShardsAcknowledged(), equalTo(true)); + final int numberOfShards = randomIntBetween(2, 7); + final String newLifecycleName = randomAlphanumericOfLength(20).toLowerCase(Locale.ROOT); + { + UpdateDataStreamSettingsAction.Response putSettingsResponse = client().execute( + UpdateDataStreamSettingsAction.INSTANCE, + new UpdateDataStreamSettingsAction.Request( + Settings.builder().put("index.number_of_shards", numberOfShards).put("index.lifecycle.name", newLifecycleName).build(), + TimeValue.THIRTY_SECONDS, + TimeValue.THIRTY_SECONDS + ).indices(dataStreamName) + ).actionGet(); + List dataStreamSettingsResponses = putSettingsResponse + .getDataStreamSettingsResponses(); + UpdateDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse = dataStreamSettingsResponses.get(0); + assertThat(dataStreamSettingsResponse.dataStreamSucceeded(), equalTo(true)); + indicesAdmin().prepareUpdateSettings(indexToBlock) + .setSettings(Settings.builder().put("index.blocks.metadata", false).build()) + .get(); + GetIndexResponse response = admin().indices() + .getIndex(new GetIndexRequest().indices(dataStreamSettingsResponse.dataStreamName())) + .actionGet(); + for (Map.Entry indexAndsettings : response.getSettings().entrySet()) { + if (indexAndsettings.getKey().equals(indexToBlock)) { + assertThat(indexAndsettings.getValue().get("index.lifecycle.name"), equalTo(null)); + } else { + assertThat(indexAndsettings.getValue().get("index.lifecycle.name"), equalTo(newLifecycleName)); + } + } + } + } + + static void putComposableIndexTemplate(String id, List patterns, @Nullable Settings settings) throws IOException { + TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request(id); + request.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(patterns) + .template(Template.builder().settings(settings)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build() + ); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + } + + public static class TestPlugin extends Plugin { + /* + * index.lifecycle.name is one of the settings allowed by TransportUpdateDataStreamSettingsAction, but it is in the ilm plugin. We + * add it here so that it is available for testing. + */ + public static final Setting LIFECYCLE_SETTING = Setting.simpleString( + "index.lifecycle.name", + "", + Setting.Property.IndexScope + ); + + @Override + public List> getSettings() { + return List.of(LIFECYCLE_SETTING); + } + } + +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java index f21237f564fb5..d39fea0435d8b 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java @@ -16,10 +16,12 @@ import org.elasticsearch.action.datastreams.DataStreamsStatsAction; import org.elasticsearch.action.datastreams.DeleteDataStreamAction; import org.elasticsearch.action.datastreams.GetDataStreamAction; +import org.elasticsearch.action.datastreams.GetDataStreamSettingsAction; import org.elasticsearch.action.datastreams.MigrateToDataStreamAction; import org.elasticsearch.action.datastreams.ModifyDataStreamsAction; import org.elasticsearch.action.datastreams.PromoteDataStreamAction; import org.elasticsearch.action.datastreams.PutDataStreamOptionsAction; +import org.elasticsearch.action.datastreams.UpdateDataStreamSettingsAction; import org.elasticsearch.action.datastreams.lifecycle.ExplainDataStreamLifecycleAction; import org.elasticsearch.action.datastreams.lifecycle.GetDataStreamLifecycleAction; import org.elasticsearch.action.datastreams.lifecycle.PutDataStreamLifecycleAction; @@ -37,10 +39,12 @@ import org.elasticsearch.datastreams.action.TransportCreateDataStreamAction; import org.elasticsearch.datastreams.action.TransportDataStreamsStatsAction; import org.elasticsearch.datastreams.action.TransportDeleteDataStreamAction; +import org.elasticsearch.datastreams.action.TransportGetDataStreamSettingsAction; import org.elasticsearch.datastreams.action.TransportGetDataStreamsAction; import org.elasticsearch.datastreams.action.TransportMigrateToDataStreamAction; import org.elasticsearch.datastreams.action.TransportModifyDataStreamsAction; import org.elasticsearch.datastreams.action.TransportPromoteDataStreamAction; +import org.elasticsearch.datastreams.action.TransportUpdateDataStreamSettingsAction; import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleErrorStore; import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService; import org.elasticsearch.datastreams.lifecycle.action.DeleteDataStreamLifecycleAction; @@ -241,6 +245,8 @@ public Collection createComponents(PluginServices services) { actions.add(new ActionHandler<>(GetDataStreamOptionsAction.INSTANCE, TransportGetDataStreamOptionsAction.class)); actions.add(new ActionHandler<>(PutDataStreamOptionsAction.INSTANCE, TransportPutDataStreamOptionsAction.class)); actions.add(new ActionHandler<>(DeleteDataStreamOptionsAction.INSTANCE, TransportDeleteDataStreamOptionsAction.class)); + actions.add(new ActionHandler<>(GetDataStreamSettingsAction.INSTANCE, TransportGetDataStreamSettingsAction.class)); + actions.add(new ActionHandler<>(UpdateDataStreamSettingsAction.INSTANCE, TransportUpdateDataStreamSettingsAction.class)); return actions; } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamSettingsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamSettingsAction.java new file mode 100644 index 0000000000000..223887b5f6987 --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamSettingsAction.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datastreams.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.datastreams.GetDataStreamSettingsAction; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class TransportGetDataStreamSettingsAction extends TransportMasterNodeReadAction< + GetDataStreamSettingsAction.Request, + GetDataStreamSettingsAction.Response> { + private final IndexNameExpressionResolver indexNameExpressionResolver; + private final SettingsFilter settingsFilter; + + @Inject + public TransportGetDataStreamSettingsAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + SettingsFilter settingsFilter, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver + ) { + super( + GetDataStreamSettingsAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + GetDataStreamSettingsAction.Request::localOnly, + GetDataStreamSettingsAction.Response::localOnly, + threadPool.executor(ThreadPool.Names.MANAGEMENT) + ); + this.indexNameExpressionResolver = indexNameExpressionResolver; + this.settingsFilter = settingsFilter; + } + + @Override + protected ClusterBlockException checkBlock(GetDataStreamSettingsAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + @Override + protected void masterOperation( + Task task, + GetDataStreamSettingsAction.Request request, + ClusterState state, + ActionListener listener + ) throws Exception { + List dataStreamNames = indexNameExpressionResolver.dataStreamNames( + clusterService.state(), + IndicesOptions.DEFAULT, + request.indices() + ); + Map dataStreamMap = state.metadata().dataStreams(); + List responseList = new ArrayList<>(dataStreamNames.size()); + for (String dataStreamName : dataStreamNames) { + DataStream dataStream = dataStreamMap.get(dataStreamName); + Settings settings = settingsFilter.filter(dataStream.getSettings()); + Settings effectiveSettings = settingsFilter.filter(dataStream.getEffectiveSettings(state.metadata())); + responseList.add(new GetDataStreamSettingsAction.DataStreamSettingsResponse(dataStreamName, settings, effectiveSettings)); + } + listener.onResponse(new GetDataStreamSettingsAction.Response(responseList)); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java new file mode 100644 index 0000000000000..9cd8f1145bb58 --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java @@ -0,0 +1,345 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsClusterStateUpdateRequest; +import org.elasticsearch.action.datastreams.UpdateDataStreamSettingsAction; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.CountDownActionListener; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetadataDataStreamsService; +import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.Index; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.common.settings.Settings.EMPTY; +import static org.elasticsearch.common.settings.Settings.builder; + +public class TransportUpdateDataStreamSettingsAction extends TransportMasterNodeAction< + UpdateDataStreamSettingsAction.Request, + UpdateDataStreamSettingsAction.Response> { + private static final Logger logger = LogManager.getLogger(TransportUpdateDataStreamSettingsAction.class); + private static final Set APPLY_TO_BACKING_INDICES = Set.of("index.lifecycle.name"); + private static final Set APPLY_TO_DATA_STREAM_ONLY = Set.of("index.number_of_shards"); + private final MetadataDataStreamsService metadataDataStreamsService; + private final MetadataUpdateSettingsService updateSettingsService; + private final IndexNameExpressionResolver indexNameExpressionResolver; + private final SystemIndices systemIndices; + private final SettingsFilter settingsFilter; + + @Inject + public TransportUpdateDataStreamSettingsAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + MetadataDataStreamsService metadataDataStreamsService, + MetadataUpdateSettingsService updateSettingsService, + IndexNameExpressionResolver indexNameExpressionResolver, + SystemIndices systemIndices, + SettingsFilter settingsFilter + ) { + super( + UpdateDataStreamSettingsAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + UpdateDataStreamSettingsAction.Request::new, + UpdateDataStreamSettingsAction.Response::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.metadataDataStreamsService = metadataDataStreamsService; + this.updateSettingsService = updateSettingsService; + this.indexNameExpressionResolver = indexNameExpressionResolver; + this.systemIndices = systemIndices; + this.settingsFilter = settingsFilter; + } + + @Override + protected void masterOperation( + Task task, + UpdateDataStreamSettingsAction.Request request, + ClusterState state, + ActionListener listener + ) throws Exception { + List dataStreamNames = indexNameExpressionResolver.dataStreamNames( + clusterService.state(), + IndicesOptions.DEFAULT, + request.indices() + ); + List dataStreamSettingsResponse = new ArrayList<>(); + CountDownActionListener countDownListener = new CountDownActionListener(dataStreamNames.size() + 1, new ActionListener<>() { + @Override + public void onResponse(Void unused) { + listener.onResponse(new UpdateDataStreamSettingsAction.Response(dataStreamSettingsResponse)); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + countDownListener.onResponse(null); + for (String dataStreamName : dataStreamNames) { + updateSingleDataStream( + dataStreamName, + request.getSettings(), + request.masterNodeTimeout(), + request.ackTimeout(), + new ActionListener<>() { + @Override + public void onResponse(UpdateDataStreamSettingsAction.DataStreamSettingsResponse dataStreamResponse) { + dataStreamSettingsResponse.add(dataStreamResponse); + countDownListener.onResponse(null); + } + + @Override + public void onFailure(Exception e) { + dataStreamSettingsResponse.add( + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( + dataStreamName, + false, + e.getMessage(), + EMPTY, + EMPTY, + UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult.EMPTY + ) + ); + countDownListener.onResponse(null); + } + } + ); + } + } + + private void updateSingleDataStream( + String dataStreamName, + Settings settingsOverrides, + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + ActionListener listener + ) { + logger.debug("updating settings for {}", dataStreamName); + Set settingsToReject = new HashSet<>(); + for (String settingName : settingsOverrides.keySet()) { + if (APPLY_TO_BACKING_INDICES.contains(settingName) == false && APPLY_TO_DATA_STREAM_ONLY.contains(settingName) == false) { + settingsToReject.add(settingName); + } + } + if (settingsToReject.isEmpty() == false) { + listener.onResponse( + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( + dataStreamName, + false, + Strings.format("Cannot set the following settings on a data stream: [%s]", String.join(",", settingsToReject)), + EMPTY, + EMPTY, + UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult.EMPTY + ) + ); + return; + } + + if (systemIndices.isSystemDataStream(dataStreamName)) { + listener.onResponse( + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( + dataStreamName, + false, + "Cannot update a system data stream", + EMPTY, + EMPTY, + UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult.EMPTY + ) + ); + return; + } + metadataDataStreamsService.updateSettings(masterNodeTimeout, ackTimeout, dataStreamName, settingsOverrides, new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + if (acknowledgedResponse.isAcknowledged()) { + updateSettingsOnIndices(dataStreamName, settingsOverrides, masterNodeTimeout, ackTimeout, listener); + } else { + listener.onResponse( + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( + dataStreamName, + false, + "Updating settings not accepted for unknown reasons", + EMPTY, + EMPTY, + UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult.EMPTY + ) + ); + } + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + + private void updateSettingsOnIndices( + String dataStreamName, + Settings requestSettings, + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + ActionListener listener + ) { + Map settingsToApply = new HashMap<>(); + List appliedToDataStreamOnly = new ArrayList<>(); + List appliedToDataStreamAndBackingIndices = new ArrayList<>(); + for (String settingName : requestSettings.keySet()) { + if (APPLY_TO_BACKING_INDICES.contains(settingName)) { + settingsToApply.put(settingName, requestSettings.get(settingName)); + appliedToDataStreamAndBackingIndices.add(settingName); + } else if (APPLY_TO_DATA_STREAM_ONLY.contains(settingName)) { + appliedToDataStreamOnly.add(settingName); + } + } + final List concreteIndices = clusterService.state().metadata().dataStreams().get(dataStreamName).getIndices(); + final List indexSettingErrors = new ArrayList<>(); + + CountDownActionListener indexCountDownListener = new CountDownActionListener(concreteIndices.size() + 1, new ActionListener<>() { + // Called when all indices for all settings are complete + @Override + public void onResponse(Void unused) { + DataStream dataStream = clusterService.state().metadata().dataStreams().get(dataStreamName); + listener.onResponse( + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( + dataStreamName, + true, + null, + settingsFilter.filter(dataStream.getSettings()), + settingsFilter.filter(dataStream.getEffectiveSettings(clusterService.state().metadata())), + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult( + appliedToDataStreamOnly, + appliedToDataStreamAndBackingIndices, + indexSettingErrors + ) + ) + ); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + indexCountDownListener.onResponse(null); // handles the case where there were zero indices + Settings applyToIndexSettings = builder().loadFromMap(settingsToApply).build(); + for (Index index : concreteIndices) { + updateSettingsOnSingleIndex(index, applyToIndexSettings, masterNodeTimeout, ackTimeout, new ActionListener<>() { + @Override + public void onResponse(UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndexSettingError indexSettingError) { + if (indexSettingError != null) { + indexSettingErrors.add(indexSettingError); + } + indexCountDownListener.onResponse(null); + } + + @Override + public void onFailure(Exception e) { + indexCountDownListener.onFailure(e); + } + }); + } + } + + private void updateSettingsOnSingleIndex( + Index index, + Settings requestSettings, + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + ActionListener listener + ) { + if (requestSettings.isEmpty()) { + listener.onResponse(null); + } else { + ClusterBlockException blockException = clusterService.state() + .blocks() + .indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, new String[] { index.getName() }); + if (blockException != null) { + listener.onResponse( + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndexSettingError( + index.getName(), + blockException.getMessage() + ) + ); + return; + } + updateSettingsService.updateSettings( + new UpdateSettingsClusterStateUpdateRequest( + masterNodeTimeout, + ackTimeout, + requestSettings, + UpdateSettingsClusterStateUpdateRequest.OnExisting.OVERWRITE, + UpdateSettingsClusterStateUpdateRequest.OnStaticSetting.REOPEN_INDICES, + index + ), + new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse response) { + UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndexSettingError error; + if (response.isAcknowledged() == false) { + error = new UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndexSettingError( + index.getName(), + "Updating settings not acknowledged for unknown reason" + ); + } else { + error = null; + } + listener.onResponse(error); + } + + @Override + public void onFailure(Exception e) { + listener.onResponse( + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndexSettingError(index.getName(), e.getMessage()) + ); + } + } + ); + } + + } + + @Override + protected ClusterBlockException checkBlock(UpdateDataStreamSettingsAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamSettingsAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamSettingsAction.java new file mode 100644 index 0000000000000..b0012bf1bcf39 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamSettingsAction.java @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.datastreams; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ChunkedToXContentObject; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +public class GetDataStreamSettingsAction extends ActionType { + public static final String NAME = "indices:monitor/data_stream/settings/get"; + public static final GetDataStreamSettingsAction INSTANCE = new GetDataStreamSettingsAction(); + + public GetDataStreamSettingsAction() { + super(NAME); + } + + public static class Request extends MasterNodeReadRequest implements IndicesRequest.Replaceable { + private String[] dataStreamNames; + + public Request(TimeValue masterNodeTimeout) { + super(masterNodeTimeout); + local = true; // this only ever runs locally + } + + public static Request localOnly(StreamInput ignored) { + return TransportAction.localOnly(); + } + + @Override + public GetDataStreamSettingsAction.Request indices(String... dataStreamNames) { + this.dataStreamNames = dataStreamNames; + return this; + } + + @Override + public boolean includeDataStreams() { + return true; + } + + @Override + public String[] indices() { + return dataStreamNames; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED; + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, "", parentTaskId, headers); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GetDataStreamSettingsAction.Request request = (GetDataStreamSettingsAction.Request) o; + return Arrays.equals(dataStreamNames, request.dataStreamNames); + } + + @Override + public int hashCode() { + return Arrays.hashCode(dataStreamNames); + } + } + + public static class Response extends ActionResponse implements ChunkedToXContentObject { + private final List dataStreamSettingsResponses; + + public Response(List dataStreamSettingsResponses) { + this.dataStreamSettingsResponses = dataStreamSettingsResponses; + } + + public static Response localOnly(StreamInput ignored) { + return TransportAction.localOnly(); + } + + public List getDataStreamSettingsResponses() { + return dataStreamSettingsResponses; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + assert false : "This ought to never be called because this action only runs locally"; + } + + @Override + public Iterator toXContentChunked(ToXContent.Params params) { + return Iterators.concat( + Iterators.single((builder, params1) -> builder.startObject().startArray("data_streams")), + dataStreamSettingsResponses.stream().map(dataStreamSettingsResponse -> (ToXContent) dataStreamSettingsResponse).iterator(), + Iterators.single((builder, params1) -> builder.endArray().endObject()) + ); + } + } + + public record DataStreamSettingsResponse(String dataStreamName, Settings settings, Settings effectiveSettings) implements ToXContent { + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("name", dataStreamName); + builder.startObject("settings"); + settings.toXContent(builder, params); + builder.endObject(); + builder.startObject("effective_settings"); + effectiveSettings.toXContent(builder, params); + builder.endObject(); + builder.endObject(); + return builder; + } + } + +} diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsAction.java new file mode 100644 index 0000000000000..42ab1dd15ff38 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsAction.java @@ -0,0 +1,262 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.datastreams; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ChunkedToXContentObject; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class UpdateDataStreamSettingsAction extends ActionType { + + public static final String NAME = "indices:admin/data_stream/settings/update"; + public static final UpdateDataStreamSettingsAction INSTANCE = new UpdateDataStreamSettingsAction(); + + public UpdateDataStreamSettingsAction() { + super(NAME); + } + + public static class Request extends AcknowledgedRequest implements IndicesRequest.Replaceable { + private final Settings settings; + private String[] dataStreamNames = Strings.EMPTY_ARRAY; + + public Request(Settings settings, TimeValue masterNodeTimeout, TimeValue ackTimeout) { + super(masterNodeTimeout, ackTimeout); + this.settings = settings; + } + + @Override + public Request indices(String... dataStreamNames) { + this.dataStreamNames = dataStreamNames; + return this; + } + + public Settings getSettings() { + return settings; + } + + @Override + public boolean includeDataStreams() { + return true; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.dataStreamNames = in.readStringArray(); + this.settings = Settings.readSettingsFromStream(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeStringArray(dataStreamNames); + settings.writeTo(out); + } + + @Override + public String[] indices() { + return dataStreamNames; + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED; + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, "", parentTaskId, headers); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Arrays.equals(dataStreamNames, request.dataStreamNames) + && settings.equals(request.settings) + && Objects.equals(masterNodeTimeout(), request.masterNodeTimeout()) + && Objects.equals(ackTimeout(), request.ackTimeout()); + } + + @Override + public int hashCode() { + return Objects.hash(Arrays.hashCode(dataStreamNames), settings, masterNodeTimeout(), ackTimeout()); + } + + } + + public static class Response extends ActionResponse implements ChunkedToXContentObject { + private final List dataStreamSettingsResponses; + + public Response(List dataStreamSettingsResponses) { + this.dataStreamSettingsResponses = dataStreamSettingsResponses; + } + + public Response(StreamInput in) throws IOException { + this(in.readCollectionAsList(DataStreamSettingsResponse::new)); + } + + public List getDataStreamSettingsResponses() { + return dataStreamSettingsResponses; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(dataStreamSettingsResponses, (out1, value) -> value.writeTo(out1)); + } + + @Override + public Iterator toXContentChunked(ToXContent.Params params) { + return Iterators.concat( + Iterators.single((builder, params1) -> builder.startObject().startArray("data_streams")), + dataStreamSettingsResponses.stream().map(dataStreamSettingsResponse -> (ToXContent) dataStreamSettingsResponse).iterator(), + Iterators.single((builder, params1) -> builder.endArray().endObject()) + ); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return Objects.equals(dataStreamSettingsResponses, response.dataStreamSettingsResponses); + } + + @Override + public int hashCode() { + return Objects.hash(dataStreamSettingsResponses); + } + } + + public record DataStreamSettingsResponse( + String dataStreamName, + boolean dataStreamSucceeded, + String dataStreamErrorMessage, + Settings settings, + Settings effectiveSettings, + IndicesSettingsResult indicesSettingsResult + ) implements ToXContent, Writeable { + + public DataStreamSettingsResponse(StreamInput in) throws IOException { + this( + in.readString(), + in.readBoolean(), + in.readOptionalString(), + Settings.readSettingsFromStream(in), + Settings.readSettingsFromStream(in), + new IndicesSettingsResult(in) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(dataStreamName); + out.writeBoolean(dataStreamSucceeded); + out.writeOptionalString(dataStreamErrorMessage); + settings.writeTo(out); + effectiveSettings.writeTo(out); + indicesSettingsResult.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("name", dataStreamName); + builder.field("applied_to_data_stream", dataStreamSucceeded); + if (dataStreamErrorMessage != null) { + builder.field("error", dataStreamErrorMessage); + } + builder.startObject("settings"); + settings.toXContent(builder, params); + builder.endObject(); + builder.startObject("effective_settings"); + effectiveSettings.toXContent(builder, params); + builder.endObject(); + builder.startObject("index_settings_results"); + indicesSettingsResult.toXContent(builder, params); + builder.endObject(); + builder.endObject(); + return builder; + } + + public record IndicesSettingsResult( + List appliedToDataStreamOnly, + List appliedToDataStreamAndBackingIndices, + List indexSettingErrors + ) implements ToXContent, Writeable { + + public static final IndicesSettingsResult EMPTY = new IndicesSettingsResult(List.of(), List.of(), List.of()); + + public IndicesSettingsResult(StreamInput in) throws IOException { + this(in.readStringCollectionAsList(), in.readStringCollectionAsList(), in.readCollectionAsList(IndexSettingError::new)); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("applied_to_data_stream_only", appliedToDataStreamOnly); + builder.field("applied_to_data_stream_and_backing_indices", appliedToDataStreamAndBackingIndices); + if (indexSettingErrors.isEmpty() == false) { + builder.field("errors", indexSettingErrors); + } + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeStringCollection(appliedToDataStreamOnly); + out.writeStringCollection(appliedToDataStreamAndBackingIndices); + out.writeCollection(indexSettingErrors, (out1, value) -> value.writeTo(out1)); + } + } + + public record IndexSettingError(String indexName, String errorMessage) implements ToXContent, Writeable { + public IndexSettingError(StreamInput in) throws IOException { + this(in.readString(), in.readString()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(indexName); + out.writeString(errorMessage); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("index", indexName); + builder.field("error", errorMessage); + builder.endObject(); + return builder; + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java index 7d103b667f2bb..567fa12278121 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java @@ -39,10 +39,13 @@ import java.io.IOException; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.lookupTemplateForDataStream; + /** * Handles data stream modification requests. */ @@ -54,6 +57,7 @@ public class MetadataDataStreamsService { private final MasterServiceTaskQueue updateLifecycleTaskQueue; private final MasterServiceTaskQueue setRolloverOnWriteTaskQueue; private final MasterServiceTaskQueue updateOptionsTaskQueue; + private final MasterServiceTaskQueue updateSettingsTaskQueue; public MetadataDataStreamsService( ClusterService clusterService, @@ -116,6 +120,45 @@ public Tuple executeTask( } }; this.updateOptionsTaskQueue = clusterService.createTaskQueue("modify-data-stream-options", Priority.NORMAL, updateOptionsExecutor); + ClusterStateTaskExecutor updateSettingsExecutor = new SimpleBatchedAckListenerTaskExecutor<>() { + + @Override + public Tuple executeTask( + UpdateSettingsTask updateSettingsTask, + ClusterState clusterState + ) throws Exception { + Metadata metadata = clusterState.metadata(); + Metadata.Builder metadataBuilder = Metadata.builder(metadata); + Map dataStreamMap = metadata.dataStreams(); + DataStream dataStream = dataStreamMap.get(updateSettingsTask.dataStreamName); + Settings existingSettings = dataStream.getSettings(); + + Template.Builder templateBuilder = Template.builder(); + Settings.Builder mergedSettingsBuilder = Settings.builder().put(existingSettings).put(updateSettingsTask.settingsOverrides); + Settings mergedSettings = mergedSettingsBuilder.build(); + + final ComposableIndexTemplate template = lookupTemplateForDataStream(updateSettingsTask.dataStreamName, metadata); + ComposableIndexTemplate mergedTemplate = template.mergeSettings(mergedSettings); + MetadataIndexTemplateService.validateTemplate( + mergedTemplate.template().settings(), + mergedTemplate.template().mappings(), + indicesService + ); + + templateBuilder.settings(mergedSettingsBuilder); + DataStream.Builder dataStreamBuilder = dataStream.copy().setSettings(mergedSettings); + metadataBuilder.removeDataStream(updateSettingsTask.dataStreamName); + metadataBuilder.put(dataStreamBuilder.build()); + ClusterState updatedClusterState = ClusterState.builder(clusterState).metadata(metadataBuilder).build(); + + return new Tuple<>(updatedClusterState, updateSettingsTask); + } + }; + this.updateSettingsTaskQueue = clusterService.createTaskQueue( + "update-data-stream-settings", + Priority.NORMAL, + updateSettingsExecutor + ); } public void modifyDataStream(final ModifyDataStreamsAction.Request request, final ActionListener listener) { @@ -339,6 +382,20 @@ public static ClusterState setRolloverOnWrite( return ClusterState.builder(currentState).metadata(builder.build()).build(); } + public void updateSettings( + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + String dataStreamName, + Settings settingsOverrides, + ActionListener listener + ) { + updateSettingsTaskQueue.submitTask( + "updating settings on data stream", + new UpdateSettingsTask(dataStreamName, settingsOverrides, ackTimeout, listener), + masterNodeTimeout + ); + } + private static void addBackingIndex( Metadata metadata, Metadata.Builder builder, @@ -563,4 +620,20 @@ public boolean targetFailureStore() { return targetFailureStore; } } + + static class UpdateSettingsTask extends AckedBatchedClusterStateUpdateTask { + private final String dataStreamName; + private final Settings settingsOverrides; + + UpdateSettingsTask( + String dataStreamName, + Settings settingsOverrides, + TimeValue ackTimeout, + ActionListener listener + ) { + super(ackTimeout, listener); + this.dataStreamName = dataStreamName; + this.settingsOverrides = settingsOverrides; + } + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 1afa350658396..42082be4169b9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1854,8 +1854,7 @@ private static void validateCompositeTemplate( }); } - private static void validateTemplate(Settings validateSettings, CompressedXContent mappings, IndicesService indicesService) - throws Exception { + static void validateTemplate(Settings validateSettings, CompressedXContent mappings, IndicesService indicesService) throws Exception { // Hard to validate settings if they're non-existent, so used empty ones if none were provided Settings settings = validateSettings; if (settings == null) { diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamSettingsActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamSettingsActionTests.java new file mode 100644 index 0000000000000..c6b2b699be00b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamSettingsActionTests.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.datastreams; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; +import static org.hamcrest.Matchers.equalTo; + +public class GetDataStreamSettingsActionTests extends ESTestCase { + + public void testResponseToXContentEmpty() throws IOException { + List responseList = new ArrayList<>(); + GetDataStreamSettingsAction.Response response = new GetDataStreamSettingsAction.Response(responseList); + try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { + builder.humanReadable(true); + response.toXContentChunked(ToXContent.EMPTY_PARAMS).forEachRemaining(xcontent -> { + try { + xcontent.toXContent(builder, EMPTY_PARAMS); + } catch (IOException e) { + fail(e); + } + }); + Map xContentMap = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + assertThat(xContentMap, equalTo(Map.of("data_streams", List.of()))); + } + } + + public void testResponseToXContent() throws IOException { + Map dataStream1Settings = Map.of("setting1", "value1", "setting2", "value2"); + Map dataStream1EffectiveSettings = Map.of("setting1", "value1", "setting2", "value2", "setting3", "value3"); + Map dataStream2Settings = Map.of("setting4", "value4", "setting5", "value5"); + Map dataStream2EffectiveSettings = Map.of("setting4", "value4", "setting5", "value5", "settings6", "value6"); + GetDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse1 = + new GetDataStreamSettingsAction.DataStreamSettingsResponse( + "dataStream1", + Settings.builder().loadFromMap(dataStream1Settings).build(), + Settings.builder().loadFromMap(dataStream1EffectiveSettings).build() + ); + GetDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse2 = + new GetDataStreamSettingsAction.DataStreamSettingsResponse( + "dataStream2", + Settings.builder().loadFromMap(dataStream2Settings).build(), + Settings.builder().loadFromMap(dataStream2EffectiveSettings).build() + ); + List responseList = List.of( + dataStreamSettingsResponse1, + dataStreamSettingsResponse2 + ); + GetDataStreamSettingsAction.Response response = new GetDataStreamSettingsAction.Response(responseList); + try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { + builder.humanReadable(true); + response.toXContentChunked(ToXContent.EMPTY_PARAMS).forEachRemaining(xcontent -> { + try { + xcontent.toXContent(builder, EMPTY_PARAMS); + } catch (IOException e) { + fail(e); + } + }); + Map xContentMap = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + assertThat( + xContentMap, + equalTo( + Map.of( + "data_streams", + List.of( + Map.of( + "name", + "dataStream1", + "settings", + dataStream1Settings, + "effective_settings", + dataStream1EffectiveSettings + ), + Map.of( + "name", + "dataStream2", + "settings", + dataStream2Settings, + "effective_settings", + dataStream2EffectiveSettings + ) + ) + ) + ) + ); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsActionRequestTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsActionRequestTests.java new file mode 100644 index 0000000000000..8ab80aee3fd80 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsActionRequestTests.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.datastreams; + +import org.elasticsearch.cluster.metadata.ComponentTemplateTests; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Arrays; +import java.util.function.Supplier; + +import static org.elasticsearch.cluster.metadata.ComponentTemplateTests.randomSettings; + +public class UpdateDataStreamSettingsActionRequestTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return UpdateDataStreamSettingsAction.Request::new; + } + + @Override + protected UpdateDataStreamSettingsAction.Request createTestInstance() { + UpdateDataStreamSettingsAction.Request request = new UpdateDataStreamSettingsAction.Request( + randomSettings(), + randomTimeValue(), + randomTimeValue() + ); + request.indices(randomIndices()); + return request; + } + + @Override + protected UpdateDataStreamSettingsAction.Request mutateInstance(UpdateDataStreamSettingsAction.Request instance) throws IOException { + String[] indices = instance.indices(); + Settings settings = instance.getSettings(); + TimeValue masterNodeTimeout = instance.masterNodeTimeout(); + TimeValue ackTimeout = instance.ackTimeout(); + switch (between(0, 3)) { + case 0 -> { + indices = randomArrayValueOtherThan(indices, this::randomIndices); + } + case 1 -> { + settings = randomValueOtherThan(settings, ComponentTemplateTests::randomSettings); + } + case 2 -> { + masterNodeTimeout = randomValueOtherThan(masterNodeTimeout, ESTestCase::randomTimeValue); + } + case 3 -> { + ackTimeout = randomValueOtherThan(ackTimeout, ESTestCase::randomTimeValue); + } + default -> throw new AssertionError("Should not be here"); + } + return new UpdateDataStreamSettingsAction.Request(settings, masterNodeTimeout, ackTimeout).indices(indices); + } + + private String[] randomIndices() { + return randomList(10, () -> randomAlphaOfLength(20)).toArray(new String[0]); + } + + public static T[] randomArrayValueOtherThan(T[] input, Supplier randomSupplier) { + return randomValueOtherThanMany(v -> Arrays.equals(input, v), randomSupplier); + } +} diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsActionResponseTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsActionResponseTests.java new file mode 100644 index 0000000000000..a249703beb30b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsActionResponseTests.java @@ -0,0 +1,213 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.datastreams; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.action.datastreams.UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndexSettingError; +import static org.elasticsearch.cluster.metadata.ComponentTemplateTests.randomSettings; +import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; +import static org.hamcrest.Matchers.equalTo; + +public class UpdateDataStreamSettingsActionResponseTests extends AbstractWireSerializingTestCase { + @Override + protected Writeable.Reader instanceReader() { + return UpdateDataStreamSettingsAction.Response::new; + } + + public void testToXContent() throws IOException { + Map dataStream1Settings = Map.of("setting1", "value1", "setting2", "value2"); + Map dataStream1EffectiveSettings = Map.of("setting1", "value1", "setting2", "value2", "setting3", "value3"); + List dataStream1AppliedToDataStreamOnly = randomList(10, () -> randomAlphanumericOfLength(10)); + List dataStream1AppliedToBackingIndices = randomList(10, () -> randomAlphanumericOfLength(10)); + List dataStream1IndexErrors = randomList( + 10, + () -> new IndexSettingError(randomAlphanumericOfLength(10), randomAlphaOfLength(10)) + ); + Map dataStream2Settings = Map.of("setting4", "value4", "setting5", "value5"); + Map dataStream2EffectiveSettings = Map.of("setting4", "value4", "setting5", "value5", "settings6", "value6"); + List dataStream2AppliedToDataStreamOnly = randomList(10, () -> randomAlphanumericOfLength(10)); + List dataStream2AppliedToBackingIndices = randomList(10, () -> randomAlphanumericOfLength(10)); + List dataStream2IndexErrors = randomList( + 10, + () -> new IndexSettingError(randomAlphanumericOfLength(10), randomAlphaOfLength(10)) + ); + boolean dataStream1Succeeded = randomBoolean(); + String dataStream1Error = randomBoolean() ? null : randomAlphaOfLength(20); + boolean dataStream2Succeeded = randomBoolean(); + String dataStream2Error = randomBoolean() ? null : randomAlphaOfLength(20); + UpdateDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse1 = + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( + "dataStream1", + dataStream1Succeeded, + dataStream1Error, + Settings.builder().loadFromMap(dataStream1Settings).build(), + Settings.builder().loadFromMap(dataStream1EffectiveSettings).build(), + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult( + dataStream1AppliedToDataStreamOnly, + dataStream1AppliedToBackingIndices, + dataStream1IndexErrors + ) + ); + UpdateDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse2 = + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( + "dataStream2", + dataStream2Succeeded, + dataStream2Error, + Settings.builder().loadFromMap(dataStream2Settings).build(), + Settings.builder().loadFromMap(dataStream2EffectiveSettings).build(), + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult( + dataStream2AppliedToDataStreamOnly, + dataStream2AppliedToBackingIndices, + dataStream2IndexErrors + ) + ); + List responseList = List.of( + dataStreamSettingsResponse1, + dataStreamSettingsResponse2 + ); + UpdateDataStreamSettingsAction.Response response = new UpdateDataStreamSettingsAction.Response(responseList); + try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { + builder.humanReadable(true); + response.toXContentChunked(ToXContent.EMPTY_PARAMS).forEachRemaining(xcontent -> { + try { + xcontent.toXContent(builder, EMPTY_PARAMS); + } catch (IOException e) { + fail(e); + } + }); + Map xContentMap = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + assertThat( + xContentMap, + equalTo( + Map.of( + "data_streams", + List.of( + buildExpectedMap( + "dataStream1", + dataStream1Succeeded, + dataStream1Error, + dataStream1Settings, + dataStream1EffectiveSettings, + dataStream1AppliedToDataStreamOnly, + dataStream1AppliedToBackingIndices, + dataStream1IndexErrors + ), + buildExpectedMap( + "dataStream2", + dataStream2Succeeded, + dataStream2Error, + dataStream2Settings, + dataStream2EffectiveSettings, + dataStream2AppliedToDataStreamOnly, + dataStream2AppliedToBackingIndices, + dataStream2IndexErrors + ) + ) + ) + ) + ); + } + } + + private Map buildExpectedMap( + String name, + boolean succeeded, + String error, + Map settings, + Map effectiveSettings, + List appliedToDataStreamOnly, + List appliedToIndices, + List indexErrors + ) { + Map result = new HashMap<>(); + result.put("name", name); + result.put("applied_to_data_stream", succeeded); + if (error != null) { + result.put("error", error); + } + result.put("settings", settings); + result.put("effective_settings", effectiveSettings); + Map indexSettingsResults = new HashMap<>(); + indexSettingsResults.put("applied_to_data_stream_only", appliedToDataStreamOnly); + indexSettingsResults.put("applied_to_data_stream_and_backing_indices", appliedToIndices); + if (indexErrors.isEmpty() == false) { + indexSettingsResults.put( + "errors", + indexErrors.stream() + .map(indexSettingError -> Map.of("index", indexSettingError.indexName(), "error", indexSettingError.errorMessage())) + .toList() + ); + } + result.put("index_settings_results", indexSettingsResults); + return result; + } + + @Override + protected UpdateDataStreamSettingsAction.Response createTestInstance() { + return new UpdateDataStreamSettingsAction.Response(randomList(10, this::randomDataStreamSettingsResponse)); + } + + private UpdateDataStreamSettingsAction.DataStreamSettingsResponse randomDataStreamSettingsResponse() { + return new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( + "dataStream1", + randomBoolean(), + randomBoolean() ? null : randomAlphaOfLength(20), + randomSettings(), + randomSettings(), + randomIndicesSettingsResult() + ); + } + + private UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult randomIndicesSettingsResult() { + return new UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult( + randomList(10, () -> randomAlphanumericOfLength(20)), + randomList(10, () -> randomAlphanumericOfLength(20)), + randomList(10, this::randomIndexSettingError) + ); + } + + private IndexSettingError randomIndexSettingError() { + return new IndexSettingError(randomAlphanumericOfLength(20), randomAlphanumericOfLength(20)); + } + + @Override + protected UpdateDataStreamSettingsAction.Response mutateInstance(UpdateDataStreamSettingsAction.Response instance) throws IOException { + List responseList = instance.getDataStreamSettingsResponses(); + List mutatedResponseList = new ArrayList<>(responseList); + switch (between(0, 1)) { + case 0 -> { + if (responseList.isEmpty()) { + mutatedResponseList.add(randomDataStreamSettingsResponse()); + } else { + mutatedResponseList.remove(randomInt(responseList.size() - 1)); + } + } + case 1 -> { + mutatedResponseList.add(randomDataStreamSettingsResponse()); + } + default -> throw new AssertionError("Should not be here"); + } + return new UpdateDataStreamSettingsAction.Response(mutatedResponseList); + } +} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index c06cfb488b761..ad222a1d47d73 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -610,6 +610,7 @@ public class Constants { "indices:data/write/reindex", "indices:data/write/update", "indices:data/write/update/byquery", + "indices:monitor/data_stream/settings/get", "indices:monitor/data_stream/stats", "indices:monitor/field_usage_stats", "indices:monitor/fleet/global_checkpoints[s]", @@ -641,6 +642,7 @@ public class Constants { "indices:admin/data_stream/index/reindex", "indices:admin/data_stream/reindex", "indices:admin/data_stream/reindex_cancel", + "indices:admin/data_stream/settings/update", "indices:admin/index/create_from_source", "indices:admin/index/copy_lifecycle_index_metadata", "internal:admin/repository/verify", From daa75f561b34633fabbca101597f072851b5952d Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 29 Apr 2025 13:31:54 -0500 Subject: [PATCH 04/17] Fixing DataStream::getEffectiveSettings for component templates (#127515) --- .../TransportGetDataStreamsActionTests.java | 55 ++++++++++++++++++- .../cluster/metadata/DataStream.java | 7 +-- .../cluster/metadata/DataStreamTests.java | 20 +++++++ 3 files changed, 73 insertions(+), 9 deletions(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java index 5d97986658bc2..dbf8a31747b3d 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings; @@ -38,6 +39,7 @@ import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -548,10 +550,41 @@ public void testGetEffectiveSettingsTemplateOnlySettings() { GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); final String templatePolicy = "templatePolicy"; final String templateIndexMode = IndexMode.LOOKUP.getName(); - final String dataStreamPolicy = "dataStreamPolicy"; - final String dataStreamIndexMode = IndexMode.LOGSDB.getName(); ClusterState state = getClusterStateWithDataStreamWithSettings( + Settings.builder() + .put(IndexMetadata.LIFECYCLE_NAME, templatePolicy) + .put(IndexSettings.MODE.getKey(), templateIndexMode) + .build(), + Settings.EMPTY, + Settings.EMPTY + ); + + GetDataStreamAction.Response response = TransportGetDataStreamsAction.innerOperation( + state, + req, + resolver, + systemIndices, + ClusterSettings.createBuiltInClusterSettings(), + dataStreamGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), + null + ); + assertNotNull(response.getDataStreams()); + assertThat(response.getDataStreams().size(), equalTo(1)); + assertThat(response.getDataStreams().get(0).getIlmPolicy(), equalTo(templatePolicy)); + assertThat(response.getDataStreams().get(0).getIndexModeName(), equalTo(templateIndexMode)); + } + + public void testGetEffectiveSettingsComponentTemplateOnlySettings() { + // Set a lifecycle only in the template, and make sure that is in the response: + GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); + final String templatePolicy = "templatePolicy"; + final String templateIndexMode = IndexMode.LOOKUP.getName(); + + ClusterState state = getClusterStateWithDataStreamWithSettings( + Settings.EMPTY, Settings.builder() .put(IndexMetadata.LIFECYCLE_NAME, templatePolicy) .put(IndexSettings.MODE.getKey(), templateIndexMode) @@ -584,6 +617,10 @@ public void testGetEffectiveSettings() { final String dataStreamIndexMode = IndexMode.LOGSDB.getName(); // Now set a lifecycle in both the template and the data stream, and make sure the response has the data stream one: ClusterState state = getClusterStateWithDataStreamWithSettings( + Settings.builder() + .put(IndexMetadata.LIFECYCLE_NAME, templatePolicy) + .put(IndexSettings.MODE.getKey(), templateIndexMode) + .build(), Settings.builder() .put(IndexMetadata.LIFECYCLE_NAME, templatePolicy) .put(IndexSettings.MODE.getKey(), templateIndexMode) @@ -610,7 +647,11 @@ public void testGetEffectiveSettings() { assertThat(response.getDataStreams().get(0).getIndexModeName(), equalTo(dataStreamIndexMode)); } - private static ClusterState getClusterStateWithDataStreamWithSettings(Settings templateSettings, Settings dataStreamSettings) { + private static ClusterState getClusterStateWithDataStreamWithSettings( + Settings templateSettings, + Settings componentTemplateSettings, + Settings dataStreamSettings + ) { String dataStreamName = "data-stream-1"; int numberOfBackingIndices = randomIntBetween(1, 5); long currentTime = System.currentTimeMillis(); @@ -623,8 +664,16 @@ private static ClusterState getClusterStateWithDataStreamWithSettings(Settings t .indexPatterns(List.of("*")) .template(Template.builder().settings(templateSettings)) .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .componentTemplates(List.of("component_template_1")) .build() ); + ComponentTemplate componentTemplate = new ComponentTemplate( + Template.builder().settings(componentTemplateSettings).build(), + null, + null, + null + ); + builder.componentTemplates(Map.of("component_template_1", componentTemplate)); List backingIndices = new ArrayList<>(); for (int backingIndexNumber = 1; backingIndexNumber <= numberOfBackingIndices; backingIndexNumber++) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 18d2ebb7e5313..959b9bf6b5266 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -376,12 +376,7 @@ public ComposableIndexTemplate getEffectiveIndexTemplate(Metadata metadata) { public Settings getEffectiveSettings(Metadata metadata) { ComposableIndexTemplate template = getMatchingIndexTemplate(metadata); - final Settings templateSettings; - if (template.template() == null || template.template().settings() == null) { - templateSettings = Settings.EMPTY; - } else { - templateSettings = template.template().settings(); - } + Settings templateSettings = MetadataIndexTemplateService.resolveSettings(template, metadata.componentTemplates()); return templateSettings.merge(settings); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index 6c51f736eeb7b..be4116e86b894 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -2444,6 +2444,26 @@ public void testGetEffectiveSettingsTemplateSettingsOnly() { assertThat(dataStream.getEffectiveSettings(metadataBuilder.build()), equalTo(templateSettings)); } + public void testGetEffectiveSettingsComponentTemplateSettingsOnly() { + // We only have settings from a component template, so we expect to get those back + DataStream dataStream = createDataStream(Settings.EMPTY); + Settings templateSettings = Settings.EMPTY; + Template.Builder indexTemplateBuilder = Template.builder().settings(templateSettings); + ComposableIndexTemplate indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStream.getName())) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .template(indexTemplateBuilder) + .componentTemplates(List.of("component-template-1")) + .build(); + Settings componentSettings = randomSettings(); + Template.Builder componentTemplateBuilder = Template.builder().settings(componentSettings); + ComponentTemplate componentTemplate1 = new ComponentTemplate(componentTemplateBuilder.build(), null, null, null); + Metadata.Builder projectMetadataBuilder = Metadata.builder() + .indexTemplates(Map.of(dataStream.getName(), indexTemplate)) + .componentTemplates(Map.of("component-template-1", componentTemplate1)); + assertThat(dataStream.getEffectiveSettings(projectMetadataBuilder.build()), equalTo(componentSettings)); + } + public void testGetEffectiveSettingsDataStreamSettingsOnly() { // We only have settings from the data stream, so we expect to get those back Settings dataStreamSettings = randomSettings(); From 521dce697684ef0049fddc427d28cd2d5d8a0b14 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 21 May 2025 12:17:56 -0500 Subject: [PATCH 05/17] Adding rest actions to get and set data stream settings (#127858) --- .../datastreams/DataStreamFeatures.java | 4 +- .../datastreams/DataStreamsPlugin.java | 7 + .../rest/RestGetDataStreamSettingsAction.java | 51 +++++ .../RestUpdateDataStreamSettingsAction.java | 59 ++++++ .../data_stream/240_data_stream_settings.yml | 194 ++++++++++++++++++ .../api/indices.get_data_stream_settings.json | 40 ++++ .../api/indices.put_data_stream_settings.json | 44 ++++ .../datastreams/GetDataStreamAction.java | 6 + .../cluster/metadata/DataStream.java | 2 + .../datastreams/GetDataStreamActionTests.java | 6 + 10 files changed, 412 insertions(+), 1 deletion(-) create mode 100644 modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamSettingsAction.java create mode 100644 modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestUpdateDataStreamSettingsAction.java create mode 100644 modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream_settings.json create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_stream_settings.json diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java index a6a55c348d41e..934f9b24269d1 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java @@ -33,6 +33,8 @@ public class DataStreamFeatures implements FeatureSpecification { "data_stream.downsample.default_aggregate_metric_fix" ); + public static final NodeFeature LOGS_STREAM_FEATURE = new NodeFeature("logs_stream"); + @Override public Map getHistoricalFeatures() { return Map.of(DATA_STREAM_LIFECYCLE, Version.V_8_11_0); @@ -51,6 +53,6 @@ public Set getFeatures() { @Override public Set getTestFeatures() { - return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX, DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX); + return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX, DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX, LOGS_STREAM_FEATURE); } } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java index d39fea0435d8b..3a79130f392cd 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.datastreams.lifecycle.GetDataStreamLifecycleAction; import org.elasticsearch.action.datastreams.lifecycle.PutDataStreamLifecycleAction; import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -72,10 +73,12 @@ import org.elasticsearch.datastreams.rest.RestCreateDataStreamAction; import org.elasticsearch.datastreams.rest.RestDataStreamsStatsAction; import org.elasticsearch.datastreams.rest.RestDeleteDataStreamAction; +import org.elasticsearch.datastreams.rest.RestGetDataStreamSettingsAction; import org.elasticsearch.datastreams.rest.RestGetDataStreamsAction; import org.elasticsearch.datastreams.rest.RestMigrateToDataStreamAction; import org.elasticsearch.datastreams.rest.RestModifyDataStreamsAction; import org.elasticsearch.datastreams.rest.RestPromoteDataStreamAction; +import org.elasticsearch.datastreams.rest.RestUpdateDataStreamSettingsAction; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.health.HealthIndicatorService; import org.elasticsearch.index.IndexSettingProvider; @@ -282,6 +285,10 @@ public List getRestHandlers( handlers.add(new RestGetDataStreamOptionsAction()); handlers.add(new RestPutDataStreamOptionsAction()); handlers.add(new RestDeleteDataStreamOptionsAction()); + if (DataStream.LOGS_STREAM_FEATURE_FLAG) { + handlers.add(new RestGetDataStreamSettingsAction()); + handlers.add(new RestUpdateDataStreamSettingsAction()); + } return handlers; } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamSettingsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamSettingsAction.java new file mode 100644 index 0000000000000..0db70b5a91f86 --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamSettingsAction.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datastreams.rest; + +import org.elasticsearch.action.datastreams.GetDataStreamSettingsAction; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestUtils; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestCancellableNodeClient; +import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +@ServerlessScope(Scope.PUBLIC) +public class RestGetDataStreamSettingsAction extends BaseRestHandler { + @Override + public String getName() { + return "get_data_stream_settings_action"; + } + + @Override + public List routes() { + return List.of(new Route(GET, "/_data_stream/{name}/_settings")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + GetDataStreamSettingsAction.Request getDataStreamRequest = new GetDataStreamSettingsAction.Request( + RestUtils.getMasterNodeTimeout(request) + ).indices(Strings.splitStringByCommaToArray(request.param("name"))); + return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute( + GetDataStreamSettingsAction.INSTANCE, + getDataStreamRequest, + new RestRefCountedChunkedToXContentListener<>(channel) + ); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestUpdateDataStreamSettingsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestUpdateDataStreamSettingsAction.java new file mode 100644 index 0000000000000..1b0a4b4637156 --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestUpdateDataStreamSettingsAction.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.datastreams.rest; + +import org.elasticsearch.action.datastreams.UpdateDataStreamSettingsAction; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestUtils; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestCancellableNodeClient; +import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.PUT; + +@ServerlessScope(Scope.PUBLIC) +public class RestUpdateDataStreamSettingsAction extends BaseRestHandler { + + @Override + public String getName() { + return "update_data_stream_settings_action"; + } + + @Override + public List routes() { + return List.of(new Route(PUT, "/_data_stream/{name}/_settings")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + Settings settings; + try (XContentParser parser = request.contentParser()) { + settings = Settings.fromXContent(parser); + } + UpdateDataStreamSettingsAction.Request putDataStreamRequest = new UpdateDataStreamSettingsAction.Request( + settings, + RestUtils.getMasterNodeTimeout(request), + RestUtils.getAckTimeout(request) + ).indices(Strings.splitStringByCommaToArray(request.param("name"))); + return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute( + UpdateDataStreamSettingsAction.INSTANCE, + putDataStreamRequest, + new RestRefCountedChunkedToXContentListener<>(channel) + ); + } +} diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml new file mode 100644 index 0000000000000..489913c7976d6 --- /dev/null +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml @@ -0,0 +1,194 @@ +setup: + - skip: + features: allowed_warnings + +--- +"Test single data stream": + - requires: + cluster_features: [ "logs_stream" ] + reason: requires setting 'logs_stream' to get or set data stream settings + - do: + allowed_warnings: + - "index template [my-template] has index patterns [my-data-stream-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [ my-data-stream-* ] + data_stream: { } + template: + settings: + number_of_replicas: 0 + lifecycle.name: my-policy + + - do: + indices.create_data_stream: + name: my-data-stream-1 + + - do: + cluster.health: + index: "my-data-stream-1" + wait_for_status: green + + - do: + indices.get_data_stream_settings: + name: my-data-stream-1 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.settings: {} } + - match: { data_streams.0.effective_settings.index.number_of_shards: null } + - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" } + - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-policy" } + + - do: + indices.get_data_stream: + name: my-data-stream-1 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.settings: {} } + - match: { data_streams.0.effective_settings: null } + + - do: + indices.put_data_stream_settings: + name: my-data-stream-1 + body: + index: + number_of_shards: 2 + lifecycle.name: my-new-policy + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.applied_to_data_stream: true } + - match: { data_streams.0.index_settings_results.applied_to_data_stream_only: [index.number_of_shards]} + - match: { data_streams.0.index_settings_results.applied_to_data_stream_and_backing_indices: [index.lifecycle.name] } + - match: { data_streams.0.settings.index.number_of_shards: "2" } + - match: { data_streams.0.settings.index.lifecycle.name: "my-new-policy" } + - match: { data_streams.0.effective_settings.index.number_of_shards: "2" } + - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" } + - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-new-policy" } + + - do: + indices.rollover: + alias: "my-data-stream-1" + + - do: + cluster.health: + index: "my-data-stream-1" + wait_for_status: green + + - do: + indices.get_data_stream_settings: + name: my-data-stream-1 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.settings.index.number_of_shards: "2" } + - match: { data_streams.0.effective_settings.index.number_of_shards: "2" } + - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" } + - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-new-policy" } + + - do: + indices.get_data_stream: + name: my-data-stream-1 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.settings.index.number_of_shards: "2" } + - match: { data_streams.0.settings.index.lifecycle.name: "my-new-policy" } + - match: { data_streams.0.effective_settings: null } + + - do: + indices.get_data_stream: + name: my-data-stream-1 + - set: { data_streams.0.indices.0.index_name: idx0name } + - set: { data_streams.0.indices.1.index_name: idx1name } + + - do: + indices.get_settings: + index: my-data-stream-1 + - match: { .$idx0name.settings.index.number_of_shards: "1" } + - match: { .$idx0name.settings.index.lifecycle.name: "my-new-policy" } + - match: { .$idx1name.settings.index.number_of_shards: "2" } + - match: { .$idx1name.settings.index.lifecycle.name: "my-new-policy" } + +--- +"Test multiple data streams": + - requires: + cluster_features: [ "logs_stream" ] + reason: requires setting 'logs_stream' to get or set data stream settings + - do: + allowed_warnings: + - "index template [my-template] has index patterns [my-data-stream-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [ my-data-stream-* ] + data_stream: { } + template: + settings: + number_of_replicas: 0 + lifecycle.name: my-policy + + - do: + indices.create_data_stream: + name: my-data-stream-1 + + - do: + indices.create_data_stream: + name: my-data-stream-2 + + - do: + indices.create_data_stream: + name: my-data-stream-3 + + - do: + cluster.health: + index: "my-data-stream-1,my-data-stream-2,my-data-stream-3" + wait_for_status: green + + - do: + indices.get_data_stream_settings: + name: "*" + - length: { data_streams: 3 } + + - do: + indices.get_data_stream_settings: + name: "my-data-stream-1,my-data-stream-2" + - length: { data_streams: 2 } + + - do: + indices.put_data_stream_settings: + name: my-data-stream-1,my-data-stream-2 + body: + index: + number_of_shards: 2 + lifecycle.name: my-new-policy + - length: { data_streams: 2 } + +--- +"Test invalid settings": + - requires: + cluster_features: [ "logs_stream" ] + reason: requires setting 'logs_stream' to get or set data stream settings + - do: + allowed_warnings: + - "index template [my-template] has index patterns [my-data-stream-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [ my-data-stream-* ] + data_stream: { } + template: + settings: + number_of_replicas: 0 + lifecycle.name: my-policy + + - do: + indices.create_data_stream: + name: my-data-stream-1 + + - do: + cluster.health: + index: "my-data-stream-1" + wait_for_status: green + + - do: + indices.put_data_stream_settings: + name: my-data-stream-1 + body: + index: + fake_setting: 1234 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.applied_to_data_stream: false } + - match: { data_streams.0.error: "Cannot set the following settings on a data stream: [index.fake_setting]" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream_settings.json new file mode 100644 index 0000000000000..5582c5ca0c54b --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream_settings.json @@ -0,0 +1,40 @@ +{ + "indices.get_data_stream_settings":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html", + "description":"Gets a data stream's settings" + }, + "stability":"stable", + "visibility": "feature_flag", + "feature_flag": "logs_stream", + "headers":{ + "accept": [ "application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_data_stream/{name}/_settings", + "methods":[ + "GET" + ], + "parts":{ + "name":{ + "type":"string", + "description":"The name of the data stream or data stream pattern" + } + } + } + ] + }, + "params":{ + "timeout":{ + "type":"time", + "description":"Specify timeout for acknowledging the cluster state update" + }, + "master_timeout":{ + "type":"time", + "description":"Specify timeout for connection to master" + } + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_stream_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_stream_settings.json new file mode 100644 index 0000000000000..ca6ed7d8faf30 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_stream_settings.json @@ -0,0 +1,44 @@ +{ + "indices.put_data_stream_settings":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html", + "description":"Updates a data stream's settings" + }, + "stability":"stable", + "visibility": "feature_flag", + "feature_flag": "logs_stream", + "headers":{ + "accept": [ "application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_data_stream/{name}/_settings", + "methods":[ + "PUT" + ], + "parts":{ + "name":{ + "type":"string", + "description":"The name of the data stream or data stream pattern" + } + } + } + ] + }, + "params":{ + "timeout":{ + "type":"time", + "description":"Specify timeout for acknowledging the cluster state update" + }, + "master_timeout":{ + "type":"time", + "description":"Specify timeout for connection to master" + } + }, + "body":{ + "description":"The data stream settings to be updated", + "required":true + } + } +} diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index 172903727ffb5..281310966de78 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -212,6 +212,7 @@ public static class DataStreamInfo implements SimpleDiffable, To public static final ParseField STATUS_FIELD = new ParseField("status"); public static final ParseField INDEX_TEMPLATE_FIELD = new ParseField("template"); + public static final ParseField SETTINGS_FIELD = new ParseField("settings"); public static final ParseField PREFER_ILM = new ParseField("prefer_ilm"); public static final ParseField MANAGED_BY = new ParseField("managed_by"); public static final ParseField NEXT_GENERATION_INDEX_MANAGED_BY = new ParseField("next_generation_managed_by"); @@ -434,6 +435,11 @@ public XContentBuilder toXContent( builder.endArray(); builder.endObject(); } + if (DataStream.LOGS_STREAM_FEATURE_FLAG) { + builder.startObject(SETTINGS_FIELD.getPreferredName()); + dataStream.getSettings().toXContent(builder, params); + builder.endObject(); + } builder.startObject(DataStream.FAILURE_STORE_FIELD.getPreferredName()); builder.field(FAILURE_STORE_ENABLED.getPreferredName(), failureStoreEffectivelyEnabled); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 959b9bf6b5266..f0305c7ad176a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateFormatters; +import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; @@ -75,6 +76,7 @@ public final class DataStream implements SimpleDiffable, ToXContentO private static final Logger LOGGER = LogManager.getLogger(DataStream.class); public static final NodeFeature DATA_STREAM_FAILURE_STORE_FEATURE = new NodeFeature("data_stream.failure_store"); + public static final boolean LOGS_STREAM_FEATURE_FLAG = new FeatureFlag("logs_stream").isEnabled(); public static final TransportVersion ADDED_FAILURE_STORE_TRANSPORT_VERSION = TransportVersions.V_8_12_0; public static final TransportVersion ADDED_AUTO_SHARDING_EVENT_VERSION = TransportVersions.V_8_14_0; public static final TransportVersion ADD_DATA_STREAM_OPTIONS_VERSION = TransportVersions.V_8_16_0; diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java index 0614356943d3a..9aa1a2f2a2c07 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; @@ -27,6 +28,7 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.cluster.metadata.ComponentTemplateTests.randomSettings; import static org.hamcrest.Matchers.equalTo; public class GetDataStreamActionTests extends ESTestCase { @@ -60,6 +62,8 @@ public void testDataStreamInfoToXContent() throws IOException { assertThat(lifecycleResult.get("data_retention"), equalTo(configuredRetention.getStringRep())); assertThat(lifecycleResult.get("effective_retention"), equalTo(globalMaxRetention.getStringRep())); assertThat(lifecycleResult.get("retention_determined_by"), equalTo("max_global_retention")); + Map> settingsMap = (Map>) resultMap.get("settings"); + assertThat(Settings.builder().loadFromMap(settingsMap).build(), equalTo(dataStreamInfo.getDataStream().getSettings())); } } @@ -100,6 +104,7 @@ private static GetDataStreamAction.Response.DataStreamInfo newDataStreamInfo(boo private static DataStream newDataStreamInstance(boolean isSystem, TimeValue retention) { List indices = List.of(new Index(randomAlphaOfLength(10), randomAlphaOfLength(10))); DataStreamLifecycle lifecycle = DataStreamLifecycle.createDataLifecycle(true, retention, null); + Settings settings = randomSettings(); return DataStream.builder(randomAlphaOfLength(50), indices) .setGeneration(randomLongBetween(1, 1000)) .setMetadata(Map.of()) @@ -107,6 +112,7 @@ private static DataStream newDataStreamInstance(boolean isSystem, TimeValue rete .setHidden(isSystem) .setReplicated(randomBoolean()) .setLifecycle(lifecycle) + .setSettings(settings) .build(); } } From a92a55dfe766b8b5162f5c523b6d31ee6e624ad7 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 23 May 2025 11:29:00 -0500 Subject: [PATCH 06/17] Adding dry_run mode for setting data stream settings (#128269) --- ...ansportUpdateDataStreamSettingsAction.java | 178 ++++++++---------- .../RestUpdateDataStreamSettingsAction.java | 2 + .../data_stream/240_data_stream_settings.yml | 100 ++++++++++ .../api/indices.put_data_stream_settings.json | 5 + .../org/elasticsearch/TransportVersions.java | 1 + .../UpdateDataStreamSettingsAction.java | 22 ++- .../metadata/MetadataDataStreamsService.java | 105 ++++++++--- ...eDataStreamSettingsActionRequestTests.java | 11 +- 8 files changed, 295 insertions(+), 129 deletions(-) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java index 9cd8f1145bb58..4814937ce15d9 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java @@ -16,7 +16,6 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.CountDownActionListener; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; @@ -102,46 +101,38 @@ protected void masterOperation( request.indices() ); List dataStreamSettingsResponse = new ArrayList<>(); - CountDownActionListener countDownListener = new CountDownActionListener(dataStreamNames.size() + 1, new ActionListener<>() { - @Override - public void onResponse(Void unused) { - listener.onResponse(new UpdateDataStreamSettingsAction.Response(dataStreamSettingsResponse)); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); + CountDownActionListener countDownListener = new CountDownActionListener( + dataStreamNames.size() + 1, + listener.delegateFailure( + (responseActionListener, unused) -> responseActionListener.onResponse( + new UpdateDataStreamSettingsAction.Response(dataStreamSettingsResponse) + ) + ) + ); countDownListener.onResponse(null); for (String dataStreamName : dataStreamNames) { updateSingleDataStream( dataStreamName, request.getSettings(), + request.isDryRun(), request.masterNodeTimeout(), request.ackTimeout(), - new ActionListener<>() { - @Override - public void onResponse(UpdateDataStreamSettingsAction.DataStreamSettingsResponse dataStreamResponse) { - dataStreamSettingsResponse.add(dataStreamResponse); - countDownListener.onResponse(null); - } - - @Override - public void onFailure(Exception e) { - dataStreamSettingsResponse.add( - new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( - dataStreamName, - false, - e.getMessage(), - EMPTY, - EMPTY, - UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult.EMPTY - ) - ); - countDownListener.onResponse(null); - } - } + ActionListener.wrap(dataStreamResponse -> { + dataStreamSettingsResponse.add(dataStreamResponse); + countDownListener.onResponse(null); + }, e -> { + dataStreamSettingsResponse.add( + new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( + dataStreamName, + false, + e.getMessage(), + EMPTY, + EMPTY, + UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult.EMPTY + ) + ); + countDownListener.onResponse(null); + }) ); } } @@ -149,6 +140,7 @@ public void onFailure(Exception e) { private void updateSingleDataStream( String dataStreamName, Settings settingsOverrides, + boolean dryRun, TimeValue masterNodeTimeout, TimeValue ackTimeout, ActionListener listener @@ -187,13 +179,17 @@ private void updateSingleDataStream( ); return; } - metadataDataStreamsService.updateSettings(masterNodeTimeout, ackTimeout, dataStreamName, settingsOverrides, new ActionListener<>() { - @Override - public void onResponse(AcknowledgedResponse acknowledgedResponse) { - if (acknowledgedResponse.isAcknowledged()) { - updateSettingsOnIndices(dataStreamName, settingsOverrides, masterNodeTimeout, ackTimeout, listener); + metadataDataStreamsService.updateSettings( + masterNodeTimeout, + ackTimeout, + dataStreamName, + settingsOverrides, + dryRun, + listener.delegateFailure((dataStreamSettingsResponseActionListener, dataStream) -> { + if (dataStream != null) { + updateSettingsOnIndices(dataStream, settingsOverrides, dryRun, masterNodeTimeout, ackTimeout, listener); } else { - listener.onResponse( + dataStreamSettingsResponseActionListener.onResponse( new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( dataStreamName, false, @@ -204,18 +200,14 @@ public void onResponse(AcknowledgedResponse acknowledgedResponse) { ) ); } - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); + }) + ); } private void updateSettingsOnIndices( - String dataStreamName, + DataStream dataStream, Settings requestSettings, + boolean dryRun, TimeValue masterNodeTimeout, TimeValue ackTimeout, ActionListener listener @@ -231,17 +223,15 @@ private void updateSettingsOnIndices( appliedToDataStreamOnly.add(settingName); } } - final List concreteIndices = clusterService.state().metadata().dataStreams().get(dataStreamName).getIndices(); + final List concreteIndices = dataStream.getIndices(); final List indexSettingErrors = new ArrayList<>(); - CountDownActionListener indexCountDownListener = new CountDownActionListener(concreteIndices.size() + 1, new ActionListener<>() { - // Called when all indices for all settings are complete - @Override - public void onResponse(Void unused) { - DataStream dataStream = clusterService.state().metadata().dataStreams().get(dataStreamName); - listener.onResponse( + CountDownActionListener indexCountDownListener = new CountDownActionListener( + concreteIndices.size() + 1, + listener.delegateFailure( + (dataStreamSettingsResponseActionListener, unused) -> dataStreamSettingsResponseActionListener.onResponse( new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( - dataStreamName, + dataStream.getName(), true, null, settingsFilter.filter(dataStream.getSettings()), @@ -252,37 +242,33 @@ public void onResponse(Void unused) { indexSettingErrors ) ) - ); - } + ) + ) + ); - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); indexCountDownListener.onResponse(null); // handles the case where there were zero indices Settings applyToIndexSettings = builder().loadFromMap(settingsToApply).build(); for (Index index : concreteIndices) { - updateSettingsOnSingleIndex(index, applyToIndexSettings, masterNodeTimeout, ackTimeout, new ActionListener<>() { - @Override - public void onResponse(UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndexSettingError indexSettingError) { + updateSettingsOnSingleIndex( + index, + applyToIndexSettings, + dryRun, + masterNodeTimeout, + ackTimeout, + indexCountDownListener.delegateFailure((listener1, indexSettingError) -> { if (indexSettingError != null) { indexSettingErrors.add(indexSettingError); } - indexCountDownListener.onResponse(null); - } - - @Override - public void onFailure(Exception e) { - indexCountDownListener.onFailure(e); - } - }); + listener1.onResponse(null); + }) + ); } } private void updateSettingsOnSingleIndex( Index index, Settings requestSettings, + boolean dryRun, TimeValue masterNodeTimeout, TimeValue ackTimeout, ActionListener listener @@ -302,18 +288,23 @@ private void updateSettingsOnSingleIndex( ); return; } - updateSettingsService.updateSettings( - new UpdateSettingsClusterStateUpdateRequest( - masterNodeTimeout, - ackTimeout, - requestSettings, - UpdateSettingsClusterStateUpdateRequest.OnExisting.OVERWRITE, - UpdateSettingsClusterStateUpdateRequest.OnStaticSetting.REOPEN_INDICES, - index - ), - new ActionListener<>() { - @Override - public void onResponse(AcknowledgedResponse response) { + if (dryRun) { + /* + * This is as far as we go with dry run mode. We get the benefit of having checked that all the indices that will be touced + * are not blocked, but there is no value in going beyond this. So just respond to the listener and move on. + */ + listener.onResponse(null); + } else { + updateSettingsService.updateSettings( + new UpdateSettingsClusterStateUpdateRequest( + masterNodeTimeout, + ackTimeout, + requestSettings, + UpdateSettingsClusterStateUpdateRequest.OnExisting.OVERWRITE, + UpdateSettingsClusterStateUpdateRequest.OnStaticSetting.REOPEN_INDICES, + index + ), + ActionListener.wrap(response -> { UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndexSettingError error; if (response.isAcknowledged() == false) { error = new UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndexSettingError( @@ -324,16 +315,13 @@ public void onResponse(AcknowledgedResponse response) { error = null; } listener.onResponse(error); - } - - @Override - public void onFailure(Exception e) { - listener.onResponse( + }, + e -> listener.onResponse( new UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndexSettingError(index.getName(), e.getMessage()) - ); - } - } - ); + ) + ) + ); + } } } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestUpdateDataStreamSettingsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestUpdateDataStreamSettingsAction.java index 1b0a4b4637156..d21efb247a231 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestUpdateDataStreamSettingsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestUpdateDataStreamSettingsAction.java @@ -45,8 +45,10 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli try (XContentParser parser = request.contentParser()) { settings = Settings.fromXContent(parser); } + boolean dryRun = request.paramAsBoolean("dry_run", false); UpdateDataStreamSettingsAction.Request putDataStreamRequest = new UpdateDataStreamSettingsAction.Request( settings, + dryRun, RestUtils.getMasterNodeTimeout(request), RestUtils.getAckTimeout(request) ).indices(Strings.splitStringByCommaToArray(request.param("name"))); diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml index 489913c7976d6..e642ef0083701 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml @@ -192,3 +192,103 @@ setup: - match: { data_streams.0.name: my-data-stream-1 } - match: { data_streams.0.applied_to_data_stream: false } - match: { data_streams.0.error: "Cannot set the following settings on a data stream: [index.fake_setting]" } + +--- +"Test dry run": + - requires: + cluster_features: [ "logs_stream" ] + reason: requires setting 'logs_stream' to get or set data stream settings + - do: + allowed_warnings: + - "index template [my-template] has index patterns [my-data-stream-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [ my-data-stream-* ] + data_stream: { } + template: + settings: + number_of_replicas: 0 + lifecycle.name: my-policy + + - do: + indices.create_data_stream: + name: my-data-stream-1 + + - do: + cluster.health: + index: "my-data-stream-1" + wait_for_status: green + + - do: + indices.get_data_stream_settings: + name: my-data-stream-1 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.settings: {} } + - match: { data_streams.0.effective_settings.index.number_of_shards: null } + - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" } + - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-policy" } + + - do: + indices.get_data_stream: + name: my-data-stream-1 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.settings: {} } + - match: { data_streams.0.effective_settings: null } + + - do: + indices.put_data_stream_settings: + name: my-data-stream-1 + dry_run: true + body: + index: + number_of_shards: 2 + lifecycle.name: my-new-policy + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.applied_to_data_stream: true } + - match: { data_streams.0.index_settings_results.applied_to_data_stream_only: [index.number_of_shards]} + - match: { data_streams.0.index_settings_results.applied_to_data_stream_and_backing_indices: [index.lifecycle.name] } + - match: { data_streams.0.settings.index.number_of_shards: "2" } + - match: { data_streams.0.settings.index.lifecycle.name: "my-new-policy" } + - match: { data_streams.0.effective_settings.index.number_of_shards: "2" } + - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" } + - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-new-policy" } + + - do: + indices.rollover: + alias: "my-data-stream-1" + + - do: + cluster.health: + index: "my-data-stream-1" + wait_for_status: green + + - do: + indices.get_data_stream_settings: + name: my-data-stream-1 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.settings: {} } + - match: { data_streams.0.effective_settings.index.number_of_shards: null } + - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" } + - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-policy" } + + - do: + indices.get_data_stream: + name: my-data-stream-1 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.settings: {} } + - match: { data_streams.0.effective_settings: null } + + - do: + indices.get_data_stream: + name: my-data-stream-1 + - set: { data_streams.0.indices.0.index_name: idx0name } + - set: { data_streams.0.indices.1.index_name: idx1name } + + - do: + indices.get_settings: + index: my-data-stream-1 + - match: { .$idx0name.settings.index.number_of_shards: "1" } + - match: { .$idx0name.settings.index.lifecycle.name: "my-policy" } + - match: { .$idx1name.settings.index.number_of_shards: "1" } + - match: { .$idx1name.settings.index.lifecycle.name: "my-policy" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_stream_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_stream_settings.json index ca6ed7d8faf30..96e6af8990357 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_stream_settings.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_stream_settings.json @@ -27,6 +27,11 @@ ] }, "params":{ + "dry_run":{ + "type":"boolean", + "description":"Perform a dry run but do not actually change any settings", + "default":false + }, "timeout":{ "type":"time", "description":"Specify timeout for acknowledging the cluster state update" diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 4d0ee63c800ce..9169b81d9fb17 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -240,6 +240,7 @@ static TransportVersion def(int id) { public static final TransportVersion ML_INFERENCE_MISTRAL_CHAT_COMPLETION_ADDED_8_19 = def(8_841_0_47); public static final TransportVersion ML_INFERENCE_ELASTIC_RERANK_ADDED_8_19 = def(8_841_0_48); public static final TransportVersion SETTINGS_IN_DATA_STREAMS = def(8_841_0_49); // TODO NO NO NO NO NO, also rename this + public static final TransportVersion SETTINGS_IN_DATA_STREAMS_DRY_RUN = def(8_841_0_50); // TODO NO NO NO NO NO, also rename this /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsAction.java index 42ab1dd15ff38..be79a0755d4bd 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsAction.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.datastreams; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.IndicesRequest; @@ -47,10 +48,16 @@ public UpdateDataStreamSettingsAction() { public static class Request extends AcknowledgedRequest implements IndicesRequest.Replaceable { private final Settings settings; private String[] dataStreamNames = Strings.EMPTY_ARRAY; + private final boolean dryRun; public Request(Settings settings, TimeValue masterNodeTimeout, TimeValue ackTimeout) { + this(settings, false, masterNodeTimeout, ackTimeout); + } + + public Request(Settings settings, boolean dryRun, TimeValue masterNodeTimeout, TimeValue ackTimeout) { super(masterNodeTimeout, ackTimeout); this.settings = settings; + this.dryRun = dryRun; } @Override @@ -63,6 +70,10 @@ public Settings getSettings() { return settings; } + public boolean isDryRun() { + return dryRun; + } + @Override public boolean includeDataStreams() { return true; @@ -72,6 +83,11 @@ public Request(StreamInput in) throws IOException { super(in); this.dataStreamNames = in.readStringArray(); this.settings = Settings.readSettingsFromStream(in); + if (in.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS)) { + this.dryRun = in.readBoolean(); + } else { + this.dryRun = false; + } } @Override @@ -79,6 +95,9 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeStringArray(dataStreamNames); settings.writeTo(out); + if (out.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS_DRY_RUN)) { + out.writeBoolean(dryRun); + } } @Override @@ -103,13 +122,14 @@ public boolean equals(Object o) { Request request = (Request) o; return Arrays.equals(dataStreamNames, request.dataStreamNames) && settings.equals(request.settings) + && dryRun == request.dryRun && Objects.equals(masterNodeTimeout(), request.masterNodeTimeout()) && Objects.equals(ackTimeout(), request.ackTimeout()); } @Override public int hashCode() { - return Objects.hash(Arrays.hashCode(dataStreamNames), settings, masterNodeTimeout(), ackTimeout()); + return Objects.hash(Arrays.hashCode(dataStreamNames), settings, dryRun, masterNodeTimeout(), ackTimeout()); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java index 567fa12278121..55275c0a26daf 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java @@ -9,6 +9,7 @@ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.datastreams.ModifyDataStreamsAction; @@ -127,30 +128,16 @@ public Tuple executeTask( UpdateSettingsTask updateSettingsTask, ClusterState clusterState ) throws Exception { - Metadata metadata = clusterState.metadata(); - Metadata.Builder metadataBuilder = Metadata.builder(metadata); - Map dataStreamMap = metadata.dataStreams(); - DataStream dataStream = dataStreamMap.get(updateSettingsTask.dataStreamName); - Settings existingSettings = dataStream.getSettings(); - - Template.Builder templateBuilder = Template.builder(); - Settings.Builder mergedSettingsBuilder = Settings.builder().put(existingSettings).put(updateSettingsTask.settingsOverrides); - Settings mergedSettings = mergedSettingsBuilder.build(); - - final ComposableIndexTemplate template = lookupTemplateForDataStream(updateSettingsTask.dataStreamName, metadata); - ComposableIndexTemplate mergedTemplate = template.mergeSettings(mergedSettings); - MetadataIndexTemplateService.validateTemplate( - mergedTemplate.template().settings(), - mergedTemplate.template().mappings(), - indicesService + DataStream dataStream = createDataStreamForUpdatedDataStreamSettings( + updateSettingsTask.dataStreamName, + updateSettingsTask.settingsOverrides, + clusterState ); - - templateBuilder.settings(mergedSettingsBuilder); - DataStream.Builder dataStreamBuilder = dataStream.copy().setSettings(mergedSettings); - metadataBuilder.removeDataStream(updateSettingsTask.dataStreamName); - metadataBuilder.put(dataStreamBuilder.build()); - ClusterState updatedClusterState = ClusterState.builder(clusterState).metadata(metadataBuilder).build(); - + Metadata projectMetadata = clusterState.metadata(); + Metadata.Builder builder = Metadata.builder(projectMetadata); + builder.removeDataStream(updateSettingsTask.dataStreamName); + builder.put(dataStream); + ClusterState updatedClusterState = ClusterState.builder(clusterState).metadata(builder).build(); return new Tuple<>(updatedClusterState, updateSettingsTask); } }; @@ -387,13 +374,64 @@ public void updateSettings( TimeValue ackTimeout, String dataStreamName, Settings settingsOverrides, - ActionListener listener + boolean dryRun, + ActionListener listener ) { - updateSettingsTaskQueue.submitTask( - "updating settings on data stream", - new UpdateSettingsTask(dataStreamName, settingsOverrides, ackTimeout, listener), - masterNodeTimeout + if (dryRun) { + /* + * If this is a dry run, we'll do the settings validation and apply the changes to the data stream locally, but we won't run + * the task that actually updates the cluster state. + */ + try { + DataStream updatedDataStream = createDataStreamForUpdatedDataStreamSettings( + dataStreamName, + settingsOverrides, + clusterService.state() + ); + listener.onResponse(updatedDataStream); + } catch (Exception e) { + listener.onFailure(e); + } + } else { + UpdateSettingsTask updateSettingsTask = new UpdateSettingsTask( + dataStreamName, + settingsOverrides, + clusterService, + ackTimeout, + listener + ); + updateSettingsTaskQueue.submitTask("updating settings on data stream", updateSettingsTask, masterNodeTimeout); + } + } + + /* + * This method validates that the settings won't cause any validation problems with existing templates. If successful, a copy of the + * data stream is returned with the new settings applied. + */ + private DataStream createDataStreamForUpdatedDataStreamSettings( + String dataStreamName, + Settings settingsOverrides, + ClusterState clusterState + ) throws Exception { + Metadata metadata = clusterState.metadata(); + Map dataStreamMap = metadata.dataStreams(); + DataStream dataStream = dataStreamMap.get(dataStreamName); + Settings existingSettings = dataStream.getSettings(); + + Template.Builder templateBuilder = Template.builder(); + Settings.Builder mergedSettingsBuilder = Settings.builder().put(existingSettings).put(settingsOverrides); + Settings mergedSettings = mergedSettingsBuilder.build(); + + final ComposableIndexTemplate template = lookupTemplateForDataStream(dataStreamName, metadata); + ComposableIndexTemplate mergedTemplate = template.mergeSettings(mergedSettings); + MetadataIndexTemplateService.validateTemplate( + mergedTemplate.template().settings(), + mergedTemplate.template().mappings(), + indicesService ); + + templateBuilder.settings(mergedSettingsBuilder); + return dataStream.copy().setSettings(mergedSettings).build(); } private static void addBackingIndex( @@ -628,10 +666,17 @@ static class UpdateSettingsTask extends AckedBatchedClusterStateUpdateTask { UpdateSettingsTask( String dataStreamName, Settings settingsOverrides, + ClusterService clusterService, TimeValue ackTimeout, - ActionListener listener + ActionListener listener ) { - super(ackTimeout, listener); + super(ackTimeout, listener.safeMap(response -> { + if (response.isAcknowledged()) { + return clusterService.state().metadata().dataStreams().get(dataStreamName); + } else { + throw new ElasticsearchException("Updating settings not accepted for unknown reasons"); + } + })); this.dataStreamName = dataStreamName; this.settingsOverrides = settingsOverrides; } diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsActionRequestTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsActionRequestTests.java index 8ab80aee3fd80..e3f7e4063a4e3 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsActionRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsActionRequestTests.java @@ -33,6 +33,7 @@ protected Writeable.Reader instanceReade protected UpdateDataStreamSettingsAction.Request createTestInstance() { UpdateDataStreamSettingsAction.Request request = new UpdateDataStreamSettingsAction.Request( randomSettings(), + randomBoolean(), randomTimeValue(), randomTimeValue() ); @@ -44,9 +45,10 @@ protected UpdateDataStreamSettingsAction.Request createTestInstance() { protected UpdateDataStreamSettingsAction.Request mutateInstance(UpdateDataStreamSettingsAction.Request instance) throws IOException { String[] indices = instance.indices(); Settings settings = instance.getSettings(); + boolean dryRun = instance.isDryRun(); TimeValue masterNodeTimeout = instance.masterNodeTimeout(); TimeValue ackTimeout = instance.ackTimeout(); - switch (between(0, 3)) { + switch (between(0, 4)) { case 0 -> { indices = randomArrayValueOtherThan(indices, this::randomIndices); } @@ -54,14 +56,17 @@ protected UpdateDataStreamSettingsAction.Request mutateInstance(UpdateDataStream settings = randomValueOtherThan(settings, ComponentTemplateTests::randomSettings); } case 2 -> { - masterNodeTimeout = randomValueOtherThan(masterNodeTimeout, ESTestCase::randomTimeValue); + dryRun = dryRun == false; } case 3 -> { + masterNodeTimeout = randomValueOtherThan(masterNodeTimeout, ESTestCase::randomTimeValue); + } + case 4 -> { ackTimeout = randomValueOtherThan(ackTimeout, ESTestCase::randomTimeValue); } default -> throw new AssertionError("Should not be here"); } - return new UpdateDataStreamSettingsAction.Request(settings, masterNodeTimeout, ackTimeout).indices(indices); + return new UpdateDataStreamSettingsAction.Request(settings, dryRun, masterNodeTimeout, ackTimeout).indices(indices); } private String[] randomIndices() { From 5d8c61ee8dc0f48113f5efe2f12877994c1d512a Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 27 May 2025 15:42:08 -0500 Subject: [PATCH 07/17] Adding prefer_ilm as a whitelisted data stream setting (#128375) --- .../TransportUpdateDataStreamSettingsAction.java | 3 ++- .../test/data_stream/240_data_stream_settings.yml | 11 +++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java index 4814937ce15d9..3e6aaf105bb7e 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; @@ -51,7 +52,7 @@ public class TransportUpdateDataStreamSettingsAction extends TransportMasterNode UpdateDataStreamSettingsAction.Request, UpdateDataStreamSettingsAction.Response> { private static final Logger logger = LogManager.getLogger(TransportUpdateDataStreamSettingsAction.class); - private static final Set APPLY_TO_BACKING_INDICES = Set.of("index.lifecycle.name"); + private static final Set APPLY_TO_BACKING_INDICES = Set.of("index.lifecycle.name", IndexSettings.PREFER_ILM); private static final Set APPLY_TO_DATA_STREAM_ONLY = Set.of("index.number_of_shards"); private final MetadataDataStreamsService metadataDataStreamsService; private final MetadataUpdateSettingsService updateSettingsService; diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml index e642ef0083701..8dd15628fa130 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml @@ -51,16 +51,20 @@ setup: body: index: number_of_shards: 2 - lifecycle.name: my-new-policy + lifecycle: + name: my-new-policy + prefer_ilm: true - match: { data_streams.0.name: my-data-stream-1 } - match: { data_streams.0.applied_to_data_stream: true } - match: { data_streams.0.index_settings_results.applied_to_data_stream_only: [index.number_of_shards]} - - match: { data_streams.0.index_settings_results.applied_to_data_stream_and_backing_indices: [index.lifecycle.name] } + - length: { data_streams.0.index_settings_results.applied_to_data_stream_and_backing_indices: 2 } - match: { data_streams.0.settings.index.number_of_shards: "2" } - match: { data_streams.0.settings.index.lifecycle.name: "my-new-policy" } + - match: { data_streams.0.settings.index.lifecycle.prefer_ilm: "true" } - match: { data_streams.0.effective_settings.index.number_of_shards: "2" } - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" } - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-new-policy" } + - match: { data_streams.0.effective_settings.index.lifecycle.prefer_ilm: "true" } - do: indices.rollover: @@ -79,6 +83,7 @@ setup: - match: { data_streams.0.effective_settings.index.number_of_shards: "2" } - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" } - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-new-policy" } + - match: { data_streams.0.effective_settings.index.lifecycle.prefer_ilm: "true" } - do: indices.get_data_stream: @@ -86,6 +91,7 @@ setup: - match: { data_streams.0.name: my-data-stream-1 } - match: { data_streams.0.settings.index.number_of_shards: "2" } - match: { data_streams.0.settings.index.lifecycle.name: "my-new-policy" } + - match: { data_streams.0.settings.index.lifecycle.prefer_ilm: "true" } - match: { data_streams.0.effective_settings: null } - do: @@ -101,6 +107,7 @@ setup: - match: { .$idx0name.settings.index.lifecycle.name: "my-new-policy" } - match: { .$idx1name.settings.index.number_of_shards: "2" } - match: { .$idx1name.settings.index.lifecycle.name: "my-new-policy" } + - match: { .$idx1name.settings.index.lifecycle.prefer_ilm: "true" } --- "Test multiple data streams": From a0dae8e02837046959a0c27e8787eb9288ce1526 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 28 May 2025 09:28:11 -0500 Subject: [PATCH 08/17] Making the data stream settngs rest-api-spec consistent with the elasticsearch-specification repository (#128535) --- .../api/indices.get_data_stream_settings.json | 8 ++------ .../api/indices.put_data_stream_settings.json | 8 ++++---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream_settings.json index 5582c5ca0c54b..e6ef22588de21 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream_settings.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream_settings.json @@ -20,20 +20,16 @@ "parts":{ "name":{ "type":"string", - "description":"The name of the data stream or data stream pattern" + "description":"Comma-separated list of data streams or data stream patterns" } } } ] }, "params":{ - "timeout":{ - "type":"time", - "description":"Specify timeout for acknowledging the cluster state update" - }, "master_timeout":{ "type":"time", - "description":"Specify timeout for connection to master" + "description":"Period to wait for a connection to the master node" } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_stream_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_stream_settings.json index 96e6af8990357..a290f5eaccf3b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_stream_settings.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_stream_settings.json @@ -20,7 +20,7 @@ "parts":{ "name":{ "type":"string", - "description":"The name of the data stream or data stream pattern" + "description":"Comma-separated list of data streams or data stream patterns" } } } @@ -29,16 +29,16 @@ "params":{ "dry_run":{ "type":"boolean", - "description":"Perform a dry run but do not actually change any settings", + "description":"Whether this request should only be a dry run rather than actually applying settings", "default":false }, "timeout":{ "type":"time", - "description":"Specify timeout for acknowledging the cluster state update" + "description":"Period to wait for a response" }, "master_timeout":{ "type":"time", - "description":"Specify timeout for connection to master" + "description":"Period to wait for a connection to the master node" } }, "body":{ From 229b943b2cfb4b9e3ddd13811c27a6331cb9ef30 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Thu, 29 May 2025 09:50:14 -0500 Subject: [PATCH 09/17] Removing the data stream settings feature flag (#128594) --- .../org/elasticsearch/datastreams/DataStreamsPlugin.java | 7 ++----- .../api/indices.get_data_stream_settings.json | 3 +-- .../api/indices.put_data_stream_settings.json | 3 +-- .../action/datastreams/GetDataStreamAction.java | 8 +++----- 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java index 3a79130f392cd..a423020570377 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java @@ -26,7 +26,6 @@ import org.elasticsearch.action.datastreams.lifecycle.GetDataStreamLifecycleAction; import org.elasticsearch.action.datastreams.lifecycle.PutDataStreamLifecycleAction; import org.elasticsearch.client.internal.OriginSettingClient; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -285,10 +284,8 @@ public List getRestHandlers( handlers.add(new RestGetDataStreamOptionsAction()); handlers.add(new RestPutDataStreamOptionsAction()); handlers.add(new RestDeleteDataStreamOptionsAction()); - if (DataStream.LOGS_STREAM_FEATURE_FLAG) { - handlers.add(new RestGetDataStreamSettingsAction()); - handlers.add(new RestUpdateDataStreamSettingsAction()); - } + handlers.add(new RestGetDataStreamSettingsAction()); + handlers.add(new RestUpdateDataStreamSettingsAction()); return handlers; } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream_settings.json index e6ef22588de21..15271890a3b79 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream_settings.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream_settings.json @@ -5,8 +5,7 @@ "description":"Gets a data stream's settings" }, "stability":"stable", - "visibility": "feature_flag", - "feature_flag": "logs_stream", + "visibility": "public", "headers":{ "accept": [ "application/json"] }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_stream_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_stream_settings.json index a290f5eaccf3b..b358c2d3c864f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_stream_settings.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_data_stream_settings.json @@ -5,8 +5,7 @@ "description":"Updates a data stream's settings" }, "stability":"stable", - "visibility": "feature_flag", - "feature_flag": "logs_stream", + "visibility": "public", "headers":{ "accept": [ "application/json"] }, diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index 281310966de78..efcce1f9d5bfb 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -435,11 +435,9 @@ public XContentBuilder toXContent( builder.endArray(); builder.endObject(); } - if (DataStream.LOGS_STREAM_FEATURE_FLAG) { - builder.startObject(SETTINGS_FIELD.getPreferredName()); - dataStream.getSettings().toXContent(builder, params); - builder.endObject(); - } + builder.startObject(SETTINGS_FIELD.getPreferredName()); + dataStream.getSettings().toXContent(builder, params); + builder.endObject(); builder.startObject(DataStream.FAILURE_STORE_FIELD.getPreferredName()); builder.field(FAILURE_STORE_ENABLED.getPreferredName(), failureStoreEffectivelyEnabled); From 870872136893baee092424747df8152f1aee02d2 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 11 Jun 2025 09:59:49 -0400 Subject: [PATCH 10/17] Collapse the transport versions (and rename) --- .../java/org/elasticsearch/TransportVersions.java | 3 +-- .../datastreams/UpdateDataStreamSettingsAction.java | 11 ++--------- .../elasticsearch/cluster/metadata/DataStream.java | 4 ++-- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 9169b81d9fb17..886421072fa97 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -239,8 +239,7 @@ static TransportVersion def(int id) { public static final TransportVersion SEARCH_SOURCE_EXCLUDE_VECTORS_PARAM_8_19 = def(8_841_0_46); public static final TransportVersion ML_INFERENCE_MISTRAL_CHAT_COMPLETION_ADDED_8_19 = def(8_841_0_47); public static final TransportVersion ML_INFERENCE_ELASTIC_RERANK_ADDED_8_19 = def(8_841_0_48); - public static final TransportVersion SETTINGS_IN_DATA_STREAMS = def(8_841_0_49); // TODO NO NO NO NO NO, also rename this - public static final TransportVersion SETTINGS_IN_DATA_STREAMS_DRY_RUN = def(8_841_0_50); // TODO NO NO NO NO NO, also rename this + public static final TransportVersion SETTINGS_IN_DATA_STREAMS_8_19 = def(8_841_0_49); // TODO NO NO NO NO NO /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsAction.java index be79a0755d4bd..a21ba63c7ed8b 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsAction.java @@ -9,7 +9,6 @@ package org.elasticsearch.action.datastreams; -import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.IndicesRequest; @@ -83,11 +82,7 @@ public Request(StreamInput in) throws IOException { super(in); this.dataStreamNames = in.readStringArray(); this.settings = Settings.readSettingsFromStream(in); - if (in.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS)) { - this.dryRun = in.readBoolean(); - } else { - this.dryRun = false; - } + this.dryRun = in.readBoolean(); } @Override @@ -95,9 +90,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeStringArray(dataStreamNames); settings.writeTo(out); - if (out.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS_DRY_RUN)) { - out.writeBoolean(dryRun); - } + out.writeBoolean(dryRun); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index f0305c7ad176a..e905bddcbd20e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -272,7 +272,7 @@ public static DataStream read(StreamInput in) throws IOException { dataStreamOptions = failureStoreEnabled ? DataStreamOptions.FAILURE_STORE_ENABLED : null; } final Settings settings; - if (in.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS_8_19)) { settings = Settings.readSettingsFromStream(in); } else { settings = Settings.EMPTY; @@ -1320,7 +1320,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(DataStream.ADD_DATA_STREAM_OPTIONS_VERSION)) { out.writeOptionalWriteable(dataStreamOptions.isEmpty() ? null : dataStreamOptions); } - if (out.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS)) { + if (out.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS_8_19)) { settings.writeTo(out); } } From b3c559eb153e1abf6c7bf5f587b684a798a4cfbe Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 11 Jun 2025 12:28:25 -0400 Subject: [PATCH 11/17] Promote this snippet into being a proper TEARDOWN --- ...grate-data-stream-from-ilm-to-dsl.asciidoc | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc index c56bf218708bc..67780c1ee5623 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc @@ -11,6 +11,16 @@ being managed by data stream lifecycle. As we'll see, {ilm-init} and data stream can co-manage a data stream; however, an index can only be managed by one system at a time. +[source,console] +-------------------------------------------------- +DELETE _data_stream/dsl-data-stream +DELETE _index_template/dsl-data-stream-template +DELETE _ilm/policy/pre-dsl-ilm-policy +-------------------------------------------------- +// TEARDOWN + +////////////////////////// + [discrete] [[migrate-dsl-ilm-tldr]] ==== TL;DR @@ -488,15 +498,3 @@ GET _data_stream/dsl-data-stream Had we removed the {ilm-init} policy from the index template when we <> it, the write index of the data stream will now be `Unmanaged` because the index wouldn't have the {ilm-init} policy configured to fallback onto. - -////////////////////////// -[source,console] --------------------------------------------------- -DELETE _data_stream/dsl-data-stream -DELETE _index_template/dsl-data-stream-template -DELETE _ilm/policy/pre-dsl-ilm-policy --------------------------------------------------- -// TEST[continued] - -////////////////////////// - From 9872b79eb2b1305d1112d4a65d079a5ad5fe3706 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 11 Jun 2025 14:49:52 -0400 Subject: [PATCH 12/17] Add "settings": {} where needed to the docs --- .../change-mappings-and-settings.asciidoc | 3 ++- .../data-streams/downsampling-manual.asciidoc | 3 ++- ...rial-migrate-data-stream-from-ilm-to-dsl.asciidoc | 12 ++++++++---- docs/reference/indices/get-data-stream.asciidoc | 6 ++++-- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/reference/data-streams/change-mappings-and-settings.asciidoc b/docs/reference/data-streams/change-mappings-and-settings.asciidoc index e07e079519377..8470a55668977 100644 --- a/docs/reference/data-streams/change-mappings-and-settings.asciidoc +++ b/docs/reference/data-streams/change-mappings-and-settings.asciidoc @@ -596,7 +596,8 @@ stream's oldest backing index. "allow_custom_routing": false, "replicated": false, "index_mode": "standard", - "rollover_on_write": false + "rollover_on_write": false, + "settings": {} } ] } diff --git a/docs/reference/data-streams/downsampling-manual.asciidoc b/docs/reference/data-streams/downsampling-manual.asciidoc index cb170a7596701..ae6e1d944ae7c 100644 --- a/docs/reference/data-streams/downsampling-manual.asciidoc +++ b/docs/reference/data-streams/downsampling-manual.asciidoc @@ -382,7 +382,8 @@ This returns: "end": "2023-07-26T13:26:42.000Z" } ] - } + }, + "settings": {} } ] } diff --git a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc index 67780c1ee5623..5e271b4894dba 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc @@ -151,7 +151,8 @@ and that the next generation index will also be managed by {ilm-init}: "allow_custom_routing": false, "replicated": false, "index_mode": "standard", - "rollover_on_write": false + "rollover_on_write": false, + "settings": {} } ] } @@ -293,7 +294,8 @@ GET _data_stream/dsl-data-stream "allow_custom_routing": false, "replicated": false, "index_mode": "standard", - "rollover_on_write": false + "rollover_on_write": false, + "settings": {} } ] } @@ -377,7 +379,8 @@ GET _data_stream/dsl-data-stream "allow_custom_routing": false, "replicated": false, "index_mode": "standard", - "rollover_on_write": false + "rollover_on_write": false, + "settings": {} } ] } @@ -479,7 +482,8 @@ GET _data_stream/dsl-data-stream "allow_custom_routing": false, "replicated": false, "index_mode": "standard", - "rollover_on_write": false + "rollover_on_write": false, + "settings": {} } ] } diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc index a46d247d9a9a0..f2bffc92e7486 100644 --- a/docs/reference/indices/get-data-stream.asciidoc +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -330,7 +330,8 @@ The API returns the following response: "system": false, "allow_custom_routing": false, "replicated": false, - "rollover_on_write": false + "rollover_on_write": false, + "settings": {} }, { "name": "my-data-stream-two", @@ -361,7 +362,8 @@ The API returns the following response: "system": false, "allow_custom_routing": false, "replicated": false, - "rollover_on_write": false + "rollover_on_write": false, + "settings": {} } ] } From 3e1948f8e9979888db15cc42649060d7fa567091 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Wed, 11 Jun 2025 17:32:51 -0400 Subject: [PATCH 13/17] Remove this line of slashes --- .../tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc index 5e271b4894dba..ef07e00d24255 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc @@ -19,8 +19,6 @@ DELETE _ilm/policy/pre-dsl-ilm-policy -------------------------------------------------- // TEARDOWN -////////////////////////// - [discrete] [[migrate-dsl-ilm-tldr]] ==== TL;DR From eaa016039b6a3fc8da3db83b0f612c50fdbc0b35 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 17 Jun 2025 10:20:57 -0500 Subject: [PATCH 14/17] making TransportGetDataStreamSettingsAction a TransportLocalClusterStateAction --- .../action/TransportGetDataStreamSettingsAction.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamSettingsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamSettingsAction.java index 223887b5f6987..42a8a55c33d32 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamSettingsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamSettingsAction.java @@ -13,7 +13,7 @@ import org.elasticsearch.action.datastreams.GetDataStreamSettingsAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.action.support.TransportLocalClusterStateAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; @@ -31,7 +31,7 @@ import java.util.List; import java.util.Map; -public class TransportGetDataStreamSettingsAction extends TransportMasterNodeReadAction< +public class TransportGetDataStreamSettingsAction extends TransportLocalClusterStateAction< GetDataStreamSettingsAction.Request, GetDataStreamSettingsAction.Response> { private final IndexNameExpressionResolver indexNameExpressionResolver; @@ -48,12 +48,10 @@ public TransportGetDataStreamSettingsAction( ) { super( GetDataStreamSettingsAction.NAME, - transportService, clusterService, - threadPool, + transportService, actionFilters, GetDataStreamSettingsAction.Request::localOnly, - GetDataStreamSettingsAction.Response::localOnly, threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.indexNameExpressionResolver = indexNameExpressionResolver; @@ -66,7 +64,7 @@ protected ClusterBlockException checkBlock(GetDataStreamSettingsAction.Request r } @Override - protected void masterOperation( + protected void localClusterStateOperation( Task task, GetDataStreamSettingsAction.Request request, ClusterState state, From 06aa0e2f8264a6e0a8b9e843d794d18c9dfc743b Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 17 Jun 2025 11:17:01 -0500 Subject: [PATCH 15/17] removing transport version comment due to #129560 --- server/src/main/java/org/elasticsearch/TransportVersions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index fb684872a76ea..5cd3420b581c8 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -241,7 +241,7 @@ static TransportVersion def(int id) { public static final TransportVersion ML_INFERENCE_ELASTIC_RERANK_ADDED_8_19 = def(8_841_0_48); public static final TransportVersion NONE_CHUNKING_STRATEGY_8_19 = def(8_841_0_49); public static final TransportVersion IDP_CUSTOM_SAML_ATTRIBUTES_ALLOW_LIST_8_19 = def(8_841_0_50); - public static final TransportVersion SETTINGS_IN_DATA_STREAMS_8_19 = def(8_841_0_51); // TODO NO NO NO NO NO + public static final TransportVersion SETTINGS_IN_DATA_STREAMS_8_19 = def(8_841_0_51); /* * STOP! READ THIS FIRST! No, really, From a15da879c209f0b6121c6b1cd1b47713ad273ffb Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 17 Jun 2025 11:18:01 -0500 Subject: [PATCH 16/17] applying DataStreamSettingsIT fix from #129564 --- .../org/elasticsearch/datastreams/DataStreamSettingsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamSettingsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamSettingsIT.java index 33ff3f4ffb1b6..ae18e06953e56 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamSettingsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamSettingsIT.java @@ -209,7 +209,7 @@ public void testPutMultipleDataStreamSettings() throws Exception { .getDataStreamSettingsResponses(); assertThat(dataStreamSettingsResponses.size(), equalTo(testDataStreamNames.size())); for (int i = 0; i < testDataStreamNames.size(); i++) { - UpdateDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse = dataStreamSettingsResponses.get(0); + UpdateDataStreamSettingsAction.DataStreamSettingsResponse dataStreamSettingsResponse = dataStreamSettingsResponses.get(i); assertThat(dataStreamSettingsResponse.dataStreamSucceeded(), equalTo(true)); assertThat(dataStreamSettingsResponse.settings().get("index.number_of_shards"), equalTo(Integer.toString(numberOfShards))); assertThat( From b847cc8db1b44c3f2e4b61c35d1f6848ba714623 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 17 Jun 2025 13:49:03 -0500 Subject: [PATCH 17/17] removing unneccessary use of MasterNodeReadRequest --- .../datastreams/DataStreamSettingsIT.java | 6 +++--- .../rest/RestGetDataStreamSettingsAction.java | 7 +++---- .../datastreams/GetDataStreamSettingsAction.java | 14 ++++---------- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamSettingsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamSettingsIT.java index ae18e06953e56..a54a32edd584e 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamSettingsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamSettingsIT.java @@ -61,7 +61,7 @@ public void testPutDataStreamSettings() throws Exception { { List getSettingsResponses = client().execute( GetDataStreamSettingsAction.INSTANCE, - new GetDataStreamSettingsAction.Request(TimeValue.THIRTY_SECONDS).indices(dataStreamName) + new GetDataStreamSettingsAction.Request().indices(dataStreamName) ).actionGet().getDataStreamSettingsResponses(); assertThat(getSettingsResponses.size(), equalTo(1)); assertThat(getSettingsResponses.get(0).settings(), equalTo(Settings.EMPTY)); @@ -103,7 +103,7 @@ public void testPutDataStreamSettings() throws Exception { assertThat(settings.get("index.lifecycle.name"), equalTo(newLifecycleName)); getSettingsResponses = client().execute( GetDataStreamSettingsAction.INSTANCE, - new GetDataStreamSettingsAction.Request(TimeValue.THIRTY_SECONDS).indices(dataStreamName) + new GetDataStreamSettingsAction.Request().indices(dataStreamName) ).actionGet().getDataStreamSettingsResponses(); assertThat(getSettingsResponses.size(), equalTo(1)); assertThat(getSettingsResponses.get(0).settings(), equalTo(dataStreamSettings)); @@ -236,7 +236,7 @@ public void testPutMultipleDataStreamSettings() throws Exception { } List getSettingsResponses = client().execute( GetDataStreamSettingsAction.INSTANCE, - new GetDataStreamSettingsAction.Request(TimeValue.THIRTY_SECONDS).indices(testDataStreamNames.toArray(new String[0])) + new GetDataStreamSettingsAction.Request().indices(testDataStreamNames.toArray(new String[0])) ).actionGet().getDataStreamSettingsResponses(); assertThat(getSettingsResponses.size(), equalTo(testDataStreamNames.size())); } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamSettingsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamSettingsAction.java index 0db70b5a91f86..5752c77e19e6f 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamSettingsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamSettingsAction.java @@ -14,7 +14,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.RestUtils; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestCancellableNodeClient; @@ -39,9 +38,9 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - GetDataStreamSettingsAction.Request getDataStreamRequest = new GetDataStreamSettingsAction.Request( - RestUtils.getMasterNodeTimeout(request) - ).indices(Strings.splitStringByCommaToArray(request.param("name"))); + GetDataStreamSettingsAction.Request getDataStreamRequest = new GetDataStreamSettingsAction.Request().indices( + Strings.splitStringByCommaToArray(request.param("name")) + ); return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute( GetDataStreamSettingsAction.INSTANCE, getDataStreamRequest, diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamSettingsAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamSettingsAction.java index b0012bf1bcf39..84e74c3ecab84 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamSettingsAction.java @@ -9,19 +9,18 @@ package org.elasticsearch.action.datastreams; +import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.TransportAction; -import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -42,12 +41,11 @@ public GetDataStreamSettingsAction() { super(NAME); } - public static class Request extends MasterNodeReadRequest implements IndicesRequest.Replaceable { + public static class Request extends ActionRequest implements IndicesRequest.Replaceable { private String[] dataStreamNames; - public Request(TimeValue masterNodeTimeout) { - super(masterNodeTimeout); - local = true; // this only ever runs locally + public Request() { + super(); } public static Request localOnly(StreamInput ignored) { @@ -106,10 +104,6 @@ public Response(List dataStreamSettingsResponses) { this.dataStreamSettingsResponses = dataStreamSettingsResponses; } - public static Response localOnly(StreamInput ignored) { - return TransportAction.localOnly(); - } - public List getDataStreamSettingsResponses() { return dataStreamSettingsResponses; }