Skip to content

Commit a3fe59f

Browse files
masseykejoegallo
masseyke
authored and
joegallo
committed
Adding settings to data streams (#126947)
1 parent 30cb238 commit a3fe59f

File tree

10 files changed

+436
-1
lines changed

10 files changed

+436
-1
lines changed

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,7 @@ public void testUpdateTimeSeriesTemporalOneBadDataStream() {
248248
ds2Indices,
249249
2,
250250
ds2.getMetadata(),
251+
ds2.getSettings(),
251252
ds2.isHidden(),
252253
ds2.isReplicated(),
253254
ds2.isSystem(),

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,7 @@ static TransportVersion def(int id) {
239239
public static final TransportVersion SEARCH_SOURCE_EXCLUDE_VECTORS_PARAM_8_19 = def(8_841_0_46);
240240
public static final TransportVersion ML_INFERENCE_MISTRAL_CHAT_COMPLETION_ADDED_8_19 = def(8_841_0_47);
241241
public static final TransportVersion ML_INFERENCE_ELASTIC_RERANK_ADDED_8_19 = def(8_841_0_48);
242+
public static final TransportVersion SETTINGS_IN_DATA_STREAMS = def(8_841_0_49); // TODO NO NO NO NO NO, also rename this
242243

243244
/*
244245
* STOP! READ THIS FIRST! No, really,

‎server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import org.elasticsearch.common.io.stream.StreamInput;
1919
import org.elasticsearch.common.io.stream.StreamOutput;
2020
import org.elasticsearch.common.io.stream.Writeable;
21+
import org.elasticsearch.common.settings.Settings;
2122
import org.elasticsearch.core.Nullable;
2223
import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper;
2324
import org.elasticsearch.index.mapper.MapperService;
@@ -309,6 +310,34 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla
309310
return builder;
310311
}
311312

313+
/*
314+
* Merges the given settings into the settings in this ComposableIndexTemplate. Any null values in the
315+
* given settings are removed from the settings in the returned ComposableIndexTemplate. If this
316+
* ComposableIndexTemplate has no settings, the given settings are the only ones in the returned template
317+
* (with any null values removed). If this ComposableIndexTemplate has no template, an empty template with
318+
* those settings is created. If the given settings are empty, this ComposableIndexTemplate is just
319+
* returned unchanged. This method never changes this object.
320+
*/
321+
public ComposableIndexTemplate mergeSettings(Settings settings) {
322+
Objects.requireNonNull(settings);
323+
if (Settings.EMPTY.equals(settings)) {
324+
return this;
325+
}
326+
ComposableIndexTemplate.Builder mergedIndexTemplateBuilder = this.toBuilder();
327+
Template.Builder mergedTemplateBuilder;
328+
Settings templateSettings;
329+
if (this.template() == null) {
330+
mergedTemplateBuilder = Template.builder();
331+
templateSettings = null;
332+
} else {
333+
mergedTemplateBuilder = Template.builder(this.template());
334+
templateSettings = this.template().settings();
335+
}
336+
mergedTemplateBuilder.settings(templateSettings == null ? settings : templateSettings.merge(settings));
337+
mergedIndexTemplateBuilder.template(mergedTemplateBuilder);
338+
return mergedIndexTemplateBuilder.build();
339+
}
340+
312341
@Override
313342
public int hashCode() {
314343
return Objects.hash(

‎server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.elasticsearch.common.bytes.BytesReference;
2929
import org.elasticsearch.common.io.stream.StreamInput;
3030
import org.elasticsearch.common.io.stream.StreamOutput;
31+
import org.elasticsearch.common.settings.Settings;
3132
import org.elasticsearch.common.time.DateFormatter;
3233
import org.elasticsearch.common.time.DateFormatters;
3334
import org.elasticsearch.common.xcontent.XContentHelper;
@@ -64,6 +65,7 @@
6465
import java.util.function.Predicate;
6566
import java.util.stream.Collectors;
6667

68+
import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.lookupTemplateForDataStream;
6769
import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
6870
import static org.elasticsearch.index.IndexSettings.LIFECYCLE_ORIGINATION_DATE;
6971
import static org.elasticsearch.index.IndexSettings.PREFER_ILM_SETTING;
@@ -105,6 +107,7 @@ public final class DataStream implements SimpleDiffable<DataStream>, ToXContentO
105107
private final long generation;
106108
@Nullable
107109
private final Map<String, Object> metadata;
110+
private final Settings settings;
108111
private final boolean hidden;
109112
private final boolean replicated;
110113
private final boolean system;
@@ -137,8 +140,46 @@ public DataStream(
137140
) {
138141
this(
139142
name,
143+
indices,
140144
generation,
141145
metadata,
146+
Settings.EMPTY,
147+
hidden,
148+
replicated,
149+
system,
150+
allowCustomRouting,
151+
indexMode,
152+
lifecycle,
153+
dataStreamOptions,
154+
failureIndices,
155+
rolloverOnWrite,
156+
autoShardingEvent
157+
);
158+
}
159+
160+
// visible for testing
161+
public DataStream(
162+
String name,
163+
List<Index> indices,
164+
long generation,
165+
Map<String, Object> metadata,
166+
Settings settings,
167+
boolean hidden,
168+
boolean replicated,
169+
boolean system,
170+
boolean allowCustomRouting,
171+
IndexMode indexMode,
172+
DataStreamLifecycle lifecycle,
173+
@Nullable DataStreamOptions dataStreamOptions,
174+
List<Index> failureIndices,
175+
boolean rolloverOnWrite,
176+
@Nullable DataStreamAutoShardingEvent autoShardingEvent
177+
) {
178+
this(
179+
name,
180+
generation,
181+
metadata,
182+
settings,
142183
hidden,
143184
replicated,
144185
system,
@@ -156,6 +197,7 @@ public DataStream(
156197
String name,
157198
long generation,
158199
Map<String, Object> metadata,
200+
Settings settings,
159201
boolean hidden,
160202
boolean replicated,
161203
boolean system,
@@ -170,6 +212,7 @@ public DataStream(
170212
this.name = name;
171213
this.generation = generation;
172214
this.metadata = metadata;
215+
this.settings = Objects.requireNonNull(settings);
173216
// The following assert is commented out, because system data streams created before 8.1 are not hidden,
174217
// but should be updated to hidden by 8.18/8.19 (SystemIndexMetadataUpgradeService)
175218
// assert system == false || hidden; // system indices must be hidden
@@ -226,10 +269,17 @@ public static DataStream read(StreamInput in) throws IOException {
226269
// is still behind a feature flag in previous version we use the default value instead of explicitly disabling it.
227270
dataStreamOptions = failureStoreEnabled ? DataStreamOptions.FAILURE_STORE_ENABLED : null;
228271
}
272+
final Settings settings;
273+
if (in.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS)) {
274+
settings = Settings.readSettingsFromStream(in);
275+
} else {
276+
settings = Settings.EMPTY;
277+
}
229278
return new DataStream(
230279
name,
231280
generation,
232281
metadata,
282+
settings,
233283
hidden,
234284
replicated,
235285
system,
@@ -320,6 +370,20 @@ public boolean rolloverOnWrite() {
320370
return backingIndices.rolloverOnWrite;
321371
}
322372

373+
public ComposableIndexTemplate getEffectiveIndexTemplate(Metadata metadata) {
374+
return getMatchingIndexTemplate(metadata).mergeSettings(settings);
375+
}
376+
377+
public Settings getEffectiveSettings(Metadata metadata) {
378+
ComposableIndexTemplate template = getMatchingIndexTemplate(metadata);
379+
Settings templateSettings = template.template() == null ? Settings.EMPTY : template.template().settings();
380+
return templateSettings.merge(settings);
381+
}
382+
383+
private ComposableIndexTemplate getMatchingIndexTemplate(Metadata metadata) {
384+
return lookupTemplateForDataStream(name, metadata);
385+
}
386+
323387
/**
324388
* We define that a data stream is considered internal either if it is a system index or if
325389
* its name starts with a dot.
@@ -431,6 +495,10 @@ public Map<String, Object> getMetadata() {
431495
return metadata;
432496
}
433497

498+
public Settings getSettings() {
499+
return settings;
500+
}
501+
434502
@Override
435503
public boolean isHidden() {
436504
return hidden;
@@ -1250,6 +1318,9 @@ public void writeTo(StreamOutput out) throws IOException {
12501318
if (out.getTransportVersion().onOrAfter(DataStream.ADD_DATA_STREAM_OPTIONS_VERSION)) {
12511319
out.writeOptionalWriteable(dataStreamOptions.isEmpty() ? null : dataStreamOptions);
12521320
}
1321+
if (out.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS)) {
1322+
settings.writeTo(out);
1323+
}
12531324
}
12541325

12551326
public static final ParseField NAME_FIELD = new ParseField("name");
@@ -1271,6 +1342,7 @@ public void writeTo(StreamOutput out) throws IOException {
12711342
public static final ParseField FAILURE_ROLLOVER_ON_WRITE_FIELD = new ParseField("failure_rollover_on_write");
12721343
public static final ParseField FAILURE_AUTO_SHARDING_FIELD = new ParseField("failure_auto_sharding");
12731344
public static final ParseField DATA_STREAM_OPTIONS_FIELD = new ParseField("options");
1345+
public static final ParseField SETTINGS_FIELD = new ParseField("settings");
12741346

12751347
@SuppressWarnings("unchecked")
12761348
private static final ConstructingObjectParser<DataStream, Void> PARSER = new ConstructingObjectParser<>(
@@ -1279,6 +1351,7 @@ public void writeTo(StreamOutput out) throws IOException {
12791351
(String) args[0],
12801352
(Long) args[2],
12811353
(Map<String, Object>) args[3],
1354+
args[17] == null ? Settings.EMPTY : (Settings) args[17],
12821355
args[4] != null && (boolean) args[4],
12831356
args[5] != null && (boolean) args[5],
12841357
args[6] != null && (boolean) args[6],
@@ -1349,6 +1422,7 @@ public void writeTo(StreamOutput out) throws IOException {
13491422
(p, c) -> DataStreamOptions.fromXContent(p),
13501423
DATA_STREAM_OPTIONS_FIELD
13511424
);
1425+
PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> Settings.fromXContent(p), SETTINGS_FIELD);
13521426
}
13531427

13541428
public static DataStream fromXContent(XContentParser parser) throws IOException {
@@ -1410,6 +1484,9 @@ public XContentBuilder toXContent(
14101484
backingIndices.autoShardingEvent.toXContent(builder, params);
14111485
builder.endObject();
14121486
}
1487+
builder.startObject(SETTINGS_FIELD.getPreferredName());
1488+
this.settings.toXContent(builder, params);
1489+
builder.endObject();
14131490
builder.endObject();
14141491
return builder;
14151492
}
@@ -1422,6 +1499,7 @@ public boolean equals(Object o) {
14221499
return name.equals(that.name)
14231500
&& generation == that.generation
14241501
&& Objects.equals(metadata, that.metadata)
1502+
&& Objects.equals(settings, that.settings)
14251503
&& hidden == that.hidden
14261504
&& system == that.system
14271505
&& replicated == that.replicated
@@ -1439,6 +1517,7 @@ public int hashCode() {
14391517
name,
14401518
generation,
14411519
metadata,
1520+
settings,
14421521
hidden,
14431522
system,
14441523
replicated,
@@ -1751,6 +1830,7 @@ public static class Builder {
17511830
private long generation = 1;
17521831
@Nullable
17531832
private Map<String, Object> metadata = null;
1833+
private Settings settings = Settings.EMPTY;
17541834
private boolean hidden = false;
17551835
private boolean replicated = false;
17561836
private boolean system = false;
@@ -1778,6 +1858,7 @@ private Builder(DataStream dataStream) {
17781858
name = dataStream.name;
17791859
generation = dataStream.generation;
17801860
metadata = dataStream.metadata;
1861+
settings = dataStream.settings;
17811862
hidden = dataStream.hidden;
17821863
replicated = dataStream.replicated;
17831864
system = dataStream.system;
@@ -1809,6 +1890,11 @@ public Builder setMetadata(Map<String, Object> metadata) {
18091890
return this;
18101891
}
18111892

1893+
public Builder setSettings(Settings settings) {
1894+
this.settings = settings;
1895+
return this;
1896+
}
1897+
18121898
public Builder setHidden(boolean hidden) {
18131899
this.hidden = hidden;
18141900
return this;
@@ -1869,6 +1955,7 @@ public DataStream build() {
18691955
name,
18701956
generation,
18711957
metadata,
1958+
settings,
18721959
hidden,
18731960
replicated,
18741961
system,

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -321,6 +321,7 @@ static ClusterState createDataStream(
321321
dataStreamName,
322322
initialGeneration,
323323
template.metadata() != null ? Map.copyOf(template.metadata()) : null,
324+
Settings.EMPTY,
324325
hidden,
325326
false,
326327
isSystem,

‎server/src/main/java/org/elasticsearch/common/settings/Settings.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -876,6 +876,27 @@ public Set<String> keySet() {
876876
return newKeySet;
877877
}
878878

879+
/*
880+
* This method merges the given newSettings into this Settings, returning either a new Settings object or this if the newSettings are
881+
* empty. If any values are null in newSettings, those keys are removed from the returned object.
882+
*/
883+
public Settings merge(Settings newSettings) {
884+
Objects.requireNonNull(newSettings);
885+
if (Settings.EMPTY.equals(newSettings)) {
886+
return this;
887+
}
888+
Settings.Builder builder = Settings.builder().put(this);
889+
for (String key : newSettings.keySet()) {
890+
String rawValue = newSettings.get(key);
891+
if (rawValue == null) {
892+
builder.remove(key);
893+
} else {
894+
builder.put(key, rawValue);
895+
}
896+
}
897+
return builder.build();
898+
}
899+
879900
/**
880901
* A builder allowing to put different settings and then {@link #build()} an immutable
881902
* settings implementation. Use {@link Settings#builder()} in order to

0 commit comments

Comments
 (0)