Skip to content

Commit ee2d2f3

Browse files
masseykemasseyke
authored
Adding settings to data streams (#126947)
1 parent 6a1ad16 commit ee2d2f3

File tree

12 files changed

+443
-36
lines changed

12 files changed

+443
-36
lines changed

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -257,6 +257,7 @@ public void testUpdateTimeSeriesTemporalOneBadDataStream() {
257257
ds2Indices,
258258
2,
259259
ds2.getMetadata(),
260+
ds2.getSettings(),
260261
ds2.isHidden(),
261262
ds2.isReplicated(),
262263
ds2.isSystem(),

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,7 @@ static TransportVersion def(int id) {
227227
public static final TransportVersion DOC_FIELDS_AS_LIST = def(9_061_0_00);
228228
public static final TransportVersion DENSE_VECTOR_OFF_HEAP_STATS = def(9_062_00_0);
229229
public static final TransportVersion RANDOM_SAMPLER_QUERY_BUILDER = def(9_063_0_00);
230+
public static final TransportVersion SETTINGS_IN_DATA_STREAMS = def(9_064_00_0);
230231

231232
/*
232233
* STOP! READ THIS FIRST! No, really,

‎server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import org.elasticsearch.common.io.stream.StreamInput;
1919
import org.elasticsearch.common.io.stream.StreamOutput;
2020
import org.elasticsearch.common.io.stream.Writeable;
21+
import org.elasticsearch.common.settings.Settings;
2122
import org.elasticsearch.core.Nullable;
2223
import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper;
2324
import org.elasticsearch.index.mapper.MapperService;
@@ -309,6 +310,34 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla
309310
return builder;
310311
}
311312

313+
/*
314+
* Merges the given settings into the settings in this ComposableIndexTemplate. Any null values in the
315+
* given settings are removed from the settings in the returned ComposableIndexTemplate. If this
316+
* ComposableIndexTemplate has no settings, the given settings are the only ones in the returned template
317+
* (with any null values removed). If this ComposableIndexTemplate has no template, an empty template with
318+
* those settings is created. If the given settings are empty, this ComposableIndexTemplate is just
319+
* returned unchanged. This method never changes this object.
320+
*/
321+
public ComposableIndexTemplate mergeSettings(Settings settings) {
322+
Objects.requireNonNull(settings);
323+
if (Settings.EMPTY.equals(settings)) {
324+
return this;
325+
}
326+
ComposableIndexTemplate.Builder mergedIndexTemplateBuilder = this.toBuilder();
327+
Template.Builder mergedTemplateBuilder;
328+
Settings templateSettings;
329+
if (this.template() == null) {
330+
mergedTemplateBuilder = Template.builder();
331+
templateSettings = null;
332+
} else {
333+
mergedTemplateBuilder = Template.builder(this.template());
334+
templateSettings = this.template().settings();
335+
}
336+
mergedTemplateBuilder.settings(templateSettings == null ? settings : templateSettings.merge(settings));
337+
mergedIndexTemplateBuilder.template(mergedTemplateBuilder);
338+
return mergedIndexTemplateBuilder.build();
339+
}
340+
312341
@Override
313342
public int hashCode() {
314343
return Objects.hash(

‎server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.elasticsearch.common.bytes.BytesReference;
3232
import org.elasticsearch.common.io.stream.StreamInput;
3333
import org.elasticsearch.common.io.stream.StreamOutput;
34+
import org.elasticsearch.common.settings.Settings;
3435
import org.elasticsearch.common.time.DateFormatter;
3536
import org.elasticsearch.common.time.DateFormatters;
3637
import org.elasticsearch.common.xcontent.XContentHelper;
@@ -67,6 +68,7 @@
6768
import java.util.function.Predicate;
6869
import java.util.stream.Collectors;
6970

71+
import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.lookupTemplateForDataStream;
7072
import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
7173
import static org.elasticsearch.index.IndexSettings.LIFECYCLE_ORIGINATION_DATE;
7274
import static org.elasticsearch.index.IndexSettings.PREFER_ILM_SETTING;
@@ -114,6 +116,7 @@ public final class DataStream implements SimpleDiffable<DataStream>, ToXContentO
114116
private final long generation;
115117
@Nullable
116118
private final Map<String, Object> metadata;
119+
private final Settings settings;
117120
private final boolean hidden;
118121
private final boolean replicated;
119122
private final boolean system;
@@ -146,8 +149,46 @@ public DataStream(
146149
) {
147150
this(
148151
name,
152+
indices,
149153
generation,
150154
metadata,
155+
Settings.EMPTY,
156+
hidden,
157+
replicated,
158+
system,
159+
allowCustomRouting,
160+
indexMode,
161+
lifecycle,
162+
dataStreamOptions,
163+
failureIndices,
164+
rolloverOnWrite,
165+
autoShardingEvent
166+
);
167+
}
168+
169+
// visible for testing
170+
public DataStream(
171+
String name,
172+
List<Index> indices,
173+
long generation,
174+
Map<String, Object> metadata,
175+
Settings settings,
176+
boolean hidden,
177+
boolean replicated,
178+
boolean system,
179+
boolean allowCustomRouting,
180+
IndexMode indexMode,
181+
DataStreamLifecycle lifecycle,
182+
@Nullable DataStreamOptions dataStreamOptions,
183+
List<Index> failureIndices,
184+
boolean rolloverOnWrite,
185+
@Nullable DataStreamAutoShardingEvent autoShardingEvent
186+
) {
187+
this(
188+
name,
189+
generation,
190+
metadata,
191+
settings,
151192
hidden,
152193
replicated,
153194
system,
@@ -165,6 +206,7 @@ public DataStream(
165206
String name,
166207
long generation,
167208
Map<String, Object> metadata,
209+
Settings settings,
168210
boolean hidden,
169211
boolean replicated,
170212
boolean system,
@@ -179,6 +221,7 @@ public DataStream(
179221
this.name = name;
180222
this.generation = generation;
181223
this.metadata = metadata;
224+
this.settings = Objects.requireNonNull(settings);
182225
assert system == false || hidden; // system indices must be hidden
183226
this.hidden = hidden;
184227
this.replicated = replicated;
@@ -231,10 +274,17 @@ public static DataStream read(StreamInput in) throws IOException {
231274
// is still behind a feature flag in previous version we use the default value instead of explicitly disabling it.
232275
dataStreamOptions = failureStoreEnabled ? DataStreamOptions.FAILURE_STORE_ENABLED : null;
233276
}
277+
final Settings settings;
278+
if (in.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS)) {
279+
settings = Settings.readSettingsFromStream(in);
280+
} else {
281+
settings = Settings.EMPTY;
282+
}
234283
return new DataStream(
235284
name,
236285
generation,
237286
metadata,
287+
settings,
238288
hidden,
239289
replicated,
240290
system,
@@ -325,6 +375,20 @@ public boolean rolloverOnWrite() {
325375
return backingIndices.rolloverOnWrite;
326376
}
327377

378+
public ComposableIndexTemplate getEffectiveIndexTemplate(ProjectMetadata projectMetadata) {
379+
return getMatchingIndexTemplate(projectMetadata).mergeSettings(settings);
380+
}
381+
382+
public Settings getEffectiveSettings(ProjectMetadata projectMetadata) {
383+
ComposableIndexTemplate template = getMatchingIndexTemplate(projectMetadata);
384+
Settings templateSettings = template.template() == null ? Settings.EMPTY : template.template().settings();
385+
return templateSettings.merge(settings);
386+
}
387+
388+
private ComposableIndexTemplate getMatchingIndexTemplate(ProjectMetadata projectMetadata) {
389+
return lookupTemplateForDataStream(name, projectMetadata);
390+
}
391+
328392
/**
329393
* We define that a data stream is considered internal either if it is a system index or if
330394
* its name starts with a dot.
@@ -436,6 +500,10 @@ public Map<String, Object> getMetadata() {
436500
return metadata;
437501
}
438502

503+
public Settings getSettings() {
504+
return settings;
505+
}
506+
439507
@Override
440508
public boolean isHidden() {
441509
return hidden;
@@ -1263,6 +1331,9 @@ public void writeTo(StreamOutput out) throws IOException {
12631331
if (out.getTransportVersion().onOrAfter(DataStream.ADD_DATA_STREAM_OPTIONS_VERSION)) {
12641332
out.writeOptionalWriteable(dataStreamOptions.isEmpty() ? null : dataStreamOptions);
12651333
}
1334+
if (out.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS)) {
1335+
settings.writeTo(out);
1336+
}
12661337
}
12671338

12681339
public static final ParseField NAME_FIELD = new ParseField("name");
@@ -1284,6 +1355,7 @@ public void writeTo(StreamOutput out) throws IOException {
12841355
public static final ParseField FAILURE_ROLLOVER_ON_WRITE_FIELD = new ParseField("failure_rollover_on_write");
12851356
public static final ParseField FAILURE_AUTO_SHARDING_FIELD = new ParseField("failure_auto_sharding");
12861357
public static final ParseField DATA_STREAM_OPTIONS_FIELD = new ParseField("options");
1358+
public static final ParseField SETTINGS_FIELD = new ParseField("settings");
12871359

12881360
@SuppressWarnings("unchecked")
12891361
private static final ConstructingObjectParser<DataStream, Void> PARSER = new ConstructingObjectParser<>(
@@ -1292,6 +1364,7 @@ public void writeTo(StreamOutput out) throws IOException {
12921364
(String) args[0],
12931365
(Long) args[2],
12941366
(Map<String, Object>) args[3],
1367+
args[17] == null ? Settings.EMPTY : (Settings) args[17],
12951368
args[4] != null && (boolean) args[4],
12961369
args[5] != null && (boolean) args[5],
12971370
args[6] != null && (boolean) args[6],
@@ -1358,6 +1431,7 @@ public void writeTo(StreamOutput out) throws IOException {
13581431
(p, c) -> DataStreamOptions.fromXContent(p),
13591432
DATA_STREAM_OPTIONS_FIELD
13601433
);
1434+
PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> Settings.fromXContent(p), SETTINGS_FIELD);
13611435
}
13621436

13631437
public static DataStream fromXContent(XContentParser parser) throws IOException {
@@ -1419,6 +1493,9 @@ public XContentBuilder toXContent(
14191493
backingIndices.autoShardingEvent.toXContent(builder, params);
14201494
builder.endObject();
14211495
}
1496+
builder.startObject(SETTINGS_FIELD.getPreferredName());
1497+
this.settings.toXContent(builder, params);
1498+
builder.endObject();
14221499
builder.endObject();
14231500
return builder;
14241501
}
@@ -1431,6 +1508,7 @@ public boolean equals(Object o) {
14311508
return name.equals(that.name)
14321509
&& generation == that.generation
14331510
&& Objects.equals(metadata, that.metadata)
1511+
&& Objects.equals(settings, that.settings)
14341512
&& hidden == that.hidden
14351513
&& system == that.system
14361514
&& replicated == that.replicated
@@ -1448,6 +1526,7 @@ public int hashCode() {
14481526
name,
14491527
generation,
14501528
metadata,
1529+
settings,
14511530
hidden,
14521531
system,
14531532
replicated,
@@ -1760,6 +1839,7 @@ public static class Builder {
17601839
private long generation = 1;
17611840
@Nullable
17621841
private Map<String, Object> metadata = null;
1842+
private Settings settings = Settings.EMPTY;
17631843
private boolean hidden = false;
17641844
private boolean replicated = false;
17651845
private boolean system = false;
@@ -1787,6 +1867,7 @@ private Builder(DataStream dataStream) {
17871867
name = dataStream.name;
17881868
generation = dataStream.generation;
17891869
metadata = dataStream.metadata;
1870+
settings = dataStream.settings;
17901871
hidden = dataStream.hidden;
17911872
replicated = dataStream.replicated;
17921873
system = dataStream.system;
@@ -1818,6 +1899,11 @@ public Builder setMetadata(Map<String, Object> metadata) {
18181899
return this;
18191900
}
18201901

1902+
public Builder setSettings(Settings settings) {
1903+
this.settings = settings;
1904+
return this;
1905+
}
1906+
18211907
public Builder setHidden(boolean hidden) {
18221908
this.hidden = hidden;
18231909
return this;
@@ -1878,6 +1964,7 @@ public DataStream build() {
18781964
name,
18791965
generation,
18801966
metadata,
1967+
settings,
18811968
hidden,
18821969
replicated,
18831970
system,

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,7 @@ static ClusterState createDataStream(
331331
dataStreamName,
332332
initialGeneration,
333333
template.metadata() != null ? Map.copyOf(template.metadata()) : null,
334+
Settings.EMPTY,
334335
hidden,
335336
false,
336337
isSystem,

‎server/src/main/java/org/elasticsearch/common/settings/Settings.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -876,6 +876,27 @@ public Set<String> keySet() {
876876
return newKeySet;
877877
}
878878

879+
/*
880+
* This method merges the given newSettings into this Settings, returning either a new Settings object or this if the newSettings are
881+
* empty. If any values are null in newSettings, those keys are removed from the returned object.
882+
*/
883+
public Settings merge(Settings newSettings) {
884+
Objects.requireNonNull(newSettings);
885+
if (Settings.EMPTY.equals(newSettings)) {
886+
return this;
887+
}
888+
Settings.Builder builder = Settings.builder().put(this);
889+
for (String key : newSettings.keySet()) {
890+
String rawValue = newSettings.get(key);
891+
if (rawValue == null) {
892+
builder.remove(key);
893+
} else {
894+
builder.put(key, rawValue);
895+
}
896+
}
897+
return builder.build();
898+
}
899+
879900
/**
880901
* A builder allowing to put different settings and then {@link #build()} an immutable
881902
* settings implementation. Use {@link Settings#builder()} in order to

0 commit comments

Comments
 (0)