Skip to content

Commit c4188b5

Browse files
salvatore-campagnaelasticmachine
salvatore-campagna
and
elasticmachine
authored
[8.x] Introduce index.mapping.source.mode setting to override _source.mode (#114433) (#114680)
* Introduce `index.mapping.source.mode` setting to override `_source.mode` (#114433) * featur : introduce index.mapping.source.mode setting Introduce a new `index.mapper.source.mode` setting which will be used to override the mapping level `_source.mode`. For now the mapping level setting will stay and be deprecated later with another PR. The setting takes precedence always precedence. When not defined the index mode is used and can be overridden by the _source.mode mapping level definition. (cherry picked from commit edcabb8) * fix: replace return switch with switch case * fix: stored source mode not supported in 8.16 We also update a few error messages to account for a few minor differences. * Revert "fix: stored source mode not supported in 8.16" This reverts commit 2e523c3. * fix: stored source mode not supported in 8.16 We also update a few error messages to account for a few minor differences. * fix: update error message for time_series --------- Co-authored-by: Elastic Machine <[email protected]>
1 parent e4a3fd6 commit c4188b5

File tree

5 files changed

+929
-22
lines changed

5 files changed

+929
-22
lines changed

‎server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.elasticsearch.index.mapper.FieldMapper;
3636
import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
3737
import org.elasticsearch.index.mapper.MapperService;
38+
import org.elasticsearch.index.mapper.SourceFieldMapper;
3839
import org.elasticsearch.index.similarity.SimilarityService;
3940
import org.elasticsearch.index.store.FsDirectoryFactory;
4041
import org.elasticsearch.index.store.Store;
@@ -187,6 +188,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
187188
FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING,
188189
IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING,
189190
IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING,
191+
SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING,
190192

191193
// validate that built-in similarities don't get redefined
192194
Setting.groupSetting("index.similarity.", (s) -> {

‎server/src/main/java/org/elasticsearch/index/IndexSettings.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.elasticsearch.features.NodeFeature;
2929
import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
3030
import org.elasticsearch.index.mapper.Mapper;
31+
import org.elasticsearch.index.mapper.SourceFieldMapper;
3132
import org.elasticsearch.index.translog.Translog;
3233
import org.elasticsearch.ingest.IngestService;
3334
import org.elasticsearch.node.Node;
@@ -807,6 +808,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
807808
private volatile long mappingDimensionFieldsLimit;
808809
private volatile boolean skipIgnoredSourceWrite;
809810
private volatile boolean skipIgnoredSourceRead;
811+
private final SourceFieldMapper.Mode indexMappingSourceMode;
810812

811813
/**
812814
* The maximum number of refresh listeners allows on this shard.
@@ -967,6 +969,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
967969
es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING);
968970
skipIgnoredSourceWrite = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING);
969971
skipIgnoredSourceRead = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING);
972+
indexMappingSourceMode = scopedSettings.get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING);
970973

971974
scopedSettings.addSettingsUpdateConsumer(
972975
MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING,
@@ -1646,6 +1649,10 @@ private void setSkipIgnoredSourceRead(boolean value) {
16461649
this.skipIgnoredSourceRead = value;
16471650
}
16481651

1652+
public SourceFieldMapper.Mode getIndexMappingSourceMode() {
1653+
return indexMappingSourceMode;
1654+
}
1655+
16491656
/**
16501657
* The bounds for {@code @timestamp} on this index or
16511658
* {@code null} if there are no bounds.

‎server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java‎

Lines changed: 99 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,13 @@
1818
import org.elasticsearch.common.Explicit;
1919
import org.elasticsearch.common.Strings;
2020
import org.elasticsearch.common.bytes.BytesReference;
21+
import org.elasticsearch.common.settings.Setting;
2122
import org.elasticsearch.common.settings.Settings;
2223
import org.elasticsearch.common.util.CollectionUtils;
2324
import org.elasticsearch.core.Nullable;
2425
import org.elasticsearch.features.NodeFeature;
2526
import org.elasticsearch.index.IndexMode;
27+
import org.elasticsearch.index.IndexSettings;
2628
import org.elasticsearch.index.IndexVersions;
2729
import org.elasticsearch.index.query.QueryShardException;
2830
import org.elasticsearch.index.query.SearchExecutionContext;
@@ -59,8 +61,20 @@ public class SourceFieldMapper extends MetadataFieldMapper {
5961

6062
public static final String LOSSY_PARAMETERS_ALLOWED_SETTING_NAME = "index.lossy.source-mapping-parameters";
6163

64+
public static final Setting<Mode> INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting(SourceFieldMapper.Mode.class, settings -> {
65+
final IndexMode indexMode = IndexSettings.MODE.get(settings);
66+
67+
switch (indexMode) {
68+
case LOGSDB:
69+
case TIME_SERIES:
70+
return Mode.SYNTHETIC.name();
71+
default:
72+
return Mode.STORED.name();
73+
}
74+
}, "index.mapping.source.mode", value -> {}, Setting.Property.Final, Setting.Property.IndexScope);
75+
6276
/** The source mode */
63-
private enum Mode {
77+
public enum Mode {
6478
DISABLED,
6579
STORED,
6680
SYNTHETIC
@@ -93,6 +107,15 @@ private enum Mode {
93107
true
94108
);
95109

110+
private static final SourceFieldMapper TSDB_DEFAULT_STORED = new SourceFieldMapper(
111+
Mode.STORED,
112+
Explicit.IMPLICIT_TRUE,
113+
Strings.EMPTY_ARRAY,
114+
Strings.EMPTY_ARRAY,
115+
IndexMode.TIME_SERIES,
116+
true
117+
);
118+
96119
private static final SourceFieldMapper TSDB_DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper(
97120
Mode.SYNTHETIC,
98121
Explicit.IMPLICIT_TRUE,
@@ -102,6 +125,15 @@ private enum Mode {
102125
false
103126
);
104127

128+
private static final SourceFieldMapper TSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED = new SourceFieldMapper(
129+
Mode.STORED,
130+
Explicit.IMPLICIT_TRUE,
131+
Strings.EMPTY_ARRAY,
132+
Strings.EMPTY_ARRAY,
133+
IndexMode.TIME_SERIES,
134+
false
135+
);
136+
105137
private static final SourceFieldMapper LOGSDB_DEFAULT = new SourceFieldMapper(
106138
Mode.SYNTHETIC,
107139
Explicit.IMPLICIT_TRUE,
@@ -111,6 +143,15 @@ private enum Mode {
111143
true
112144
);
113145

146+
private static final SourceFieldMapper LOGSDB_DEFAULT_STORED = new SourceFieldMapper(
147+
Mode.STORED,
148+
Explicit.IMPLICIT_TRUE,
149+
Strings.EMPTY_ARRAY,
150+
Strings.EMPTY_ARRAY,
151+
IndexMode.LOGSDB,
152+
true
153+
);
154+
114155
private static final SourceFieldMapper LOGSDB_DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper(
115156
Mode.SYNTHETIC,
116157
Explicit.IMPLICIT_TRUE,
@@ -120,6 +161,15 @@ private enum Mode {
120161
false
121162
);
122163

164+
private static final SourceFieldMapper LOGSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED = new SourceFieldMapper(
165+
Mode.STORED,
166+
Explicit.IMPLICIT_TRUE,
167+
Strings.EMPTY_ARRAY,
168+
Strings.EMPTY_ARRAY,
169+
IndexMode.LOGSDB,
170+
false
171+
);
172+
123173
/*
124174
* Synthetic source was added as the default for TSDB in v.8.7. The legacy field mapper below
125175
* is used in bwc tests and mixed clusters containing time series indexes created in an earlier version.
@@ -194,6 +244,8 @@ public static class Builder extends MetadataFieldMapper.Builder {
194244
m -> Arrays.asList(toType(m).excludes)
195245
);
196246

247+
private final Settings settings;
248+
197249
private final IndexMode indexMode;
198250

199251
private final boolean supportsNonDefaultParameterValues;
@@ -207,6 +259,7 @@ public Builder(
207259
boolean enableRecoverySource
208260
) {
209261
super(Defaults.NAME);
262+
this.settings = settings;
210263
this.indexMode = indexMode;
211264
this.supportsNonDefaultParameterValues = supportsCheckForNonDefaultParams == false
212265
|| settings.getAsBoolean(LOSSY_PARAMETERS_ALLOWED_SETTING_NAME, true);
@@ -223,10 +276,10 @@ protected Parameter<?>[] getParameters() {
223276
return new Parameter<?>[] { enabled, mode, includes, excludes };
224277
}
225278

226-
private boolean isDefault() {
227-
Mode m = mode.get();
228-
if (m != null
229-
&& (((indexMode != null && indexMode.isSyntheticSourceEnabled() && m == Mode.SYNTHETIC) == false) || m == Mode.DISABLED)) {
279+
private boolean isDefault(final Mode sourceMode) {
280+
if (sourceMode != null
281+
&& (((indexMode != null && indexMode.isSyntheticSourceEnabled() && sourceMode == Mode.SYNTHETIC) == false)
282+
|| sourceMode == Mode.DISABLED)) {
230283
return false;
231284
}
232285
return enabled.get().value() && includes.getValue().isEmpty() && excludes.getValue().isEmpty();
@@ -242,12 +295,14 @@ public SourceFieldMapper build() {
242295
throw new MapperParsingException("Cannot set both [mode] and [enabled] parameters");
243296
}
244297
}
245-
if (isDefault()) {
246-
return switch (indexMode) {
247-
case TIME_SERIES -> enableRecoverySource ? TSDB_DEFAULT : TSDB_DEFAULT_NO_RECOVERY_SOURCE;
248-
case LOGSDB -> enableRecoverySource ? LOGSDB_DEFAULT : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE;
249-
default -> enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE;
250-
};
298+
// NOTE: if the `index.mapper.source.mode` exists it takes precedence to determine the source mode for `_source`
299+
// otherwise the mode is determined according to `index.mode` and `_source.mode`.
300+
final Mode sourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings)
301+
? INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings)
302+
: mode.get();
303+
if (isDefault(sourceMode)) {
304+
return resolveSourceMode(indexMode, sourceMode, enableRecoverySource);
305+
251306
}
252307
if (supportsNonDefaultParameterValues == false) {
253308
List<String> disallowed = new ArrayList<>();
@@ -271,8 +326,9 @@ public SourceFieldMapper build() {
271326
);
272327
}
273328
}
329+
274330
SourceFieldMapper sourceFieldMapper = new SourceFieldMapper(
275-
mode.get(),
331+
sourceMode,
276332
enabled.get(),
277333
includes.getValue().toArray(Strings.EMPTY_ARRAY),
278334
excludes.getValue().toArray(Strings.EMPTY_ARRAY),
@@ -287,21 +343,42 @@ public SourceFieldMapper build() {
287343

288344
}
289345

346+
private static SourceFieldMapper resolveSourceMode(final IndexMode indexMode, final Mode sourceMode, boolean enableRecoverySource) {
347+
if (indexMode == IndexMode.STANDARD) {
348+
return enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE;
349+
}
350+
final SourceFieldMapper syntheticWithoutRecoverySource = indexMode == IndexMode.TIME_SERIES
351+
? TSDB_DEFAULT_NO_RECOVERY_SOURCE
352+
: LOGSDB_DEFAULT_NO_RECOVERY_SOURCE;
353+
final SourceFieldMapper syntheticWithRecoverySource = indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT : LOGSDB_DEFAULT;
354+
final SourceFieldMapper storedWithoutRecoverySource = indexMode == IndexMode.TIME_SERIES
355+
? TSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED
356+
: LOGSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED;
357+
final SourceFieldMapper storedWithRecoverySource = indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT_STORED : LOGSDB_DEFAULT_STORED;
358+
359+
switch (sourceMode) {
360+
case SYNTHETIC:
361+
return enableRecoverySource ? syntheticWithRecoverySource : syntheticWithoutRecoverySource;
362+
case STORED:
363+
return enableRecoverySource ? storedWithRecoverySource : storedWithoutRecoverySource;
364+
case DISABLED:
365+
throw new IllegalArgumentException("_source cannot be disabled in index using [" + indexMode + "] index mode");
366+
default:
367+
throw new IllegalStateException("Unexpected value: " + sourceMode);
368+
}
369+
}
370+
290371
public static final TypeParser PARSER = new ConfigurableTypeParser(c -> {
291-
var indexMode = c.getIndexSettings().getMode();
372+
final IndexMode indexMode = c.getIndexSettings().getMode();
292373
boolean enableRecoverySource = INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(c.getSettings());
374+
final Mode settingSourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(c.getSettings());
375+
293376
if (indexMode.isSyntheticSourceEnabled()) {
294-
if (indexMode == IndexMode.TIME_SERIES) {
295-
if (c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0)) {
296-
return enableRecoverySource ? TSDB_DEFAULT : TSDB_DEFAULT_NO_RECOVERY_SOURCE;
297-
} else {
298-
return enableRecoverySource ? TSDB_LEGACY_DEFAULT : TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE;
299-
}
300-
} else if (indexMode == IndexMode.LOGSDB) {
301-
return enableRecoverySource ? LOGSDB_DEFAULT : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE;
377+
if (indexMode == IndexMode.TIME_SERIES && c.getIndexSettings().getIndexVersionCreated().before(IndexVersions.V_8_7_0)) {
378+
return enableRecoverySource ? TSDB_LEGACY_DEFAULT : TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE;
302379
}
303380
}
304-
return enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE;
381+
return resolveSourceMode(indexMode, settingSourceMode, enableRecoverySource);
305382
},
306383
c -> new Builder(
307384
c.getIndexSettings().getMode(),

‎x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import org.elasticsearch.index.MapperTestUtils;
1919
import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
2020
import org.elasticsearch.index.mapper.MapperService;
21+
import org.elasticsearch.index.mapper.SourceFieldMapper;
2122
import org.elasticsearch.test.ESTestCase;
2223
import org.elasticsearch.xpack.ccr.Ccr;
2324
import org.elasticsearch.xpack.ccr.CcrSettings;
@@ -334,6 +335,7 @@ public void testDynamicIndexSettingsAreClassified() {
334335
replicatedSettings.add(IndexSettings.PREFER_ILM_SETTING);
335336
replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING);
336337
replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING);
338+
replicatedSettings.add(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING);
337339

338340
for (Setting<?> setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) {
339341
// removed settings have no effect, they are only there for BWC

0 commit comments

Comments
 (0)