Skip to content

Commit edcabb8

Browse files
salvatore-campagnasalvatore-campagna
authored
Introduce index.mapping.source.mode setting to override _source.mode (#114433)
* featur : introduce index.mapping.source.mode setting Introduce a new `index.mapper.source.mode` setting which will be used to override the mapping level `_source.mode`. For now the mapping level setting will stay and be deprecated later with another PR. The setting takes precedence always precedence. When not defined the index mode is used and can be overridden by the _source.mode mapping level definition.
1 parent b23984d commit edcabb8

File tree

5 files changed

+950
-22
lines changed

5 files changed

+950
-22
lines changed

‎server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.elasticsearch.index.mapper.FieldMapper;
3636
import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
3737
import org.elasticsearch.index.mapper.MapperService;
38+
import org.elasticsearch.index.mapper.SourceFieldMapper;
3839
import org.elasticsearch.index.similarity.SimilarityService;
3940
import org.elasticsearch.index.store.FsDirectoryFactory;
4041
import org.elasticsearch.index.store.Store;
@@ -186,6 +187,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
186187
FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING,
187188
IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING,
188189
IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING,
190+
SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING,
189191

190192
// validate that built-in similarities don't get redefined
191193
Setting.groupSetting("index.similarity.", (s) -> {

‎server/src/main/java/org/elasticsearch/index/IndexSettings.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.elasticsearch.features.NodeFeature;
2929
import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
3030
import org.elasticsearch.index.mapper.Mapper;
31+
import org.elasticsearch.index.mapper.SourceFieldMapper;
3132
import org.elasticsearch.index.translog.Translog;
3233
import org.elasticsearch.ingest.IngestService;
3334
import org.elasticsearch.node.Node;
@@ -820,6 +821,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
820821
private volatile long mappingDimensionFieldsLimit;
821822
private volatile boolean skipIgnoredSourceWrite;
822823
private volatile boolean skipIgnoredSourceRead;
824+
private final SourceFieldMapper.Mode indexMappingSourceMode;
823825

824826
/**
825827
* The maximum number of refresh listeners allows on this shard.
@@ -980,6 +982,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
980982
es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING);
981983
skipIgnoredSourceWrite = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING);
982984
skipIgnoredSourceRead = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING);
985+
indexMappingSourceMode = scopedSettings.get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING);
983986

984987
scopedSettings.addSettingsUpdateConsumer(
985988
MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING,
@@ -1659,6 +1662,10 @@ private void setSkipIgnoredSourceRead(boolean value) {
16591662
this.skipIgnoredSourceRead = value;
16601663
}
16611664

1665+
public SourceFieldMapper.Mode getIndexMappingSourceMode() {
1666+
return indexMappingSourceMode;
1667+
}
1668+
16621669
/**
16631670
* The bounds for {@code @timestamp} on this index or
16641671
* {@code null} if there are no bounds.

‎server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java

Lines changed: 92 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,13 @@
1818
import org.elasticsearch.common.Explicit;
1919
import org.elasticsearch.common.Strings;
2020
import org.elasticsearch.common.bytes.BytesReference;
21+
import org.elasticsearch.common.settings.Setting;
2122
import org.elasticsearch.common.settings.Settings;
2223
import org.elasticsearch.common.util.CollectionUtils;
2324
import org.elasticsearch.core.Nullable;
2425
import org.elasticsearch.features.NodeFeature;
2526
import org.elasticsearch.index.IndexMode;
27+
import org.elasticsearch.index.IndexSettings;
2628
import org.elasticsearch.index.IndexVersions;
2729
import org.elasticsearch.index.query.QueryShardException;
2830
import org.elasticsearch.index.query.SearchExecutionContext;
@@ -62,8 +64,16 @@ public class SourceFieldMapper extends MetadataFieldMapper {
6264

6365
public static final String LOSSY_PARAMETERS_ALLOWED_SETTING_NAME = "index.lossy.source-mapping-parameters";
6466

67+
public static final Setting<Mode> INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting(SourceFieldMapper.Mode.class, settings -> {
68+
final IndexMode indexMode = IndexSettings.MODE.get(settings);
69+
return switch (indexMode) {
70+
case IndexMode.LOGSDB, IndexMode.TIME_SERIES -> Mode.SYNTHETIC.name();
71+
default -> Mode.STORED.name();
72+
};
73+
}, "index.mapping.source.mode", value -> {}, Setting.Property.Final, Setting.Property.IndexScope);
74+
6575
/** The source mode */
66-
private enum Mode {
76+
public enum Mode {
6777
DISABLED,
6878
STORED,
6979
SYNTHETIC
@@ -96,6 +106,15 @@ private enum Mode {
96106
true
97107
);
98108

109+
private static final SourceFieldMapper TSDB_DEFAULT_STORED = new SourceFieldMapper(
110+
Mode.STORED,
111+
Explicit.IMPLICIT_TRUE,
112+
Strings.EMPTY_ARRAY,
113+
Strings.EMPTY_ARRAY,
114+
IndexMode.TIME_SERIES,
115+
true
116+
);
117+
99118
private static final SourceFieldMapper TSDB_DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper(
100119
Mode.SYNTHETIC,
101120
Explicit.IMPLICIT_TRUE,
@@ -105,6 +124,15 @@ private enum Mode {
105124
false
106125
);
107126

127+
private static final SourceFieldMapper TSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED = new SourceFieldMapper(
128+
Mode.STORED,
129+
Explicit.IMPLICIT_TRUE,
130+
Strings.EMPTY_ARRAY,
131+
Strings.EMPTY_ARRAY,
132+
IndexMode.TIME_SERIES,
133+
false
134+
);
135+
108136
private static final SourceFieldMapper LOGSDB_DEFAULT = new SourceFieldMapper(
109137
Mode.SYNTHETIC,
110138
Explicit.IMPLICIT_TRUE,
@@ -114,6 +142,15 @@ private enum Mode {
114142
true
115143
);
116144

145+
private static final SourceFieldMapper LOGSDB_DEFAULT_STORED = new SourceFieldMapper(
146+
Mode.STORED,
147+
Explicit.IMPLICIT_TRUE,
148+
Strings.EMPTY_ARRAY,
149+
Strings.EMPTY_ARRAY,
150+
IndexMode.LOGSDB,
151+
true
152+
);
153+
117154
private static final SourceFieldMapper LOGSDB_DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper(
118155
Mode.SYNTHETIC,
119156
Explicit.IMPLICIT_TRUE,
@@ -123,6 +160,15 @@ private enum Mode {
123160
false
124161
);
125162

163+
private static final SourceFieldMapper LOGSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED = new SourceFieldMapper(
164+
Mode.STORED,
165+
Explicit.IMPLICIT_TRUE,
166+
Strings.EMPTY_ARRAY,
167+
Strings.EMPTY_ARRAY,
168+
IndexMode.LOGSDB,
169+
false
170+
);
171+
126172
/*
127173
* Synthetic source was added as the default for TSDB in v.8.7. The legacy field mapper below
128174
* is used in bwc tests and mixed clusters containing time series indexes created in an earlier version.
@@ -197,6 +243,8 @@ public static class Builder extends MetadataFieldMapper.Builder {
197243
m -> Arrays.asList(toType(m).excludes)
198244
);
199245

246+
private final Settings settings;
247+
200248
private final IndexMode indexMode;
201249

202250
private final boolean supportsNonDefaultParameterValues;
@@ -210,6 +258,7 @@ public Builder(
210258
boolean enableRecoverySource
211259
) {
212260
super(Defaults.NAME);
261+
this.settings = settings;
213262
this.indexMode = indexMode;
214263
this.supportsNonDefaultParameterValues = supportsCheckForNonDefaultParams == false
215264
|| settings.getAsBoolean(LOSSY_PARAMETERS_ALLOWED_SETTING_NAME, true);
@@ -226,10 +275,10 @@ protected Parameter<?>[] getParameters() {
226275
return new Parameter<?>[] { enabled, mode, includes, excludes };
227276
}
228277

229-
private boolean isDefault() {
230-
Mode m = mode.get();
231-
if (m != null
232-
&& (((indexMode != null && indexMode.isSyntheticSourceEnabled() && m == Mode.SYNTHETIC) == false) || m == Mode.DISABLED)) {
278+
private boolean isDefault(final Mode sourceMode) {
279+
if (sourceMode != null
280+
&& (((indexMode != null && indexMode.isSyntheticSourceEnabled() && sourceMode == Mode.SYNTHETIC) == false)
281+
|| sourceMode == Mode.DISABLED)) {
233282
return false;
234283
}
235284
return enabled.get().value() && includes.getValue().isEmpty() && excludes.getValue().isEmpty();
@@ -242,12 +291,14 @@ public SourceFieldMapper build() {
242291
throw new MapperParsingException("Cannot set both [mode] and [enabled] parameters");
243292
}
244293
}
245-
if (isDefault()) {
246-
return switch (indexMode) {
247-
case TIME_SERIES -> enableRecoverySource ? TSDB_DEFAULT : TSDB_DEFAULT_NO_RECOVERY_SOURCE;
248-
case LOGSDB -> enableRecoverySource ? LOGSDB_DEFAULT : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE;
249-
default -> enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE;
250-
};
294+
// NOTE: if the `index.mapper.source.mode` exists it takes precedence to determine the source mode for `_source`
295+
// otherwise the mode is determined according to `index.mode` and `_source.mode`.
296+
final Mode sourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings)
297+
? INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings)
298+
: mode.get();
299+
if (isDefault(sourceMode)) {
300+
return resolveSourceMode(indexMode, sourceMode, enableRecoverySource);
301+
251302
}
252303
if (supportsNonDefaultParameterValues == false) {
253304
List<String> disallowed = new ArrayList<>();
@@ -271,8 +322,9 @@ public SourceFieldMapper build() {
271322
);
272323
}
273324
}
325+
274326
SourceFieldMapper sourceFieldMapper = new SourceFieldMapper(
275-
mode.get(),
327+
sourceMode,
276328
enabled.get(),
277329
includes.getValue().toArray(Strings.EMPTY_ARRAY),
278330
excludes.getValue().toArray(Strings.EMPTY_ARRAY),
@@ -287,21 +339,39 @@ public SourceFieldMapper build() {
287339

288340
}
289341

342+
private static SourceFieldMapper resolveSourceMode(final IndexMode indexMode, final Mode sourceMode, boolean enableRecoverySource) {
343+
if (indexMode == IndexMode.STANDARD) {
344+
return enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE;
345+
}
346+
final SourceFieldMapper syntheticWithoutRecoverySource = indexMode == IndexMode.TIME_SERIES
347+
? TSDB_DEFAULT_NO_RECOVERY_SOURCE
348+
: LOGSDB_DEFAULT_NO_RECOVERY_SOURCE;
349+
final SourceFieldMapper syntheticWithRecoverySource = indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT : LOGSDB_DEFAULT;
350+
final SourceFieldMapper storedWithoutRecoverySource = indexMode == IndexMode.TIME_SERIES
351+
? TSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED
352+
: LOGSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED;
353+
final SourceFieldMapper storedWithRecoverySource = indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT_STORED : LOGSDB_DEFAULT_STORED;
354+
355+
return switch (sourceMode) {
356+
case SYNTHETIC -> enableRecoverySource ? syntheticWithRecoverySource : syntheticWithoutRecoverySource;
357+
case STORED -> enableRecoverySource ? storedWithRecoverySource : storedWithoutRecoverySource;
358+
case DISABLED -> throw new IllegalArgumentException(
359+
"_source can not be disabled in index using [" + indexMode + "] index mode"
360+
);
361+
};
362+
}
363+
290364
public static final TypeParser PARSER = new ConfigurableTypeParser(c -> {
291-
var indexMode = c.getIndexSettings().getMode();
365+
final IndexMode indexMode = c.getIndexSettings().getMode();
292366
boolean enableRecoverySource = INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(c.getSettings());
367+
final Mode settingSourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(c.getSettings());
368+
293369
if (indexMode.isSyntheticSourceEnabled()) {
294-
if (indexMode == IndexMode.TIME_SERIES) {
295-
if (c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0)) {
296-
return enableRecoverySource ? TSDB_DEFAULT : TSDB_DEFAULT_NO_RECOVERY_SOURCE;
297-
} else {
298-
return enableRecoverySource ? TSDB_LEGACY_DEFAULT : TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE;
299-
}
300-
} else if (indexMode == IndexMode.LOGSDB) {
301-
return enableRecoverySource ? LOGSDB_DEFAULT : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE;
370+
if (indexMode == IndexMode.TIME_SERIES && c.getIndexSettings().getIndexVersionCreated().before(IndexVersions.V_8_7_0)) {
371+
return enableRecoverySource ? TSDB_LEGACY_DEFAULT : TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE;
302372
}
303373
}
304-
return enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE;
374+
return resolveSourceMode(indexMode, settingSourceMode, enableRecoverySource);
305375
},
306376
c -> new Builder(
307377
c.getIndexSettings().getMode(),

‎x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import org.elasticsearch.index.MapperTestUtils;
1919
import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
2020
import org.elasticsearch.index.mapper.MapperService;
21+
import org.elasticsearch.index.mapper.SourceFieldMapper;
2122
import org.elasticsearch.test.ESTestCase;
2223
import org.elasticsearch.xpack.ccr.Ccr;
2324
import org.elasticsearch.xpack.ccr.CcrSettings;
@@ -334,6 +335,7 @@ public void testDynamicIndexSettingsAreClassified() {
334335
replicatedSettings.add(IndexSettings.PREFER_ILM_SETTING);
335336
replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING);
336337
replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING);
338+
replicatedSettings.add(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING);
337339

338340
for (Setting<?> setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) {
339341
// removed settings have no effect, they are only there for BWC

0 commit comments

Comments
 (0)