Skip to content

Add index setting to control whether fields are indexed by default. #132525

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 7 commits into
base: main
Choose a base branch
from
Draft

Add index setting to control whether fields are indexed by default. #132525

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
cb244d6
Add index setting to control whether fields are indexed by default.
martijnvg Aug 7, 2025
2b585a2
[CI] Auto commit changes from spotless
Aug 7, 2025
642e969
boolean field type
martijnvg Aug 7, 2025
c0e4014
flattened field type
martijnvg Aug 7, 2025
2ce22c3
scaled_float field type
martijnvg Aug 7, 2025
83753f5
fix test compile error
martijnvg Aug 7, 2025
43008cd
Make DataStreamTimestampFieldMapper's `@timestamp` field validation l…
martijnvg Aug 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions ...c/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,24 +119,25 @@ public void testValidateInvalidFieldType() {
);
}

public void testValidateNotIndexed() {
public void testValidateNotSearchable() {
Exception e = expectThrows(IllegalArgumentException.class, () -> createMapperService(timestampMapping(true, b -> {
b.startObject("@timestamp");
b.field("type", "date");
b.field("index", false);
b.field("doc_values", false);
b.endObject();
})));
assertThat(e.getMessage(), equalTo("data stream timestamp field [@timestamp] is not indexed"));
assertThat(e.getMessage(), equalTo("data stream timestamp field [@timestamp] is not searchable"));
}

public void testValidateNotDocValues() {
public void testValidateNotAggregatable() {
Exception e = expectThrows(IllegalArgumentException.class, () -> createMapperService(timestampMapping(true, b -> {
b.startObject("@timestamp");
b.field("type", "date");
b.field("doc_values", false);
b.endObject();
})));
assertThat(e.getMessage(), equalTo("data stream timestamp field [@timestamp] doesn't have doc values"));
assertThat(e.getMessage(), equalTo("data stream timestamp field [@timestamp] is not aggregatable"));
}

public void testValidateNullValue() {
Expand Down Expand Up @@ -436,7 +437,7 @@ public void testFieldTypeWithDocValuesSkipper_DocValuesFalseEnabledDocValuesSkip
);
assertEquals(
ex.getMessage(),
"data stream timestamp field [" + DataStreamTimestampFieldMapper.DEFAULT_PATH + "] doesn't have doc values"
"data stream timestamp field [" + DataStreamTimestampFieldMapper.DEFAULT_PATH + "] is not aggregatable"
);
}

Expand All @@ -460,7 +461,7 @@ public void testFieldTypeWithDocValuesSkipper_DocValuesFalseDisabledDocValuesSki
);
assertEquals(
ex.getMessage(),
"data stream timestamp field [" + DataStreamTimestampFieldMapper.DEFAULT_PATH + "] doesn't have doc values"
"data stream timestamp field [" + DataStreamTimestampFieldMapper.DEFAULT_PATH + "] is not aggregatable"
);
}

Expand Down
32 changes: 25 additions & 7 deletions ...er-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,21 +134,24 @@ public static class Builder extends FieldMapper.Builder {
private final IndexMode indexMode;
private final IndexVersion indexCreatedVersion;
private final SourceKeepMode indexSourceKeepMode;
private final boolean indexDisabledByDefault;

public Builder(
String name,
Settings settings,
IndexMode indexMode,
IndexVersion indexCreatedVersion,
SourceKeepMode indexSourceKeepMode
SourceKeepMode indexSourceKeepMode,
boolean indexDisabledByDefault
) {
this(
name,
IGNORE_MALFORMED_SETTING.get(settings),
COERCE_SETTING.get(settings),
indexMode,
indexCreatedVersion,
indexSourceKeepMode
indexSourceKeepMode,
indexDisabledByDefault
);
}

Expand All @@ -158,7 +161,8 @@ public Builder(
boolean coerceByDefault,
IndexMode indexMode,
IndexVersion indexCreatedVersion,
SourceKeepMode indexSourceKeepMode
SourceKeepMode indexSourceKeepMode,
boolean indexDisabledByDefault
) {
super(name);
this.ignoreMalformed = Parameter.explicitBoolParam(
Expand All @@ -170,6 +174,10 @@ public Builder(
this.coerce = Parameter.explicitBoolParam("coerce", true, m -> toType(m).coerce, coerceByDefault);
this.indexMode = indexMode;
this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> {
if (indexDisabledByDefault) {
return false;
}

if (indexMode == IndexMode.TIME_SERIES) {
var metricType = getMetric().getValue();
return metricType != TimeSeriesParams.MetricType.COUNTER && metricType != TimeSeriesParams.MetricType.GAUGE;
Expand All @@ -190,6 +198,7 @@ public Builder(
});
this.indexCreatedVersion = indexCreatedVersion;
this.indexSourceKeepMode = indexSourceKeepMode;
this.indexDisabledByDefault = indexDisabledByDefault;
}

Builder scalingFactor(double scalingFactor) {
Expand Down Expand Up @@ -257,7 +266,8 @@ public ScaledFloatFieldMapper build(MapperBuilderContext context) {
c.getSettings(),
c.getIndexSettings().getMode(),
c.indexVersionCreated(),
c.getIndexSettings().sourceKeepMode()
c.getIndexSettings().sourceKeepMode(),
c.getIndexSettings().isIndexDisabledByDefault()
)
);

Expand Down Expand Up @@ -592,6 +602,7 @@ public String toString() {
private final IndexVersion indexCreatedVersion;
private final String offsetsFieldName;
private final SourceKeepMode indexSourceKeepMode;
private final boolean indexDisabledByDefault;

private ScaledFloatFieldMapper(
String simpleName,
Expand All @@ -617,6 +628,7 @@ private ScaledFloatFieldMapper(
this.indexCreatedVersion = builder.indexCreatedVersion;
this.offsetsFieldName = offsetsFieldName;
this.indexSourceKeepMode = builder.indexSourceKeepMode;
this.indexDisabledByDefault = builder.indexDisabledByDefault;
}

boolean coerce() {
Expand Down Expand Up @@ -645,9 +657,15 @@ protected String contentType() {

@Override
public FieldMapper.Builder getMergeBuilder() {
return new Builder(leafName(), ignoreMalformedByDefault, coerceByDefault, indexMode, indexCreatedVersion, indexSourceKeepMode)
.metric(metricType)
.init(this);
return new Builder(
leafName(),
ignoreMalformedByDefault,
coerceByDefault,
indexMode,
indexCreatedVersion,
indexSourceKeepMode,
indexDisabledByDefault
).metric(metricType).init(this);
}

@Override
Expand Down
9 changes: 4 additions & 5 deletions ...extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,17 +222,16 @@ public void testFieldData() throws IOException {
}

public void testFetchSourceValue() throws IOException {
MappedFieldType mapper = new ScaledFloatFieldMapper.Builder("field", false, false, null, null, null).scalingFactor(100)
MappedFieldType mapper = new ScaledFloatFieldMapper.Builder("field", false, false, null, null, null, false).scalingFactor(100)
.build(MapperBuilderContext.root(false, false))
.fieldType();
assertEquals(List.of(3.14), fetchSourceValue(mapper, 3.1415926));
assertEquals(List.of(3.14), fetchSourceValue(mapper, "3.1415"));
assertEquals(List.of(), fetchSourceValue(mapper, ""));

MappedFieldType nullValueMapper = new ScaledFloatFieldMapper.Builder("field", false, false, null, null, null).scalingFactor(100)
.nullValue(2.71)
.build(MapperBuilderContext.root(false, false))
.fieldType();
MappedFieldType nullValueMapper = new ScaledFloatFieldMapper.Builder("field", false, false, null, null, null, false).scalingFactor(
100
).nullValue(2.71).build(MapperBuilderContext.root(false, false)).fieldType();
assertEquals(List.of(2.71), fetchSourceValue(nullValueMapper, ""));
}
}
3 changes: 3 additions & 0 deletions server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,9 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
if (IndexSettings.DOC_VALUES_SKIPPER) {
settings.add(IndexSettings.USE_DOC_VALUES_SKIPPER);
}
if (IndexSettings.INDEX_DISABLED_BY_DEFAULT_FEATURE_FLAG.isEnabled()) {
settings.add(IndexSettings.INDEX_DISABLED_BY_DEFAULT);
}
if (SYNTHETIC_VECTORS) {
settings.add(IndexSettings.INDEX_MAPPING_SOURCE_SYNTHETIC_VECTORS_SETTING);
}
Expand Down
14 changes: 14 additions & 0 deletions server/src/main/java/org/elasticsearch/index/IndexSettings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -685,6 +685,14 @@ public boolean isES87TSDBCodecEnabled() {
Property.Final
);

public static final FeatureFlag INDEX_DISABLED_BY_DEFAULT_FEATURE_FLAG = new FeatureFlag("index_disabled_by_default");
public static final Setting<Boolean> INDEX_DISABLED_BY_DEFAULT = Setting.boolSetting(
"index.mapping.index_disabled_by_default",
false,
Property.IndexScope,
Property.Final
);

/**
* The {@link IndexMode "mode"} of the index.
*/
Expand Down Expand Up @@ -944,6 +952,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
private final boolean recoverySourceEnabled;
private final boolean recoverySourceSyntheticEnabled;
private final boolean useDocValuesSkipper;
private final boolean indexDisabledByDefault;

/**
* The maximum number of refresh listeners allows on this shard.
Expand Down Expand Up @@ -1128,6 +1137,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
recoverySourceSyntheticEnabled = DiscoveryNode.isStateless(nodeSettings) == false
&& scopedSettings.get(RECOVERY_USE_SYNTHETIC_SOURCE_SETTING);
useDocValuesSkipper = DOC_VALUES_SKIPPER && scopedSettings.get(USE_DOC_VALUES_SKIPPER);
indexDisabledByDefault = INDEX_DISABLED_BY_DEFAULT_FEATURE_FLAG.isEnabled() && scopedSettings.get(INDEX_DISABLED_BY_DEFAULT);
seqNoIndexOptions = scopedSettings.get(SEQ_NO_INDEX_OPTIONS_SETTING);
if (recoverySourceSyntheticEnabled) {
if (DiscoveryNode.isStateless(settings)) {
Expand Down Expand Up @@ -1841,6 +1851,10 @@ public boolean useDocValuesSkipper() {
return useDocValuesSkipper;
}

public boolean isIndexDisabledByDefault() {
return indexDisabledByDefault;
}

/**
* The bounds for {@code @timestamp} on this index or
* {@code null} if there are no bounds.
Expand Down
24 changes: 18 additions & 6 deletions server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ private static BooleanFieldMapper toType(FieldMapper in) {
public static final class Builder extends FieldMapper.DimensionBuilder {

private final Parameter<Boolean> docValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true);
private final Parameter<Boolean> indexed = Parameter.indexParam(m -> toType(m).indexed, true);
private final Parameter<Boolean> indexed;
private final Parameter<Boolean> stored = Parameter.storeParam(m -> toType(m).stored, false);
private final Parameter<Explicit<Boolean>> ignoreMalformed;
private final Parameter<Boolean> nullValue = new Parameter<>(
Expand Down Expand Up @@ -106,15 +106,18 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
private final SourceKeepMode indexSourceKeepMode;

private final Parameter<Boolean> dimension;
private final boolean indexDisabledByDefault;

public Builder(
String name,
ScriptCompiler scriptCompiler,
boolean ignoreMalformedByDefault,
IndexVersion indexCreatedVersion,
SourceKeepMode indexSourceKeepMode
SourceKeepMode indexSourceKeepMode,
boolean indexDisabledByDefault
) {
super(name);
indexed = Parameter.indexParam(m -> toType(m).indexed, indexDisabledByDefault == false);
this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
this.indexCreatedVersion = Objects.requireNonNull(indexCreatedVersion);
this.ignoreMalformed = Parameter.explicitBoolParam(
Expand All @@ -140,6 +143,7 @@ public Builder(
});

this.indexSourceKeepMode = indexSourceKeepMode;
this.indexDisabledByDefault = indexDisabledByDefault;
}

public Builder dimension(boolean dimension) {
Expand Down Expand Up @@ -217,7 +221,8 @@ private FieldValues<Boolean> scriptValues() {
c.scriptCompiler(),
IGNORE_MALFORMED_SETTING.get(c.getSettings()),
c.indexVersionCreated(),
c.getIndexSettings().sourceKeepMode()
c.getIndexSettings().sourceKeepMode(),
c.getIndexSettings().isIndexDisabledByDefault()
)
);

Expand Down Expand Up @@ -523,6 +528,7 @@ public Query rangeQuery(

private final String offsetsFieldName;
private final SourceKeepMode indexSourceKeepMode;
private final boolean indexDisabledByDefault;

protected BooleanFieldMapper(
String simpleName,
Expand All @@ -546,6 +552,7 @@ protected BooleanFieldMapper(
this.storeMalformedFields = storeMalformedFields;
this.offsetsFieldName = offsetsFieldName;
this.indexSourceKeepMode = builder.indexSourceKeepMode;
this.indexDisabledByDefault = builder.indexDisabledByDefault;
}

@Override
Expand Down Expand Up @@ -634,9 +641,14 @@ protected void indexScriptValues(

@Override
public FieldMapper.Builder getMergeBuilder() {
return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion, indexSourceKeepMode).dimension(
fieldType().isDimension()
).init(this);
return new Builder(
leafName(),
scriptCompiler,
ignoreMalformedByDefault,
indexCreatedVersion,
indexSourceKeepMode,
indexDisabledByDefault
).dimension(fieldType().isDimension()).init(this);
}

@Override
Expand Down
8 changes: 4 additions & 4 deletions server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,11 +139,11 @@ public void doValidate(MappingLookup lookup) {
}

DateFieldMapper dateFieldMapper = (DateFieldMapper) mapper;
if (dateFieldMapper.fieldType().isIndexed() == false && dateFieldMapper.fieldType().hasDocValuesSkipper() == false) {
throw new IllegalArgumentException("data stream timestamp field [" + DEFAULT_PATH + "] is not indexed");
if (dateFieldMapper.fieldType().isSearchable() == false) {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should be done in a separate change.

throw new IllegalArgumentException("data stream timestamp field [" + DEFAULT_PATH + "] is not searchable");
}
if (dateFieldMapper.fieldType().hasDocValues() == false) {
throw new IllegalArgumentException("data stream timestamp field [" + DEFAULT_PATH + "] doesn't have doc values");
if (dateFieldMapper.fieldType().isAggregatable() == false) {
throw new IllegalArgumentException("data stream timestamp field [" + DEFAULT_PATH + "] is not aggregatable");
}
if (dateFieldMapper.getNullValue() != null) {
throw new IllegalArgumentException(
Expand Down
Loading