Add index setting to control whether fields are indexed by default.

modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java

@@ -119,17 +119,18 @@ public void testValidateInvalidFieldType() {
         );
     }
 
-    public void testValidateNotIndexed() {
+    public void testValidateNotSearchable() {
         Exception e = expectThrows(IllegalArgumentException.class, () -> createMapperService(timestampMapping(true, b -> {
             b.startObject("@timestamp");
             b.field("type", "date");
             b.field("index", false);
+            b.field("doc_values", false);
             b.endObject();
         })));
         assertThat(e.getMessage(), equalTo("data stream timestamp field [@timestamp] is not indexed"));
     }
 
-    public void testValidateNotDocValues() {
+    public void testValidateNotAggregatable() {
         Exception e = expectThrows(IllegalArgumentException.class, () -> createMapperService(timestampMapping(true, b -> {
             b.startObject("@timestamp");
             b.field("type", "date");

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java

@@ -136,21 +136,24 @@ public static class Builder extends FieldMapper.Builder {
         private final IndexMode indexMode;
         private final IndexVersion indexCreatedVersion;
         private final SourceKeepMode indexSourceKeepMode;
+        private final boolean indexDisabledByDefault;
 
         public Builder(
             String name,
             Settings settings,
             IndexMode indexMode,
             IndexVersion indexCreatedVersion,
-            SourceKeepMode indexSourceKeepMode
+            SourceKeepMode indexSourceKeepMode,
+            boolean indexDisabledByDefault
         ) {
             this(
                 name,
                 IGNORE_MALFORMED_SETTING.get(settings),
                 COERCE_SETTING.get(settings),
                 indexMode,
                 indexCreatedVersion,
-                indexSourceKeepMode
+                indexSourceKeepMode,
+                indexDisabledByDefault
             );
         }
 
@@ -160,7 +163,8 @@ public Builder(
             boolean coerceByDefault,
             IndexMode indexMode,
             IndexVersion indexCreatedVersion,
-            SourceKeepMode indexSourceKeepMode
+            SourceKeepMode indexSourceKeepMode,
+            boolean indexDisabledByDefault
         ) {
             super(name);
             this.ignoreMalformed = Parameter.explicitBoolParam(
@@ -172,6 +176,10 @@ public Builder(
             this.coerce = Parameter.explicitBoolParam("coerce", true, m -> toType(m).coerce, coerceByDefault);
             this.indexMode = indexMode;
             this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> {
+                if (indexDisabledByDefault) {
+                    return false;
+                }
+
                 if (indexMode == IndexMode.TIME_SERIES) {
                     var metricType = getMetric().getValue();
                     return metricType != TimeSeriesParams.MetricType.COUNTER && metricType != TimeSeriesParams.MetricType.GAUGE;
@@ -192,6 +200,7 @@ public Builder(
             });
             this.indexCreatedVersion = indexCreatedVersion;
             this.indexSourceKeepMode = indexSourceKeepMode;
+            this.indexDisabledByDefault = indexDisabledByDefault;
         }
 
         Builder scalingFactor(double scalingFactor) {
@@ -259,7 +268,8 @@ public ScaledFloatFieldMapper build(MapperBuilderContext context) {
             c.getSettings(),
             c.getIndexSettings().getMode(),
             c.indexVersionCreated(),
-            c.getIndexSettings().sourceKeepMode()
+            c.getIndexSettings().sourceKeepMode(),
+            c.getIndexSettings().isIndexDisabledByDefault()
         )
     );
 
@@ -604,6 +614,7 @@ public String toString() {
     private final IndexVersion indexCreatedVersion;
     private final String offsetsFieldName;
     private final SourceKeepMode indexSourceKeepMode;
+    private final boolean indexDisabledByDefault;
 
     private ScaledFloatFieldMapper(
         String simpleName,
@@ -629,6 +640,7 @@ private ScaledFloatFieldMapper(
         this.indexCreatedVersion = builder.indexCreatedVersion;
         this.offsetsFieldName = offsetsFieldName;
         this.indexSourceKeepMode = builder.indexSourceKeepMode;
+        this.indexDisabledByDefault = builder.indexDisabledByDefault;
     }
 
     boolean coerce() {
@@ -657,9 +669,15 @@ protected String contentType() {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), ignoreMalformedByDefault, coerceByDefault, indexMode, indexCreatedVersion, indexSourceKeepMode)
-            .metric(metricType)
-            .init(this);
+        return new Builder(
+            leafName(),
+            ignoreMalformedByDefault,
+            coerceByDefault,
+            indexMode,
+            indexCreatedVersion,
+            indexSourceKeepMode,
+            indexDisabledByDefault
+        ).metric(metricType).init(this);
     }
 
     @Override

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java

@@ -222,17 +222,16 @@ public void testFieldData() throws IOException {
     }
 
     public void testFetchSourceValue() throws IOException {
-        MappedFieldType mapper = new ScaledFloatFieldMapper.Builder("field", false, false, null, null, null).scalingFactor(100)
+        MappedFieldType mapper = new ScaledFloatFieldMapper.Builder("field", false, false, null, null, null, false).scalingFactor(100)
             .build(MapperBuilderContext.root(false, false))
             .fieldType();
         assertEquals(List.of(3.14), fetchSourceValue(mapper, 3.1415926));
         assertEquals(List.of(3.14), fetchSourceValue(mapper, "3.1415"));
         assertEquals(List.of(), fetchSourceValue(mapper, ""));
 
-        MappedFieldType nullValueMapper = new ScaledFloatFieldMapper.Builder("field", false, false, null, null, null).scalingFactor(100)
-            .nullValue(2.71)
-            .build(MapperBuilderContext.root(false, false))
-            .fieldType();
+        MappedFieldType nullValueMapper = new ScaledFloatFieldMapper.Builder("field", false, false, null, null, null, false).scalingFactor(
+            100
+        ).nullValue(2.71).build(MapperBuilderContext.root(false, false)).fieldType();
         assertEquals(List.of(2.71), fetchSourceValue(nullValueMapper, ""));
     }
 }

server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -247,6 +247,9 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         if (IndexSettings.DOC_VALUES_SKIPPER) {
             settings.add(IndexSettings.USE_DOC_VALUES_SKIPPER);
         }
+        if (IndexSettings.INDEX_DISABLED_BY_DEFAULT_FEATURE_FLAG.isEnabled()) {
+            settings.add(IndexSettings.INDEX_DISABLED_BY_DEFAULT);
+        }
         if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG) {
             settings.add(IndexSettings.USE_SYNTHETIC_ID);
         }

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -734,6 +734,14 @@ public Iterator<Setting<?>> settings() {
         Property.Final
     );
 
+    public static final FeatureFlag INDEX_DISABLED_BY_DEFAULT_FEATURE_FLAG = new FeatureFlag("index_disabled_by_default");
+    public static final Setting<Boolean> INDEX_DISABLED_BY_DEFAULT = Setting.boolSetting(
+        "index.mapping.index_disabled_by_default",
+        false,
+        Property.IndexScope,
+        Property.Final
+    );
+
     /**
      * The {@link IndexMode "mode"} of the index.
      */
@@ -996,6 +1004,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
     private final boolean recoverySourceSyntheticEnabled;
     private final boolean useDocValuesSkipper;
     private final boolean useTimeSeriesSyntheticId;
+    private final boolean indexDisabledByDefault;
 
     /**
      * The maximum number of refresh listeners allows on this shard.
@@ -1181,6 +1190,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         recoverySourceSyntheticEnabled = DiscoveryNode.isStateless(nodeSettings) == false
             && scopedSettings.get(RECOVERY_USE_SYNTHETIC_SOURCE_SETTING);
         useDocValuesSkipper = DOC_VALUES_SKIPPER && scopedSettings.get(USE_DOC_VALUES_SKIPPER);
+        indexDisabledByDefault = INDEX_DISABLED_BY_DEFAULT_FEATURE_FLAG.isEnabled() && scopedSettings.get(INDEX_DISABLED_BY_DEFAULT);
         seqNoIndexOptions = scopedSettings.get(SEQ_NO_INDEX_OPTIONS_SETTING);
         final var useSyntheticId = IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && scopedSettings.get(USE_SYNTHETIC_ID);
         if (indexMetadata.useTimeSeriesSyntheticId() != useSyntheticId) {
@@ -1943,6 +1953,10 @@ public boolean useTimeSeriesSyntheticId() {
         return useTimeSeriesSyntheticId;
     }
 
+    public boolean isIndexDisabledByDefault() {
+        return indexDisabledByDefault;
+    }
+
     /**
      * The bounds for {@code @timestamp} on this index or
      * {@code null} if there are no bounds.

server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

@@ -80,7 +80,7 @@ private static BooleanFieldMapper toType(FieldMapper in) {
     public static final class Builder extends FieldMapper.DimensionBuilder {
 
         private final Parameter<Boolean> docValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true);
-        private final Parameter<Boolean> indexed = Parameter.indexParam(m -> toType(m).indexed, true);
+        private final Parameter<Boolean> indexed;
         private final Parameter<Boolean> stored = Parameter.storeParam(m -> toType(m).stored, false);
         private final Parameter<Explicit<Boolean>> ignoreMalformed;
         private final Parameter<Boolean> nullValue = new Parameter<>(
@@ -108,15 +108,18 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
         private final SourceKeepMode indexSourceKeepMode;
 
         private final Parameter<Boolean> dimension;
+        private final boolean indexDisabledByDefault;
 
         public Builder(
             String name,
             ScriptCompiler scriptCompiler,
             boolean ignoreMalformedByDefault,
             IndexVersion indexCreatedVersion,
-            SourceKeepMode indexSourceKeepMode
+            SourceKeepMode indexSourceKeepMode,
+            boolean indexDisabledByDefault
         ) {
             super(name);
+            indexed = Parameter.indexParam(m -> toType(m).indexed, indexDisabledByDefault == false);
             this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
             this.indexCreatedVersion = Objects.requireNonNull(indexCreatedVersion);
             this.ignoreMalformed = Parameter.explicitBoolParam(
@@ -142,6 +145,7 @@ public Builder(
             });
 
             this.indexSourceKeepMode = indexSourceKeepMode;
+            this.indexDisabledByDefault = indexDisabledByDefault;
         }
 
         public Builder dimension(boolean dimension) {
@@ -219,7 +223,8 @@ private FieldValues<Boolean> scriptValues() {
             c.scriptCompiler(),
             IGNORE_MALFORMED_SETTING.get(c.getSettings()),
             c.indexVersionCreated(),
-            c.getIndexSettings().sourceKeepMode()
+            c.getIndexSettings().sourceKeepMode(),
+            c.getIndexSettings().isIndexDisabledByDefault()
         )
     );
 
@@ -530,6 +535,7 @@ public Query rangeQuery(
 
     private final String offsetsFieldName;
     private final SourceKeepMode indexSourceKeepMode;
+    private final boolean indexDisabledByDefault;
 
     protected BooleanFieldMapper(
         String simpleName,
@@ -553,6 +559,7 @@ protected BooleanFieldMapper(
         this.storeMalformedFields = storeMalformedFields;
         this.offsetsFieldName = offsetsFieldName;
         this.indexSourceKeepMode = builder.indexSourceKeepMode;
+        this.indexDisabledByDefault = builder.indexDisabledByDefault;
     }
 
     @Override
@@ -641,9 +648,14 @@ protected void indexScriptValues(
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion, indexSourceKeepMode).dimension(
-            fieldType().isDimension()
-        ).init(this);
+        return new Builder(
+            leafName(),
+            scriptCompiler,
+            ignoreMalformedByDefault,
+            indexCreatedVersion,
+            indexSourceKeepMode,
+            indexDisabledByDefault
+        ).dimension(fieldType().isDimension()).init(this);
     }
 
     @Override

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -257,7 +257,7 @@ private static DateFieldMapper toType(FieldMapper in) {
 
     public static final class Builder extends FieldMapper.Builder {
 
-        private final Parameter<Boolean> index = Parameter.indexParam(m -> toType(m).indexed, true);
+        private final Parameter<Boolean> index;
         private final Parameter<Boolean> docValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true);
         private final Parameter<Boolean> store = Parameter.storeParam(m -> toType(m).store, false);
 
@@ -288,6 +288,7 @@ public static final class Builder extends FieldMapper.Builder {
         private final IndexVersion indexCreatedVersion;
         private final ScriptCompiler scriptCompiler;
         private final IndexSettings indexSettings;
+        private final boolean indexDisabledByDefault;
 
         public Builder(
             String name,
@@ -297,6 +298,8 @@ public Builder(
             IndexSettings indexSettings
         ) {
             super(name);
+            this.indexDisabledByDefault = indexSettings.isIndexDisabledByDefault();
+            this.index = Parameter.indexParam(m -> toType(m).indexed, indexDisabledByDefault == false);
             this.resolution = resolution;
             this.indexCreatedVersion = indexSettings.getIndexVersionCreated();
             this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
@@ -1069,6 +1072,7 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) {
 
     private final boolean isDataStreamTimestampField;
     private final IndexSettings indexSettings;
+    private final boolean indexDisabledByDefault;
 
     private DateFieldMapper(
         String leafName,
@@ -1095,6 +1099,7 @@ private DateFieldMapper(
         this.scriptValues = builder.scriptValues();
         this.isDataStreamTimestampField = mappedFieldType.name().equals(DataStreamTimestampFieldMapper.DEFAULT_PATH);
         this.indexSettings = builder.indexSettings;
+        this.indexDisabledByDefault = builder.indexDisabledByDefault;
     }
 
     /**

server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java

@@ -326,14 +326,14 @@ boolean createDynamicField(Mapper.Builder builder, DocumentParserContext context
         @Override
         public boolean newDynamicStringField(DocumentParserContext context, String name) throws IOException {
             MapperBuilderContext mapperBuilderContext = context.createDynamicMapperBuilderContext();
+            var indexSettings = context.indexSettings();
             if (mapperBuilderContext.parentObjectContainsDimensions()) {
                 return createDynamicField(
-                    new KeywordFieldMapper.Builder(name, context.indexSettings().getIndexVersionCreated()),
+                    new KeywordFieldMapper.Builder(name, indexSettings.isIndexDisabledByDefault(), indexSettings.getIndexVersionCreated()),
                     context,
                     mapperBuilderContext
                 );
             } else {
-                var indexSettings = context.indexSettings();
                 return createDynamicField(
                     new TextFieldMapper.Builder(
                         name,
@@ -343,7 +343,7 @@ public boolean newDynamicStringField(DocumentParserContext context, String name)
                         SourceFieldMapper.isSynthetic(indexSettings),
                         false
                     ).addMultiField(
-                        new KeywordFieldMapper.Builder("keyword", context.indexSettings().getIndexVersionCreated(), true).ignoreAbove(256)
+                        new KeywordFieldMapper.Builder("keyword", indexSettings.getIndexVersionCreated(), true).ignoreAbove(256)
                     ),
                     context
                 );
@@ -360,7 +360,8 @@ public boolean newDynamicLongField(DocumentParserContext context, String name) t
                     context.indexSettings().getSettings(),
                     context.indexSettings().getIndexVersionCreated(),
                     context.indexSettings().getMode(),
-                    context.indexSettings().sourceKeepMode()
+                    context.indexSettings().sourceKeepMode(),
+                    context.indexSettings().isIndexDisabledByDefault()
                 ),
                 context
             );
@@ -379,7 +380,8 @@ public boolean newDynamicDoubleField(DocumentParserContext context, String name)
                     context.indexSettings().getSettings(),
                     context.indexSettings().getIndexVersionCreated(),
                     context.indexSettings().getMode(),
-                    context.indexSettings().sourceKeepMode()
+                    context.indexSettings().sourceKeepMode(),
+                    context.indexSettings().isIndexDisabledByDefault()
                 ),
                 context
             );
@@ -395,7 +397,8 @@ public boolean newDynamicBooleanField(DocumentParserContext context, String name
                     ScriptCompiler.NONE,
                     ignoreMalformed,
                     context.indexSettings().getIndexVersionCreated(),
-                    context.indexSettings().sourceKeepMode()
+                    context.indexSettings().sourceKeepMode(),
+                    context.indexSettings().isIndexDisabledByDefault()
                 ),
                 context
             );