Refactoring doc values sparse index enabling for the host.name field

server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -42,6 +42,7 @@
 import org.elasticsearch.indices.IndicesRequestCache;
 import org.elasticsearch.indices.ShardLimitValidator;
 
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
@@ -226,7 +227,15 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         Store.FORCE_RAM_TERM_DICT
     );
 
-    public static final IndexScopedSettings DEFAULT_SCOPED_SETTINGS = new IndexScopedSettings(Settings.EMPTY, BUILT_IN_INDEX_SETTINGS);
+    public static final IndexScopedSettings DEFAULT_SCOPED_SETTINGS = IndexSettings.DOC_VALUES_SPARSE_INDEX.isEnabled()
+        ? includeExtraSettings()
+        : new IndexScopedSettings(Settings.EMPTY, BUILT_IN_INDEX_SETTINGS);
+
+    private static IndexScopedSettings includeExtraSettings() {
+        final Set<Setting<?>> settings = new HashSet<>(BUILT_IN_INDEX_SETTINGS);
+        settings.add(IndexSettings.USE_DOC_VALUES_SPARSE_INDEX);
+        return new IndexScopedSettings(Settings.EMPTY, settings);
+    }
 
     public IndexScopedSettings(Settings settings, Set<Setting<?>> settingsSet) {
         super(settings, settingsSet, Property.IndexScope);

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.index.mapper.Mapper;
@@ -683,6 +684,14 @@ public boolean isES87TSDBCodecEnabled() {
         Property.Final
     );
 
+    public static final FeatureFlag DOC_VALUES_SPARSE_INDEX = new FeatureFlag("doc_values_sparse_index");
+    public static final Setting<Boolean> USE_DOC_VALUES_SPARSE_INDEX = Setting.boolSetting(
+        "index.mapping.use_doc_values_sparse_index",
+        false,
+        Property.IndexScope,
+        Property.Final
+    );
+
     /**
      * The {@link IndexMode "mode"} of the index.
      */
@@ -904,6 +913,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
     private final SourceFieldMapper.Mode indexMappingSourceMode;
     private final boolean recoverySourceEnabled;
     private final boolean recoverySourceSyntheticEnabled;
+    private final boolean useDocValuesSparseIndex;
 
     /**
      * The maximum number of refresh listeners allows on this shard.
@@ -1084,6 +1094,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         indexMappingSourceMode = scopedSettings.get(INDEX_MAPPER_SOURCE_MODE_SETTING);
         recoverySourceEnabled = RecoverySettings.INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(nodeSettings);
         recoverySourceSyntheticEnabled = scopedSettings.get(RECOVERY_USE_SYNTHETIC_SOURCE_SETTING);
+        useDocValuesSparseIndex = DOC_VALUES_SPARSE_INDEX.isEnabled() && scopedSettings.get(USE_DOC_VALUES_SPARSE_INDEX);
         if (recoverySourceSyntheticEnabled) {
             if (DiscoveryNode.isStateless(settings)) {
                 throw new IllegalArgumentException("synthetic recovery source is only allowed in stateful");
@@ -1803,6 +1814,10 @@ public boolean isRecoverySourceSyntheticEnabled() {
         return recoverySourceSyntheticEnabled;
     }
 
+    public boolean useDocValuesSparseIndex() {
+        return useDocValuesSparseIndex;
+    }
+
     /**
      * The bounds for {@code @timestamp} on this index or
      * {@code null} if there are no bounds.

server/src/main/java/org/elasticsearch/index/IndexSortConfig.java

@@ -302,6 +302,10 @@ private static void validateIndexSortField(SortField sortField) {
         }
     }
 
+    public FieldSortSpec[] getFieldSortSpecs() {
+        return sortSpecs;
+    }
+
     public static class FieldSortSpec {
         final String field;
         SortOrder order;

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -21,7 +21,6 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.util.CollectionUtils;
-import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.IndexMode;
@@ -63,8 +62,6 @@
 
 public abstract class FieldMapper extends Mapper {
     private static final Logger logger = LogManager.getLogger(FieldMapper.class);
-
-    public static final FeatureFlag DOC_VALUES_SPARSE_INDEX = new FeatureFlag("doc_values_sparse_index");
     public static final Setting<Boolean> IGNORE_MALFORMED_SETTING = Setting.boolSetting("index.mapping.ignore_malformed", settings -> {
         if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB
             && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.ENABLE_IGNORE_MALFORMED_LOGSDB)) {

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -83,7 +83,9 @@
 
 import static org.apache.lucene.index.IndexWriter.MAX_TERM_LENGTH;
 import static org.elasticsearch.core.Strings.format;
+import static org.elasticsearch.index.IndexSettings.DOC_VALUES_SPARSE_INDEX;
 import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING;
+import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SPARSE_INDEX;
 
 /**
  * A field mapper for keywords. This mapper accepts strings and indexes them as-is.
@@ -201,6 +203,7 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
         private final IndexAnalyzers indexAnalyzers;
         private final ScriptCompiler scriptCompiler;
         private final IndexVersion indexCreatedVersion;
+        private final boolean useDocValuesSparseIndex;
 
         public Builder(final String name, final MappingParserContext mappingParserContext) {
             this(
@@ -210,7 +213,8 @@ public Builder(final String name, final MappingParserContext mappingParserContex
                 IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()),
                 mappingParserContext.getIndexSettings().getIndexVersionCreated(),
                 mappingParserContext.getIndexSettings().getMode(),
-                mappingParserContext.getIndexSettings().getIndexSortConfig()
+                mappingParserContext.getIndexSettings().getIndexSortConfig(),
+                DOC_VALUES_SPARSE_INDEX.isEnabled() && USE_DOC_VALUES_SPARSE_INDEX.get(mappingParserContext.getSettings())
             );
         }
 
@@ -221,7 +225,7 @@ public Builder(final String name, final MappingParserContext mappingParserContex
             int ignoreAboveDefault,
             IndexVersion indexCreatedVersion
         ) {
-            this(name, indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, IndexMode.STANDARD, null);
+            this(name, indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, IndexMode.STANDARD, null, false);
         }
 
         private Builder(
@@ -231,7 +235,8 @@ private Builder(
             int ignoreAboveDefault,
             IndexVersion indexCreatedVersion,
             IndexMode indexMode,
-            IndexSortConfig indexSortConfig
+            IndexSortConfig indexSortConfig,
+            boolean useDocValuesSparseIndex
         ) {
             super(name);
             this.indexAnalyzers = indexAnalyzers;
@@ -268,6 +273,7 @@ private Builder(
                 });
             this.indexSortConfig = indexSortConfig;
             this.indexMode = indexMode;
+            this.useDocValuesSparseIndex = useDocValuesSparseIndex;
         }
 
         public Builder(String name, IndexVersion indexCreatedVersion) {
@@ -394,7 +400,13 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType
 
         @Override
         public KeywordFieldMapper build(MapperBuilderContext context) {
-            FieldType fieldtype = resolveFieldType(indexCreatedVersion, indexSortConfig, indexMode, context.buildFullName(leafName()));
+            FieldType fieldtype = resolveFieldType(
+                useDocValuesSparseIndex,
+                indexCreatedVersion,
+                indexSortConfig,
+                indexMode,
+                context.buildFullName(leafName())
+            );
             fieldtype.setOmitNorms(this.hasNorms.getValue() == false);
             fieldtype.setStored(this.stored.getValue());
             fieldtype.setDocValuesType(this.hasDocValues.getValue() ? DocValuesType.SORTED_SET : DocValuesType.NONE);
@@ -417,17 +429,19 @@ public KeywordFieldMapper build(MapperBuilderContext context) {
                 buildFieldType(context, fieldtype),
                 builderParams(this, context),
                 context.isSourceSynthetic(),
+                useDocValuesSparseIndex,
                 this
             );
         }
 
         private FieldType resolveFieldType(
+            final boolean useDocValuesSparseIndex,
             final IndexVersion indexCreatedVersion,
             final IndexSortConfig indexSortConfig,
             final IndexMode indexMode,
             final String fullFieldName
         ) {
-            if (FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled()
+            if (useDocValuesSparseIndex
                 && indexCreatedVersion.onOrAfter(IndexVersions.HOSTNAME_DOC_VALUES_SPARSE_INDEX)
                 && shouldUseDocValuesSparseIndex(indexSortConfig, indexMode, fullFieldName)) {
                 return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES);
@@ -468,14 +482,17 @@ private boolean shouldUseDocValuesSparseIndex(
             final IndexMode indexMode,
             final String fullFieldName
         ) {
-            if (indexed.isSet() && indexed.getValue()) {
-                return false;
-            }
-            return indexed.isConfigured() == false
-                && hasDocValues.getValue()
+            return hasDocValues.getValue()
                 && IndexMode.LOGSDB.equals(indexMode)
                 && HOST_NAME.equals(fullFieldName)
-                && (indexSortConfig != null && indexSortConfig.hasPrimarySortOnField(HOST_NAME));
+                && indexSortConfigByHostName(indexSortConfig);
+        }
+
+        private boolean indexSortConfigByHostName(final IndexSortConfig indexSortConfig) {
+            if (indexSortConfig == null) {
+                return false;
+            }
+            return Arrays.stream(indexSortConfig.getFieldSortSpecs()).anyMatch(fieldSortSpec -> HOST_NAME.equals(fieldSortSpec.getField()));
         }
     }
 
@@ -1041,13 +1058,15 @@ public boolean hasDocValuesSparseIndex() {
     private final int ignoreAboveDefault;
     private final IndexMode indexMode;
     private final IndexSortConfig indexSortConfig;
+    private final boolean useDocValuesSParseIndex;
 
     private KeywordFieldMapper(
         String simpleName,
         FieldType fieldType,
         KeywordFieldType mappedFieldType,
         BuilderParams builderParams,
         boolean isSyntheticSource,
+        boolean useDocValuesSParseIndex,
         Builder builder
     ) {
         super(simpleName, mappedFieldType, builderParams);
@@ -1066,6 +1085,7 @@ private KeywordFieldMapper(
         this.ignoreAboveDefault = builder.ignoreAboveDefault;
         this.indexMode = builder.indexMode;
         this.indexSortConfig = builder.indexSortConfig;
+        this.useDocValuesSParseIndex = useDocValuesSParseIndex;
     }
 
     @Override
@@ -1183,9 +1203,16 @@ public Map<String, NamedAnalyzer> indexAnalyzers() {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, indexMode, indexSortConfig)
-            .dimension(fieldType().isDimension())
-            .init(this);
+        return new Builder(
+            leafName(),
+            indexAnalyzers,
+            scriptCompiler,
+            ignoreAboveDefault,
+            indexCreatedVersion,
+            indexMode,
+            indexSortConfig,
+            useDocValuesSParseIndex
+        ).dimension(fieldType().isDimension()).init(this);
     }
 
     @Override