diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 35289352d7daf..0686b7d7683e2 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -42,6 +42,9 @@ import org.elasticsearch.indices.IndicesRequestCache; import org.elasticsearch.indices.ShardLimitValidator; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -51,180 +54,192 @@ */ public final class IndexScopedSettings extends AbstractScopedSettings { - public static final Set> BUILT_IN_INDEX_SETTINGS = Set.of( - MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY, - MergeSchedulerConfig.AUTO_THROTTLE_SETTING, - MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING, - MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING, - IndexMetadata.SETTING_INDEX_VERSION_CREATED, - IndexMetadata.SETTING_INDEX_VERSION_COMPATIBILITY, - IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_SETTING, - IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING, - IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING, - IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING, - IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING, - IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING, - IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING, - IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING, - IndexMetadata.INDEX_READ_ONLY_SETTING, - IndexMetadata.INDEX_BLOCKS_READ_SETTING, - IndexMetadata.INDEX_BLOCKS_WRITE_SETTING, - IndexMetadata.INDEX_BLOCKS_METADATA_SETTING, - IndexMetadata.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING, - IndexMetadata.INDEX_PRIORITY_SETTING, - IndexMetadata.INDEX_DATA_PATH_SETTING, - IndexMetadata.INDEX_HIDDEN_SETTING, - IndexMetadata.INDEX_FORMAT_SETTING, - IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME, - IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID, - IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_NAME, - IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_UUID, - IndexMetadata.INDEX_DOWNSAMPLE_STATUS, - IndexMetadata.INDEX_DOWNSAMPLE_INTERVAL, - SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING, - SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING, - SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO_SETTING, - SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE_SETTING, - SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING, - SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG_SETTING, - SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO_SETTING, - SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE_SETTING, - SearchSlowLog.INDEX_SEARCH_SLOWLOG_INCLUDE_USER_SETTING, - IndexingSlowLog.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN_SETTING, - IndexingSlowLog.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG_SETTING, - IndexingSlowLog.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO_SETTING, - IndexingSlowLog.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_TRACE_SETTING, - IndexingSlowLog.INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING, - IndexingSlowLog.INDEX_INDEXING_SLOWLOG_MAX_SOURCE_CHARS_TO_LOG_SETTING, - IndexingSlowLog.INDEX_INDEXING_SLOWLOG_INCLUDE_USER_SETTING, - MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, - MergePolicyConfig.INDEX_MERGE_POLICY_TYPE_SETTING, - MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING, - MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING, - MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING, - MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING, - MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING, - MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING, - MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING, - MergePolicyConfig.INDEX_MERGE_POLICY_MERGE_FACTOR_SETTING, - IndexSortConfig.INDEX_SORT_FIELD_SETTING, - IndexSortConfig.INDEX_SORT_ORDER_SETTING, - IndexSortConfig.INDEX_SORT_MISSING_SETTING, - IndexSortConfig.INDEX_SORT_MODE_SETTING, - IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING, - IndexSettings.INDEX_WARMER_ENABLED_SETTING, - IndexSettings.INDEX_REFRESH_INTERVAL_SETTING, - IndexSettings.INDEX_FAST_REFRESH_SETTING, - IndexSettings.MAX_RESULT_WINDOW_SETTING, - IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING, - IndexSettings.MAX_TOKEN_COUNT_SETTING, - IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING, - IndexSettings.MAX_SCRIPT_FIELDS_SETTING, - IndexSettings.MAX_NGRAM_DIFF_SETTING, - IndexSettings.MAX_SHINGLE_DIFF_SETTING, - IndexSettings.MAX_RESCORE_WINDOW_SETTING, - IndexSettings.MAX_ANALYZED_OFFSET_SETTING, - IndexSettings.WEIGHT_MATCHES_MODE_ENABLED_SETTING, - IndexSettings.MAX_TERMS_COUNT_SETTING, - IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING, - IndexSettings.DEFAULT_FIELD_SETTING, - IndexSettings.QUERY_STRING_LENIENT_SETTING, - IndexSettings.ALLOW_UNMAPPED, - IndexSettings.INDEX_CHECK_ON_STARTUP, - IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD, - IndexSettings.MAX_SLICES_PER_SCROLL, - IndexSettings.MAX_REGEX_LENGTH_SETTING, - ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING, - IndexSettings.INDEX_GC_DELETES_SETTING, - IndexSettings.INDEX_SOFT_DELETES_SETTING, - IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING, - IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING, - IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING, - UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, - EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING, - EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING, - IndexSettings.INDEX_FLUSH_AFTER_MERGE_THRESHOLD_SIZE_SETTING, - IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING, - IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_AGE_SETTING, - IndexSettings.INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING, - IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING, - IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING, - IndexSettings.INDEX_SEARCH_IDLE_AFTER, - IndexSettings.INDEX_SEARCH_THROTTLED, - IndexFieldDataService.INDEX_FIELDDATA_CACHE_KEY, - IndexSettings.IGNORE_ABOVE_SETTING, - FieldMapper.IGNORE_MALFORMED_SETTING, - FieldMapper.COERCE_SETTING, - Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING, - MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, - MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING, - MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING, - MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, - MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING, - MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING, - MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, - MapperService.INDEX_MAPPER_DYNAMIC_SETTING, - BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING, - IndexModule.INDEX_STORE_TYPE_SETTING, - IndexModule.INDEX_STORE_PRE_LOAD_SETTING, - IndexModule.INDEX_RECOVERY_TYPE_SETTING, - IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING, - FsDirectoryFactory.INDEX_LOCK_FACTOR_SETTING, - EngineConfig.INDEX_CODEC_SETTING, - IndexMetadata.SETTING_WAIT_FOR_ACTIVE_SHARDS, - IndexSettings.DEFAULT_PIPELINE, - IndexSettings.FINAL_PIPELINE, - MetadataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING, - MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING, - ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_SETTING, - DiskThresholdDecider.SETTING_IGNORE_DISK_WATERMARKS, - ShardLimitValidator.INDEX_SETTING_SHARD_LIMIT_GROUP, - DataTier.TIER_PREFERENCE_SETTING, - IndexSettings.BLOOM_FILTER_ID_FIELD_ENABLED_SETTING, - IndexSettings.LIFECYCLE_ORIGINATION_DATE_SETTING, - IndexSettings.LIFECYCLE_PARSE_ORIGINATION_DATE_SETTING, - IndexSettings.TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING, - IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS, - IndexSettings.LOGSDB_SORT_ON_HOST_NAME, - IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD, - IndexSettings.PREFER_ILM_SETTING, - DataStreamFailureStoreDefinition.FAILURE_STORE_DEFINITION_VERSION_SETTING, - FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING, - IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING, - IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, - IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING, - IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING, - InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT, + public static final Set> BUILT_IN_INDEX_SETTINGS; - // validate that built-in similarities don't get redefined - Setting.groupSetting("index.similarity.", (s) -> { - Map groups = s.getAsGroups(); - for (String key : SimilarityService.BUILT_IN.keySet()) { - if (groups.containsKey(key)) { - throw new IllegalArgumentException( - "illegal value for [index.similarity." + key + "] cannot redefine built-in similarity" - ); - } - } - }, Property.IndexScope, Property.ServerlessPublic), // this allows similarity settings to be passed - Setting.groupSetting("index.analysis.", Property.IndexScope, Property.ServerlessPublic), // this allows analysis settings to be - // passed + static { + Set> settings = new HashSet<>( + Arrays.asList( + MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY, + MergeSchedulerConfig.AUTO_THROTTLE_SETTING, + MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING, + MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING, + IndexMetadata.SETTING_INDEX_VERSION_CREATED, + IndexMetadata.SETTING_INDEX_VERSION_COMPATIBILITY, + IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_SETTING, + IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING, + IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING, + IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING, + IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING, + IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING, + IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING, + IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING, + IndexMetadata.INDEX_READ_ONLY_SETTING, + IndexMetadata.INDEX_BLOCKS_READ_SETTING, + IndexMetadata.INDEX_BLOCKS_WRITE_SETTING, + IndexMetadata.INDEX_BLOCKS_METADATA_SETTING, + IndexMetadata.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING, + IndexMetadata.INDEX_PRIORITY_SETTING, + IndexMetadata.INDEX_DATA_PATH_SETTING, + IndexMetadata.INDEX_HIDDEN_SETTING, + IndexMetadata.INDEX_FORMAT_SETTING, + IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME, + IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID, + IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_NAME, + IndexMetadata.INDEX_DOWNSAMPLE_ORIGIN_UUID, + IndexMetadata.INDEX_DOWNSAMPLE_STATUS, + IndexMetadata.INDEX_DOWNSAMPLE_INTERVAL, + SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING, + SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING, + SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO_SETTING, + SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE_SETTING, + SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING, + SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG_SETTING, + SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO_SETTING, + SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE_SETTING, + SearchSlowLog.INDEX_SEARCH_SLOWLOG_INCLUDE_USER_SETTING, + IndexingSlowLog.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN_SETTING, + IndexingSlowLog.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG_SETTING, + IndexingSlowLog.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO_SETTING, + IndexingSlowLog.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_TRACE_SETTING, + IndexingSlowLog.INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING, + IndexingSlowLog.INDEX_INDEXING_SLOWLOG_MAX_SOURCE_CHARS_TO_LOG_SETTING, + IndexingSlowLog.INDEX_INDEXING_SLOWLOG_INCLUDE_USER_SETTING, + MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, + MergePolicyConfig.INDEX_MERGE_POLICY_TYPE_SETTING, + MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING, + MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING, + MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING, + MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING, + MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING, + MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING, + MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING, + MergePolicyConfig.INDEX_MERGE_POLICY_MERGE_FACTOR_SETTING, + IndexSortConfig.INDEX_SORT_FIELD_SETTING, + IndexSortConfig.INDEX_SORT_ORDER_SETTING, + IndexSortConfig.INDEX_SORT_MISSING_SETTING, + IndexSortConfig.INDEX_SORT_MODE_SETTING, + IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING, + IndexSettings.INDEX_WARMER_ENABLED_SETTING, + IndexSettings.INDEX_REFRESH_INTERVAL_SETTING, + IndexSettings.INDEX_FAST_REFRESH_SETTING, + IndexSettings.MAX_RESULT_WINDOW_SETTING, + IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING, + IndexSettings.MAX_TOKEN_COUNT_SETTING, + IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING, + IndexSettings.MAX_SCRIPT_FIELDS_SETTING, + IndexSettings.MAX_NGRAM_DIFF_SETTING, + IndexSettings.MAX_SHINGLE_DIFF_SETTING, + IndexSettings.MAX_RESCORE_WINDOW_SETTING, + IndexSettings.MAX_ANALYZED_OFFSET_SETTING, + IndexSettings.WEIGHT_MATCHES_MODE_ENABLED_SETTING, + IndexSettings.MAX_TERMS_COUNT_SETTING, + IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING, + IndexSettings.DEFAULT_FIELD_SETTING, + IndexSettings.QUERY_STRING_LENIENT_SETTING, + IndexSettings.ALLOW_UNMAPPED, + IndexSettings.INDEX_CHECK_ON_STARTUP, + IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD, + IndexSettings.MAX_SLICES_PER_SCROLL, + IndexSettings.MAX_REGEX_LENGTH_SETTING, + ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING, + IndexSettings.INDEX_GC_DELETES_SETTING, + IndexSettings.INDEX_SOFT_DELETES_SETTING, + IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING, + IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING, + IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING, + UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, + EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING, + EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING, + IndexSettings.INDEX_FLUSH_AFTER_MERGE_THRESHOLD_SIZE_SETTING, + IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING, + IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_AGE_SETTING, + IndexSettings.INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING, + IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING, + IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING, + IndexSettings.INDEX_SEARCH_IDLE_AFTER, + IndexSettings.INDEX_SEARCH_THROTTLED, + IndexFieldDataService.INDEX_FIELDDATA_CACHE_KEY, + IndexSettings.IGNORE_ABOVE_SETTING, + FieldMapper.IGNORE_MALFORMED_SETTING, + FieldMapper.COERCE_SETTING, + Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING, + MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, + MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING, + MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING, + MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, + MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING, + MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING, + MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, + MapperService.INDEX_MAPPER_DYNAMIC_SETTING, + BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING, + IndexModule.INDEX_STORE_TYPE_SETTING, + IndexModule.INDEX_STORE_PRE_LOAD_SETTING, + IndexModule.INDEX_RECOVERY_TYPE_SETTING, + IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING, + FsDirectoryFactory.INDEX_LOCK_FACTOR_SETTING, + EngineConfig.INDEX_CODEC_SETTING, + IndexMetadata.SETTING_WAIT_FOR_ACTIVE_SHARDS, + IndexSettings.DEFAULT_PIPELINE, + IndexSettings.FINAL_PIPELINE, + MetadataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING, + MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING, + ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_SETTING, + DiskThresholdDecider.SETTING_IGNORE_DISK_WATERMARKS, + ShardLimitValidator.INDEX_SETTING_SHARD_LIMIT_GROUP, + DataTier.TIER_PREFERENCE_SETTING, + IndexSettings.BLOOM_FILTER_ID_FIELD_ENABLED_SETTING, + IndexSettings.LIFECYCLE_ORIGINATION_DATE_SETTING, + IndexSettings.LIFECYCLE_PARSE_ORIGINATION_DATE_SETTING, + IndexSettings.TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING, + IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS, + IndexSettings.LOGSDB_SORT_ON_HOST_NAME, + IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD, + IndexSettings.PREFER_ILM_SETTING, + DataStreamFailureStoreDefinition.FAILURE_STORE_DEFINITION_VERSION_SETTING, + FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING, + IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING, + IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, + IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING, + IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING, + InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT, - // TSDB index settings - IndexSettings.MODE, - IndexMetadata.INDEX_ROUTING_PATH, - IndexSettings.TIME_SERIES_START_TIME, - IndexSettings.TIME_SERIES_END_TIME, + // validate that built-in similarities don't get redefined + Setting.groupSetting("index.similarity.", (s) -> { + Map groups = s.getAsGroups(); + for (String key : SimilarityService.BUILT_IN.keySet()) { + if (groups.containsKey(key)) { + throw new IllegalArgumentException( + "illegal value for [index.similarity." + key + "] cannot redefine built-in similarity" + ); + } + } + }, Property.IndexScope, Property.ServerlessPublic), // this allows similarity settings to be passed + Setting.groupSetting("index.analysis.", Property.IndexScope, Property.ServerlessPublic), // this allows analysis settings to + // be + // passed - // Legacy index settings we must keep around for BWC from 7.x - EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS, - IndexMetadata.INDEX_ROLLUP_SOURCE_NAME, - IndexMetadata.INDEX_ROLLUP_SOURCE_UUID, - IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING, - IndexingSlowLog.INDEX_INDEXING_SLOWLOG_LEVEL_SETTING, - SearchSlowLog.INDEX_SEARCH_SLOWLOG_LEVEL, - Store.FORCE_RAM_TERM_DICT - ); + // TSDB index settings + IndexSettings.MODE, + IndexMetadata.INDEX_ROUTING_PATH, + IndexSettings.TIME_SERIES_START_TIME, + IndexSettings.TIME_SERIES_END_TIME, + + // Legacy index settings we must keep around for BWC from 7.x + EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS, + IndexMetadata.INDEX_ROLLUP_SOURCE_NAME, + IndexMetadata.INDEX_ROLLUP_SOURCE_UUID, + IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING, + IndexingSlowLog.INDEX_INDEXING_SLOWLOG_LEVEL_SETTING, + SearchSlowLog.INDEX_SEARCH_SLOWLOG_LEVEL, + Store.FORCE_RAM_TERM_DICT + ) + ); + + if (IndexSettings.DOC_VALUES_SKIPPER.isEnabled()) { + settings.add(IndexSettings.USE_DOC_VALUES_SKIPPER); + } + BUILT_IN_INDEX_SETTINGS = Collections.unmodifiableSet(settings); + }; public static final IndexScopedSettings DEFAULT_SCOPED_SETTINGS = new IndexScopedSettings(Settings.EMPTY, BUILT_IN_INDEX_SETTINGS); diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 525f90accdf34..4c7bda1c52f17 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -685,6 +685,14 @@ public boolean isES87TSDBCodecEnabled() { Property.Final ); + public static final FeatureFlag DOC_VALUES_SKIPPER = new FeatureFlag("doc_values_skipper"); + public static final Setting USE_DOC_VALUES_SKIPPER = Setting.boolSetting( + "index.mapping.use_doc_values_skipper", + IndexSettings.DOC_VALUES_SKIPPER.isEnabled(), + Property.IndexScope, + Property.Final + ); + /** * The {@link IndexMode "mode"} of the index. */ @@ -922,6 +930,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private final SourceFieldMapper.Mode indexMappingSourceMode; private final boolean recoverySourceEnabled; private final boolean recoverySourceSyntheticEnabled; + private final boolean useDocValuesSkipper; /** * The maximum number of refresh listeners allows on this shard. @@ -1103,6 +1112,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti recoverySourceEnabled = RecoverySettings.INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(nodeSettings); recoverySourceSyntheticEnabled = DiscoveryNode.isStateless(nodeSettings) == false && scopedSettings.get(RECOVERY_USE_SYNTHETIC_SOURCE_SETTING); + useDocValuesSkipper = DOC_VALUES_SKIPPER.isEnabled() && scopedSettings.get(USE_DOC_VALUES_SKIPPER); if (recoverySourceSyntheticEnabled) { if (DiscoveryNode.isStateless(settings)) { throw new IllegalArgumentException("synthetic recovery source is only allowed in stateful"); @@ -1822,6 +1832,10 @@ public boolean isRecoverySourceSyntheticEnabled() { return recoverySourceSyntheticEnabled; } + public boolean useDocValuesSkipper() { + return useDocValuesSkipper; + } + /** * The bounds for {@code @timestamp} on this index or * {@code null} if there are no bounds. diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java index 94d1cc5182457..28de14ec04221 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -302,6 +302,15 @@ private static void validateIndexSortField(SortField sortField) { } } + public boolean hasSortOnField(final String fieldName) { + for (FieldSortSpec sortSpec : sortSpecs) { + if (sortSpec.field.equals(fieldName)) { + return true; + } + } + return false; + } + public static class FieldSortSpec { final String field; SortOrder order; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 2dfbf871ea183..11db4d3d5aa64 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.util.CollectionUtils; -import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexMode; @@ -63,8 +62,6 @@ public abstract class FieldMapper extends Mapper { private static final Logger logger = LogManager.getLogger(FieldMapper.class); - - public static final FeatureFlag DOC_VALUES_SPARSE_INDEX = new FeatureFlag("doc_values_sparse_index"); public static final Setting IGNORE_MALFORMED_SETTING = Setting.boolSetting("index.mapping.ignore_malformed", settings -> { if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.ENABLE_IGNORE_MALFORMED_LOGSDB)) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 5cd968dff52a4..df308a4bf983c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -84,6 +84,7 @@ import static org.apache.lucene.index.IndexWriter.MAX_TERM_LENGTH; import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; +import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SKIPPER; /** * A field mapper for keywords. This mapper accepts strings and indexes them as-is. @@ -201,6 +202,7 @@ public static final class Builder extends FieldMapper.DimensionBuilder { private final IndexAnalyzers indexAnalyzers; private final ScriptCompiler scriptCompiler; private final IndexVersion indexCreatedVersion; + private final boolean useDocValuesSkipper; public Builder(final String name, final MappingParserContext mappingParserContext) { this( @@ -210,7 +212,8 @@ public Builder(final String name, final MappingParserContext mappingParserContex IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()), mappingParserContext.getIndexSettings().getIndexVersionCreated(), mappingParserContext.getIndexSettings().getMode(), - mappingParserContext.getIndexSettings().getIndexSortConfig() + mappingParserContext.getIndexSettings().getIndexSortConfig(), + USE_DOC_VALUES_SKIPPER.get(mappingParserContext.getSettings()) ); } @@ -221,7 +224,7 @@ public Builder(final String name, final MappingParserContext mappingParserContex int ignoreAboveDefault, IndexVersion indexCreatedVersion ) { - this(name, indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, IndexMode.STANDARD, null); + this(name, indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, IndexMode.STANDARD, null, false); } private Builder( @@ -231,7 +234,8 @@ private Builder( int ignoreAboveDefault, IndexVersion indexCreatedVersion, IndexMode indexMode, - IndexSortConfig indexSortConfig + IndexSortConfig indexSortConfig, + boolean useDocValuesSkipper ) { super(name); this.indexAnalyzers = indexAnalyzers; @@ -268,6 +272,7 @@ private Builder( }); this.indexSortConfig = indexSortConfig; this.indexMode = indexMode; + this.useDocValuesSkipper = useDocValuesSkipper; } public Builder(String name, IndexVersion indexCreatedVersion) { @@ -394,7 +399,13 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType @Override public KeywordFieldMapper build(MapperBuilderContext context) { - FieldType fieldtype = resolveFieldType(indexCreatedVersion, indexSortConfig, indexMode, context.buildFullName(leafName())); + FieldType fieldtype = resolveFieldType( + useDocValuesSkipper, + indexCreatedVersion, + indexSortConfig, + indexMode, + context.buildFullName(leafName()) + ); fieldtype.setOmitNorms(this.hasNorms.getValue() == false); fieldtype.setStored(this.stored.getValue()); fieldtype.setDocValuesType(this.hasDocValues.getValue() ? DocValuesType.SORTED_SET : DocValuesType.NONE); @@ -417,65 +428,40 @@ public KeywordFieldMapper build(MapperBuilderContext context) { buildFieldType(context, fieldtype), builderParams(this, context), context.isSourceSynthetic(), + useDocValuesSkipper, this ); } private FieldType resolveFieldType( + final boolean useDocValuesSkipper, final IndexVersion indexCreatedVersion, final IndexSortConfig indexSortConfig, final IndexMode indexMode, final String fullFieldName ) { - if (FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled() + if (useDocValuesSkipper && indexCreatedVersion.onOrAfter(IndexVersions.HOSTNAME_DOC_VALUES_SPARSE_INDEX) - && shouldUseDocValuesSparseIndex(indexSortConfig, indexMode, fullFieldName)) { + && shouldUseDocValuesSkipper(hasDocValues.getValue(), indexSortConfig, indexMode, fullFieldName)) { return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES); } return new FieldType(Defaults.FIELD_TYPE); } - /** - * Determines whether to use a sparse index representation for doc values. - * - *

If the field is explicitly indexed by setting {@code index: true}, we do not use - * a sparse doc values index but instead rely on the inverted index, as is typically - * the case for keyword fields.

- * - *

This method checks several conditions to decide if the sparse index format - * should be applied:

- * - *
    - *
  • Returns {@code false} immediately if the field is explicitly indexed.
    • - *
  • Ensures the field is not explicitly configured as indexed (i.e., {@code index} has its default value).
    • - *
  • Requires doc values to be enabled.
    • - *
  • Index mode must be {@link IndexMode#LOGSDB}.
    • - *
  • Field name must be {@code host.name}.
    • - *
  • The {@code host.name} field must be a primary sort field.
    • - *
- * - *

Returns {@code true} if all conditions are met, indicating that sparse doc values - * should be used. Otherwise, returns {@code false}.

- * - * @param indexSortConfig The index sort configuration, used to check primary sorting. - * @param indexMode The mode of the index, which must be {@link IndexMode#LOGSDB}. - * @param fullFieldName The name of the field being checked, which must be {@code host.name}. - * @return {@code true} if sparse doc values should be used, otherwise {@code false}. - */ - - private boolean shouldUseDocValuesSparseIndex( + private static boolean shouldUseDocValuesSkipper( + final boolean hasDocValues, final IndexSortConfig indexSortConfig, final IndexMode indexMode, final String fullFieldName ) { - if (indexed.isSet() && indexed.getValue()) { - return false; - } - return indexed.isConfigured() == false - && hasDocValues.getValue() + return hasDocValues && IndexMode.LOGSDB.equals(indexMode) && HOST_NAME.equals(fullFieldName) - && (indexSortConfig != null && indexSortConfig.hasPrimarySortOnField(HOST_NAME)); + && indexSortConfigByHostName(indexSortConfig); + } + + private static boolean indexSortConfigByHostName(final IndexSortConfig indexSortConfig) { + return indexSortConfig != null && indexSortConfig.hasIndexSort() && indexSortConfig.hasSortOnField(HOST_NAME); } } @@ -492,7 +478,7 @@ public static final class KeywordFieldType extends StringFieldType { private final boolean isSyntheticSource; private final IndexMode indexMode; private final IndexSortConfig indexSortConfig; - private final boolean hasDocValuesSparseIndex; + private final boolean hasDocValuesSkipper; public KeywordFieldType( String name, @@ -520,7 +506,7 @@ public KeywordFieldType( this.isSyntheticSource = isSyntheticSource; this.indexMode = builder.indexMode; this.indexSortConfig = builder.indexSortConfig; - this.hasDocValuesSparseIndex = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; + this.hasDocValuesSkipper = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; } public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map meta) { @@ -534,7 +520,7 @@ public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Ma this.isSyntheticSource = false; this.indexMode = IndexMode.STANDARD; this.indexSortConfig = null; - this.hasDocValuesSparseIndex = false; + this.hasDocValuesSkipper = false; } public KeywordFieldType(String name) { @@ -559,7 +545,7 @@ public KeywordFieldType(String name, FieldType fieldType) { this.isSyntheticSource = false; this.indexMode = IndexMode.STANDARD; this.indexSortConfig = null; - this.hasDocValuesSparseIndex = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; + this.hasDocValuesSkipper = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; } public KeywordFieldType(String name, NamedAnalyzer analyzer) { @@ -573,7 +559,7 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) { this.isSyntheticSource = false; this.indexMode = IndexMode.STANDARD; this.indexSortConfig = null; - this.hasDocValuesSparseIndex = false; + this.hasDocValuesSkipper = false; } @Override @@ -1021,8 +1007,8 @@ public IndexSortConfig getIndexSortConfig() { return indexSortConfig; } - public boolean hasDocValuesSparseIndex() { - return hasDocValuesSparseIndex; + public boolean hasDocValuesSkipper() { + return hasDocValuesSkipper; } } @@ -1041,6 +1027,7 @@ public boolean hasDocValuesSparseIndex() { private final int ignoreAboveDefault; private final IndexMode indexMode; private final IndexSortConfig indexSortConfig; + private final boolean useDocValuesSkipper; private KeywordFieldMapper( String simpleName, @@ -1048,6 +1035,7 @@ private KeywordFieldMapper( KeywordFieldType mappedFieldType, BuilderParams builderParams, boolean isSyntheticSource, + boolean useDocValuesSkipper, Builder builder ) { super(simpleName, mappedFieldType, builderParams); @@ -1066,6 +1054,7 @@ private KeywordFieldMapper( this.ignoreAboveDefault = builder.ignoreAboveDefault; this.indexMode = builder.indexMode; this.indexSortConfig = builder.indexSortConfig; + this.useDocValuesSkipper = useDocValuesSkipper; } @Override @@ -1183,9 +1172,16 @@ public Map indexAnalyzers() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, indexMode, indexSortConfig) - .dimension(fieldType().isDimension()) - .init(this); + return new Builder( + leafName(), + indexAnalyzers, + scriptCompiler, + ignoreAboveDefault, + indexCreatedVersion, + indexMode, + indexSortConfig, + useDocValuesSkipper + ).dimension(fieldType().isDimension()).init(this); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 924c32f527f7a..198988832cb55 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -772,13 +772,12 @@ public void testDocValuesLoadedFromStoredSynthetic() throws IOException { assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); } - public void testFieldTypeWithSkipDocValues_LogsDbMode() throws IOException { - assumeTrue("Needs feature flag to be enabled", FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled()); - + public void testFieldTypeWithSkipDocValues_LogsDbModeDisabledSetting() throws IOException { final MapperService mapperService = createMapperService( Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), false) .build(), mapping(b -> { b.startObject("host.name"); @@ -789,157 +788,187 @@ public void testFieldTypeWithSkipDocValues_LogsDbMode() throws IOException { final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertTrue(mapper.fieldType().hasDocValues()); - assertFalse(mapper.fieldType().isIndexed()); - assertTrue(mapper.fieldType().hasDocValuesSparseIndex()); + assertTrue(mapper.fieldType().isIndexed()); + assertFalse(mapper.fieldType().hasDocValuesSkipper()); } - public void testFieldTypeDefault_StandardMode() throws IOException { - assumeTrue("Needs feature flag to be enabled", FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled()); + public void testFieldTypeWithSkipDocValues_LogsDbMode() throws IOException { + final Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .build(); + final MapperService mapperService = createMapperService(settings, mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.endObject(); + })); - final MapperService mapperService = createMapperService( - Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) - .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") - .build(), - mapping(b -> { - b.startObject("host.name"); - b.field("type", "keyword"); - b.endObject(); - }) - ); + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); + assertTrue(mapper.fieldType().hasDocValues()); + assertFalse(IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings) && mapper.fieldType().isIndexed()); + if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) { + assertTrue(mapper.fieldType().hasDocValuesSkipper()); + } else { + assertFalse(mapper.fieldType().hasDocValuesSkipper()); + } + } + + public void testFieldTypeDefault_StandardMode() throws IOException { + final Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .build(); + final MapperService mapperService = createMapperService(settings, mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.endObject(); + })); final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertTrue(mapper.fieldType().hasDocValues()); assertTrue(mapper.fieldType().isIndexed()); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + assertFalse(mapper.fieldType().hasDocValuesSkipper()); } public void testFieldTypeDefault_NonMatchingFieldName() throws IOException { - assumeTrue("Needs feature flag to be enabled", FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled()); - - final MapperService mapperService = createMapperService( - Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "hostname") - .build(), - mapping(b -> { - b.startObject("hostname"); - b.field("type", "keyword"); - b.endObject(); - }) - ); + final Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "hostname") + .build(); + final MapperService mapperService = createMapperService(settings, mapping(b -> { + b.startObject("hostname"); + b.field("type", "keyword"); + b.endObject(); + })); final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("hostname"); assertTrue(mapper.fieldType().hasDocValues()); assertTrue(mapper.fieldType().isIndexed()); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + assertFalse(mapper.fieldType().hasDocValuesSkipper()); } - public void testFieldTypeDefault_ConfiguredIndexed() throws IOException { - assumeTrue("Needs feature flag to be enabled", FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled()); + public void testFieldTypeDefault_ConfiguredIndexedWithSettingOverride() throws IOException { + final Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .build(); + final MapperService mapperService = createMapperService(settings, mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.field("index", true); + b.endObject(); + })); - final MapperService mapperService = createMapperService( - Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") - .build(), - mapping(b -> { - b.startObject("host.name"); - b.field("type", "keyword"); - b.field("index", true); - b.endObject(); - }) - ); + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); + assertTrue(mapper.fieldType().hasDocValues()); + assertFalse(IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings) && mapper.fieldType().isIndexed()); + if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) { + assertTrue(mapper.fieldType().hasDocValuesSkipper()); + } else { + assertFalse(mapper.fieldType().hasDocValuesSkipper()); + } + } + + public void testFieldTypeDefault_ConfiguredIndexedWithoutSettingOverride() throws IOException { + final Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .build(); + final MapperService mapperService = createMapperService(settings, mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.field("index", true); + b.endObject(); + })); final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertTrue(mapper.fieldType().hasDocValues()); - assertTrue(mapper.fieldType().isIndexed()); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + assertFalse(IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings) && mapper.fieldType().isIndexed()); + if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) { + assertTrue(mapper.fieldType().hasDocValuesSkipper()); + } else { + assertFalse(mapper.fieldType().hasDocValuesSkipper()); + } } public void testFieldTypeDefault_ConfiguredDocValues() throws IOException { - assumeTrue("Needs feature flag to be enabled", FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled()); - - final MapperService mapperService = createMapperService( - Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") - .build(), - mapping(b -> { - b.startObject("host.name"); - b.field("type", "keyword"); - b.field("doc_values", true); - b.endObject(); - }) - ); + final Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .build(); + final MapperService mapperService = createMapperService(settings, mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.field("doc_values", true); + b.endObject(); + })); final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertTrue(mapper.fieldType().hasDocValues()); - assertFalse(mapper.fieldType().isIndexed()); - assertTrue(mapper.fieldType().hasDocValuesSparseIndex()); + assertFalse(IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings) && mapper.fieldType().isIndexed()); + if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) { + assertTrue(mapper.fieldType().hasDocValuesSkipper()); + } else { + assertFalse(mapper.fieldType().hasDocValuesSkipper()); + } } public void testFieldTypeDefault_LogsDbMode_NonSortField() throws IOException { - assumeTrue("Needs feature flag to be enabled", FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled()); - - final MapperService mapperService = createMapperService( - Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()).build(), - mapping(b -> { - b.startObject("host.name"); - b.field("type", "keyword"); - b.endObject(); - }) - ); + final Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), true) + .build(); + final MapperService mapperService = createMapperService(settings, mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.endObject(); + })); final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertTrue(mapper.fieldType().hasDocValues()); assertTrue(mapper.fieldType().isIndexed()); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + assertFalse(mapper.fieldType().hasDocValuesSkipper()); } public void testFieldTypeWithSkipDocValues_IndexedFalseDocValuesTrue() throws IOException { - assumeTrue("Needs feature flag to be enabled", FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled()); - - final MapperService mapperService = createMapperService( - Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") - .build(), - mapping(b -> { - b.startObject("host.name"); - b.field("type", "keyword"); - b.field("index", false); - b.field("doc_values", true); - b.endObject(); - }) - ); + final Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .build(); + final MapperService mapperService = createMapperService(settings, mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.field("index", false); + b.field("doc_values", true); + b.endObject(); + })); final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertTrue(mapper.fieldType().hasDocValues()); - assertFalse(mapper.fieldType().isIndexed()); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + assertFalse(IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings) && mapper.fieldType().isIndexed()); + if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) { + assertTrue(mapper.fieldType().hasDocValuesSkipper()); + } else { + assertFalse(mapper.fieldType().hasDocValuesSkipper()); + } } public void testFieldTypeDefault_IndexedFalseDocValuesFalse() throws IOException { - assumeTrue("Needs feature flag to be enabled", FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled()); - - final MapperService mapperService = createMapperService( - Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") - .build(), - mapping(b -> { - b.startObject("host.name"); - b.field("type", "keyword"); - b.field("index", false); - b.field("doc_values", false); - b.endObject(); - }) - ); + final Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .build(); + final MapperService mapperService = createMapperService(settings, mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.field("index", false); + b.field("doc_values", false); + b.endObject(); + })); final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertFalse(mapper.fieldType().hasDocValues()); assertFalse(mapper.fieldType().isIndexed()); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + assertFalse(mapper.fieldType().hasDocValuesSkipper()); } }