diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java index a265e69a0a8d6..f4031e5d0cb45 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java @@ -119,16 +119,6 @@ public void testValidateInvalidFieldType() { ); } - public void testValidateNotIndexed() { - Exception e = expectThrows(IllegalArgumentException.class, () -> createMapperService(timestampMapping(true, b -> { - b.startObject("@timestamp"); - b.field("type", "date"); - b.field("index", false); - b.endObject(); - }))); - assertThat(e.getMessage(), equalTo("data stream timestamp field [@timestamp] is not indexed")); - } - public void testValidateNotDocValues() { Exception e = expectThrows(IllegalArgumentException.class, () -> createMapperService(timestampMapping(true, b -> { b.startObject("@timestamp"); diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java index 9cd574063b06d..97f668d37e22d 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java @@ -671,4 +671,9 @@ protected IngestScriptSupport ingestScriptSupport() { protected List getSortShortcutSupport() { return List.of(); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index d377b5e3abbc7..bcbb9c6ff60c5 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -193,12 +193,23 @@ protected Parameter[] getParameters() { return new Parameter[] { indexed, hasDocValues, stored, ignoreMalformed, meta, scalingFactor, coerce, nullValue, metric }; } + private IndexType indexType() { + if (indexed.getValue()) { + return IndexType.points(true, hasDocValues.getValue()); + } + if (hasDocValues.getValue() + && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.STANDARD_INDEXES_USE_SKIPPERS) + && indexSettings.useDocValuesSkipper()) { + return IndexType.skippers(); + } + return IndexType.points(false, hasDocValues.getValue()); + } + @Override public ScaledFloatFieldMapper build(MapperBuilderContext context) { - IndexType indexType = IndexType.points(indexed.get(), hasDocValues.get()); ScaledFloatFieldType type = new ScaledFloatFieldType( context.buildFullName(leafName()), - indexType, + indexType(), stored.getValue(), meta.getValue(), scalingFactor.getValue(), diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java index 3a567490fc218..ab721cee7c385 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java @@ -390,4 +390,9 @@ public void testLoadSyntheticSourceFromStringOrBytesRef() throws IOException { protected List getSortShortcutSupport() { return List.of(); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java index 174bc44cbaecc..5fb245839f7a8 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java @@ -220,4 +220,9 @@ protected IngestScriptSupport ingestScriptSupport() { protected List getSortShortcutSupport() { return List.of(); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java index c229752917560..182976616dfad 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java @@ -214,4 +214,9 @@ protected IngestScriptSupport ingestScriptSupport() { protected List getSortShortcutSupport() { return List.of(); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java index 9965259e294da..8caecd0a6f4cf 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java @@ -882,4 +882,9 @@ protected IngestScriptSupport ingestScriptSupport() { protected List getSortShortcutSupport() { return List.of(); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java index 90dd1a7081ee2..d062ddcd21803 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java @@ -269,4 +269,9 @@ public void testAggregationsDocValuesDisabled() throws IOException { protected List getSortShortcutSupport() { return List.of(new SortShortcutSupport(this::minimalMapping, this::writeField, true)); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java index d94d12eb9226e..05ef79b7924d0 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java @@ -323,4 +323,9 @@ protected IngestScriptSupport ingestScriptSupport() { protected List getSortShortcutSupport() { return List.of(new SortShortcutSupport(this::minimalMapping, this::writeField, true)); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index b808cffd75c7a..5bb70fd80e2de 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -709,4 +709,9 @@ protected IngestScriptSupport ingestScriptSupport() { protected List getSortShortcutSupport() { return List.of(); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java index a4ea946636e58..81590dce5ce4f 100644 --- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java @@ -153,4 +153,9 @@ protected IngestScriptSupport ingestScriptSupport() { protected List getSortShortcutSupport() { return List.of(new SortShortcutSupport(this::minimalMapping, this::writeField, false)); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 6db422dc503c4..f0c124cdc2c90 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -110,6 +110,7 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> task.skipTest("logsdb/10_settings/override sort order settings", "we changed the error message") task.skipTest("tsdb/10_settings/set start_time and end_time without timeseries mode", "we don't validate for index_mode=tsdb when setting start_date/end_date anymore") task.skipTest("tsdb/10_settings/set start_time, end_time and routing_path via put settings api without time_series mode", "we don't validate for index_mode=tsdb when setting start_date/end_date anymore") + task.skipTest("search/140_pre_filter_search_shards/prefilter on non-indexed date fields", "prefiltering can now use skippers on dv-only fields") // Expected deprecation warning to compat yaml tests: task.addAllowedWarningRegex("Use of the \\[max_size\\] rollover condition has been deprecated in favour of the \\[max_primary_shard_size\\] condition and will be removed in a later version") }) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml index 0ae00dff6ce63..f105f6cb7e986 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml @@ -240,8 +240,8 @@ setup: --- "prefilter on non-indexed date fields": - requires: - cluster_features: "gte_v8.1.0" - reason: "doc values search was added in 8.1.0" + cluster_features: [ "gte_v8.1.0", "mapper.skippers_on_unindexed_fields" ] + reason: "doc values search was added in 8.1.0; skippers allow pre-filtering" - do: index: @@ -274,7 +274,7 @@ setup: - match: { _shards.failed: 0 } - match: { hits.total: 2 } - # this is a case where we would normally skip due to rewrite but we can't because we only have doc values + # this is a case where we can use docvalues skippers to pre-filter index_1 - do: search: rest_total_hits_as_int: true @@ -283,6 +283,6 @@ setup: - match: { _shards.total: 3 } - match: { _shards.successful: 3 } - - match: { _shards.skipped : 0 } + - match: { _shards.skipped : 1 } - match: { _shards.failed: 0 } - match: { hits.total: 2 } diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 134cf076c53bd..2425b11f0e035 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -197,6 +197,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion TIME_SERIES_ALL_FIELDS_USE_SKIPPERS = def(9_046_0_00, Version.LUCENE_10_3_1); public static final IndexVersion UPGRADE_TO_LUCENE_10_3_2 = def(9_047_0_00, Version.LUCENE_10_3_2); public static final IndexVersion SECURITY_MIGRATIONS_METADATA_FLATTENED_UPDATE = def(9_048_0_00, Version.LUCENE_10_3_2); + public static final IndexVersion STANDARD_INDEXES_USE_SKIPPERS = def(9_049_0_00, Version.LUCENE_10_3_2); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 4436b9f977ec9..0483da6671082 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -149,6 +149,10 @@ private IndexType indexType() { if (docValues.get() == false) { return IndexType.NONE; } + if (indexSettings.useDocValuesSkipper() + && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.STANDARD_INDEXES_USE_SKIPPERS)) { + return IndexType.skippers(); + } return useTimeSeriesDocValuesSkippers(indexSettings, dimension.get()) ? IndexType.skippers() : IndexType.docValuesOnly(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 6e97412cafcef..3a0f34d922a9e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -400,10 +400,15 @@ private Long parseNullValue(DateFieldType fieldType) { } private IndexType indexType(String fullFieldName) { - boolean hasDocValuesSkipper = shouldUseDocValuesSkipper(indexSettings, docValues.getValue(), fullFieldName); - if (hasDocValuesSkipper) { + if (shouldUseDocValuesSkipper(indexSettings, docValues.getValue(), fullFieldName)) { return IndexType.skippers(); } + if (index.get() == false && docValues.get()) { + if (indexSettings.useDocValuesSkipper() + && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.STANDARD_INDEXES_USE_SKIPPERS)) { + return IndexType.skippers(); + } + } if (indexCreatedVersion.isLegacyIndexVersion()) { return IndexType.archivedPoints(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 5d9d5d7ce7582..c5a2a3e7e05e1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -182,6 +182,12 @@ private IndexType indexType() { if (useTimeSeriesDocValuesSkippers(indexSettings, dimension.get())) { return IndexType.skippers(); } + if (indexed.get() == false && hasDocValues.get()) { + if (indexSettings.useDocValuesSkipper() + && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.STANDARD_INDEXES_USE_SKIPPERS)) { + return IndexType.skippers(); + } + } return IndexType.points(indexed.get(), hasDocValues.get()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 2b558d12b9e83..710e79aa0d016 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -441,7 +441,10 @@ public KeywordFieldMapper build(MapperBuilderContext context) { private FieldType resolveFieldType(final boolean forceDocValuesSkipper, final String fullFieldName) { FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE); - if (forceDocValuesSkipper || shouldUseHostnameSkipper(fullFieldName) || shouldUseTimeSeriesSkipper()) { + if (forceDocValuesSkipper + || shouldUseHostnameSkipper(fullFieldName) + || shouldUseTimeSeriesSkipper() + || shouldUseStandardSkipper()) { fieldtype = new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES); } fieldtype.setOmitNorms(this.hasNorms.getValue() == false); @@ -474,6 +477,13 @@ private boolean shouldUseHostnameSkipper(final String fullFieldName) { && indexSortConfigByHostName(indexSettings.getIndexSortConfig()); } + private boolean shouldUseStandardSkipper() { + return hasDocValues.get() + && indexed.get() == false + && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.STANDARD_INDEXES_USE_SKIPPERS) + && indexSettings.useDocValuesSkipper(); + } + private static boolean indexSortConfigByHostName(final IndexSortConfig indexSortConfig) { return indexSortConfig != null && indexSortConfig.hasIndexSort() && indexSortConfig.hasSortOnField(HOST_NAME); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 7b96b1b7cc835..a0992a114dce1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -64,6 +64,7 @@ public class MapperFeatures implements FeatureSpecification { public static final NodeFeature GENERIC_VECTOR_FORMAT = new NodeFeature("mapper.vectors.generic_vector_format"); public static final NodeFeature FIX_DENSE_VECTOR_WRONG_FIELDS = new NodeFeature("mapper.fix_dense_vector_wrong_fields"); static final NodeFeature BBQ_DISK_STATS_SUPPORT = new NodeFeature("mapper.bbq_disk_stats_support"); + static final NodeFeature SKIPPERS_ON_UNINDEXED_FIELDS = new NodeFeature("mapper.skippers_on_unindexed_fields"); static final NodeFeature STORED_FIELDS_SPEC_MERGE_BUG = new NodeFeature("mapper.stored_fields_spec_merge_bug"); @Override @@ -110,6 +111,7 @@ public Set getTestFeatures() { BASE64_DENSE_VECTORS, FIX_DENSE_VECTOR_WRONG_FIELDS, BBQ_DISK_STATS_SUPPORT, + SKIPPERS_ON_UNINDEXED_FIELDS, STORED_FIELDS_SPEC_MERGE_BUG ); if (ES93GenericFlatVectorsFormat.GENERIC_VECTOR_FORMAT.isEnabled()) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 57c4a8db113df..b80f463204146 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -223,8 +223,14 @@ private IndexType indexType() { if (indexSettings.getIndexVersionCreated().isLegacyIndexVersion()) { return IndexType.archivedPoints(); } - if (indexed.get() == false && hasDocValues.get() && useTimeSeriesDocValuesSkippers(indexSettings, dimension.get())) { - return IndexType.skippers(); + if (indexed.get() == false && hasDocValues.get()) { + if (useTimeSeriesDocValuesSkippers(indexSettings, dimension.get())) { + return IndexType.skippers(); + } + if (indexSettings.useDocValuesSkipper() + && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.STANDARD_INDEXES_USE_SKIPPERS)) { + return IndexType.skippers(); + } } return IndexType.points(indexed.get(), hasDocValues.get()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java index 8c3800bdf6132..7d5265a891378 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java @@ -268,4 +268,9 @@ protected IngestScriptSupport ingestScriptSupport() { protected List getSortShortcutSupport() { return List.of(); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 079521b26b666..fab561aa804d9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -990,4 +990,9 @@ protected IngestScriptSupport ingestScriptSupport() { protected List getSortShortcutSupport() { return List.of(); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java index c560448f6bb33..67ee14136ae30 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java @@ -724,4 +724,9 @@ protected IngestScriptSupport ingestScriptSupport() { protected List getSortShortcutSupport() { return List.of(); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index 6dd85a3f8c5e6..859aa82087490 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -50,6 +50,11 @@ protected boolean supportsIgnoreMalformed() { return false; } + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } + public void testExistsQueryDocValuesDisabled() throws IOException { MapperService mapperService = createMapperService(fieldMapping(b -> { minimalMapping(b); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 4947af5eeba11..beea07b83079a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1780,4 +1780,9 @@ public void testNormsEnabledWhenIndexModeIsTsdb_bwcCheck() throws IOException { protected List getSortShortcutSupport() { return List.of(); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java index 3939fcaa6e0e8..6eec91ce826b5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java @@ -1021,4 +1021,9 @@ public void assertStoredFieldsEquals(String info, IndexReader leftReader, IndexR protected List getSortShortcutSupport() { return List.of(new SortShortcutSupport(this::minimalMapping, this::writeField, true)); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SyntheticVectorsMapperTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SyntheticVectorsMapperTestCase.java index 35d3afcde8a69..b37070b5a5267 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SyntheticVectorsMapperTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SyntheticVectorsMapperTestCase.java @@ -271,4 +271,9 @@ private void assertSyntheticVectors(String mapping, BytesReference source, XCont protected List getSortShortcutSupport() { return List.of(); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 6e20d84fef85f..ae1df6578e876 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -74,6 +74,7 @@ import org.elasticsearch.search.lookup.Source; import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.lookup.SourceProvider; +import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -1845,4 +1846,48 @@ public final void testSortShortcuts() throws IOException { } assertParseMinimalWarnings(); } + + protected boolean supportsDocValuesSkippers() { + return true; + } + + public void testDocValuesSkippers() throws IOException { + assumeTrue("Mapper does not support doc values skippers", supportsDocValuesSkippers()); + + IndexVersion preSkipperVersion = IndexVersionUtils.randomPreviousCompatibleVersion( + random(), + IndexVersions.STANDARD_INDEXES_USE_SKIPPERS + ); + IndexVersion withSkipperVersion = IndexVersions.STANDARD_INDEXES_USE_SKIPPERS; + + Settings skippersDisabled = Settings.builder().put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), false).build(); + Settings skippersEnabled = Settings.builder().put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), true).build(); + + { + MapperService mapperService = createMapperService(preSkipperVersion, fieldMapping(b -> { + minimalMapping(b); + b.field("doc_values", true); + b.field("index", false); + })); + assertThat(mapperService.fieldType("field").indexType(), equalTo(IndexType.docValuesOnly())); + } + + { + MapperService mapperService = createMapperService(withSkipperVersion, skippersEnabled, fieldMapping(b -> { + minimalMapping(b); + b.field("doc_values", true); + b.field("index", false); + })); + assertThat(mapperService.fieldType("field").indexType(), equalTo(IndexType.skippers())); + } + + { + MapperService mapperService = createMapperService(withSkipperVersion, skippersDisabled, fieldMapping(b -> { + minimalMapping(b); + b.field("doc_values", true); + b.field("index", false); + })); + assertThat(mapperService.fieldType("field").indexType(), equalTo(IndexType.docValuesOnly())); + } + } } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java index b26b33cc01544..973908972f8df 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java @@ -528,4 +528,9 @@ public void testSyntheticSourceKeepArrays() { protected List getSortShortcutSupport() { return List.of(); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestFieldMapperTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestFieldMapperTests.java index 82fdf53636bab..97e6a2aa1f89b 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestFieldMapperTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/TDigestFieldMapperTests.java @@ -510,4 +510,9 @@ public List invalidExample() throws IOException { public void testSyntheticSourceKeepArrays() { // The mapper expects to parse an array of values by default, it's not compatible with array of arrays. } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java index 384a321d524be..fab615f72c669 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java @@ -219,4 +219,9 @@ public void testInvalidOffsets() throws IOException { protected List getSortShortcutSupport() { return List.of(); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index 3ff591cc85939..1a3d6a2be38af 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -2374,4 +2374,9 @@ private void givenModelSettings(String inferenceId, MinimalServiceSettings model protected List getSortShortcutSupport() { return List.of(); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java index 8a684122cb658..53895e9cea8b7 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java @@ -475,4 +475,9 @@ protected IngestScriptSupport ingestScriptSupport() { protected List getSortShortcutSupport() { return List.of(); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java index 122c4ec9004a4..3183c55494708 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java @@ -623,4 +623,9 @@ protected boolean supportsCopyTo() { protected List getSortShortcutSupport() { return List.of(new SortShortcutSupport(this::minimalMapping, this::writeField, true)); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index 031b75f342793..0a4067d7d7f19 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -354,4 +354,9 @@ protected List getSortShortcutSupport() { new SortShortcutSupport(this::minimalMapping, this::writeField, false) ); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapperTests.java b/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapperTests.java index 5c26222ff1618..71ecdb0808e1d 100644 --- a/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapperTests.java @@ -224,4 +224,9 @@ public void testDisableIndex() throws IOException { protected List getSortShortcutSupport() { return List.of(); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java b/x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java index d796052ad36b2..16a4186407ed6 100644 --- a/x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java +++ b/x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java @@ -803,4 +803,9 @@ protected IngestScriptSupport ingestScriptSupport() { protected List getSortShortcutSupport() { return List.of(); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index eb92b2b0a6774..a2d963b941a29 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -191,8 +191,14 @@ Number parsedNullValue() { } private IndexType indexType() { - if (indexed.get() == false && hasDocValues.get() && useTimeSeriesDocValuesSkippers(indexSettings, dimension.get())) { - return IndexType.skippers(); + if (indexed.get() == false && hasDocValues.get()) { + if (useTimeSeriesDocValuesSkippers(indexSettings, dimension.get())) { + return IndexType.skippers(); + } + if (indexSettings.useDocValuesSkipper() + && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.STANDARD_INDEXES_USE_SKIPPERS)) { + return IndexType.skippers(); + } } return IndexType.points(indexed.get(), hasDocValues.get()); } diff --git a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index f6de58e04108e..ed5629aebfb6d 100644 --- a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -203,4 +203,9 @@ public List invalidExample() throws IOException { protected List getSortShortcutSupport() { return List.of(new SortShortcutSupport(this::minimalMapping, this::writeField, true)); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java index 69c3792a0f665..2ca01a5367157 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java @@ -777,31 +777,27 @@ public void testSnapshotMountedIndexWithTimestampsRecordsTimestampRangeInIndexMe final IndexLongFieldRange eventIngestedRange = indexMetadata.getEventIngestedRange(); assertTrue(eventIngestedRange.isComplete()); - if (indexed) { - assertThat(timestampRange, not(sameInstance(IndexLongFieldRange.UNKNOWN))); - assertThat(eventIngestedRange, not(sameInstance(IndexLongFieldRange.UNKNOWN))); - if (docCount == 0) { - assertThat(timestampRange, sameInstance(IndexLongFieldRange.EMPTY)); - assertThat(eventIngestedRange, sameInstance(IndexLongFieldRange.EMPTY)); - } else { - assertThat(timestampRange, not(sameInstance(IndexLongFieldRange.EMPTY))); - assertThat(eventIngestedRange, not(sameInstance(IndexLongFieldRange.EMPTY))); - - // both @timestamp and event.ingested have the same resolution in this test - DateFieldMapper.Resolution resolution = dateType.equals("date") - ? DateFieldMapper.Resolution.MILLISECONDS - : DateFieldMapper.Resolution.NANOSECONDS; - - assertThat(timestampRange.getMin(), greaterThanOrEqualTo(resolution.convert(Instant.parse("2020-11-26T00:00:00Z")))); - assertThat(timestampRange.getMin(), lessThanOrEqualTo(resolution.convert(Instant.parse("2020-11-27T00:00:00Z")))); - - assertThat(eventIngestedRange.getMin(), greaterThanOrEqualTo(resolution.convert(Instant.parse("2020-11-26T00:00:00Z")))); - assertThat(eventIngestedRange.getMin(), lessThanOrEqualTo(resolution.convert(Instant.parse("2020-11-27T00:00:00Z")))); - } + assertThat(timestampRange, not(sameInstance(IndexLongFieldRange.UNKNOWN))); + assertThat(eventIngestedRange, not(sameInstance(IndexLongFieldRange.UNKNOWN))); + if (docCount == 0) { + assertThat(timestampRange, sameInstance(IndexLongFieldRange.EMPTY)); + assertThat(eventIngestedRange, sameInstance(IndexLongFieldRange.EMPTY)); } else { - assertThat(timestampRange, sameInstance(IndexLongFieldRange.UNKNOWN)); - assertThat(eventIngestedRange, sameInstance(IndexLongFieldRange.UNKNOWN)); + assertThat(timestampRange, not(sameInstance(IndexLongFieldRange.EMPTY))); + assertThat(eventIngestedRange, not(sameInstance(IndexLongFieldRange.EMPTY))); + + // both @timestamp and event.ingested have the same resolution in this test + DateFieldMapper.Resolution resolution = dateType.equals("date") + ? DateFieldMapper.Resolution.MILLISECONDS + : DateFieldMapper.Resolution.NANOSECONDS; + + assertThat(timestampRange.getMin(), greaterThanOrEqualTo(resolution.convert(Instant.parse("2020-11-26T00:00:00Z")))); + assertThat(timestampRange.getMin(), lessThanOrEqualTo(resolution.convert(Instant.parse("2020-11-27T00:00:00Z")))); + + assertThat(eventIngestedRange.getMin(), greaterThanOrEqualTo(resolution.convert(Instant.parse("2020-11-26T00:00:00Z")))); + assertThat(eventIngestedRange.getMin(), lessThanOrEqualTo(resolution.convert(Instant.parse("2020-11-27T00:00:00Z")))); } + } public void testSnapshotOfSearchableSnapshotIncludesNoDataButCanBeRestored() throws Exception { diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/CartesianFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/CartesianFieldMapperTests.java index a2560bb38c6ce..4babda8f00073 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/CartesianFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/CartesianFieldMapperTests.java @@ -157,4 +157,9 @@ public void testZValueGeoJSON() throws IOException { assertThat(e.getMessage(), containsString("failed to parse field [" + FIELD_NAME + "] of type")); assertThat(e.getCause().getMessage(), containsString("found Z value [34567.33] but [ignore_z_value] parameter is [false]")); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoFieldMapperTests.java index 64f9f2df4c707..4ce60899feeb2 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoFieldMapperTests.java @@ -57,4 +57,9 @@ protected Object getSampleObjectForDocument() { } protected abstract String getFieldName(); + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } } diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index a82bb21bb3707..9f1cd898b9c35 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -1277,4 +1277,9 @@ public List invalidExample() throws IOException { protected List getSortShortcutSupport() { return List.of(new SortShortcutSupport(this::minimalMapping, this::writeField, false)); } + + @Override + protected boolean supportsDocValuesSkippers() { + return false; + } }