Enable skippers for fields with index=false and doc_values=true

modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java

@@ -119,16 +119,6 @@ public void testValidateInvalidFieldType() {
         );
     }
 
-    public void testValidateNotIndexed() {
-        Exception e = expectThrows(IllegalArgumentException.class, () -> createMapperService(timestampMapping(true, b -> {
-            b.startObject("@timestamp");
-            b.field("type", "date");
-            b.field("index", false);
-            b.endObject();
-        })));
-        assertThat(e.getMessage(), equalTo("data stream timestamp field [@timestamp] is not indexed"));
-    }
-
     public void testValidateNotDocValues() {
         Exception e = expectThrows(IllegalArgumentException.class, () -> createMapperService(timestampMapping(true, b -> {
             b.startObject("@timestamp");

modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java

@@ -671,4 +671,9 @@ protected IngestScriptSupport ingestScriptSupport() {
     protected List<SortShortcutSupport> getSortShortcutSupport() {
         return List.of();
     }
+
+    @Override
+    protected boolean supportsDocValuesSkippers() {
+        return false;
+    }
 }

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java

@@ -193,12 +193,23 @@ protected Parameter<?>[] getParameters() {
             return new Parameter<?>[] { indexed, hasDocValues, stored, ignoreMalformed, meta, scalingFactor, coerce, nullValue, metric };
         }
 
+        private IndexType indexType() {
+            if (indexed.getValue()) {
+                return IndexType.points(true, hasDocValues.getValue());
+            }
+            if (hasDocValues.getValue()
+                && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.STANDARD_INDEXES_USE_SKIPPERS)
+                && indexSettings.useDocValuesSkipper()) {
+                return IndexType.skippers();
+            }
+            return IndexType.points(false, hasDocValues.getValue());
+        }
+
         @Override
         public ScaledFloatFieldMapper build(MapperBuilderContext context) {
-            IndexType indexType = IndexType.points(indexed.get(), hasDocValues.get());
             ScaledFloatFieldType type = new ScaledFloatFieldType(
                 context.buildFullName(leafName()),
-                indexType,
+                indexType(),
                 stored.getValue(),
                 meta.getValue(),
                 scalingFactor.getValue(),

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java

@@ -390,4 +390,9 @@ public void testLoadSyntheticSourceFromStringOrBytesRef() throws IOException {
     protected List<SortShortcutSupport> getSortShortcutSupport() {
         return List.of();
     }
+
+    @Override
+    protected boolean supportsDocValuesSkippers() {
+        return false;
+    }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java

@@ -220,4 +220,9 @@ protected IngestScriptSupport ingestScriptSupport() {
     protected List<SortShortcutSupport> getSortShortcutSupport() {
         return List.of();
     }
+
+    @Override
+    protected boolean supportsDocValuesSkippers() {
+        return false;
+    }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java

@@ -214,4 +214,9 @@ protected IngestScriptSupport ingestScriptSupport() {
     protected List<SortShortcutSupport> getSortShortcutSupport() {
         return List.of();
     }
+
+    @Override
+    protected boolean supportsDocValuesSkippers() {
+        return false;
+    }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java

@@ -882,4 +882,9 @@ protected IngestScriptSupport ingestScriptSupport() {
     protected List<SortShortcutSupport> getSortShortcutSupport() {
         return List.of();
     }
+
+    @Override
+    protected boolean supportsDocValuesSkippers() {
+        return false;
+    }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java

@@ -269,4 +269,9 @@ public void testAggregationsDocValuesDisabled() throws IOException {
     protected List<SortShortcutSupport> getSortShortcutSupport() {
         return List.of(new SortShortcutSupport(this::minimalMapping, this::writeField, true));
     }
+
+    @Override
+    protected boolean supportsDocValuesSkippers() {
+        return false;
+    }
 }

plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java

@@ -323,4 +323,9 @@ protected IngestScriptSupport ingestScriptSupport() {
     protected List<SortShortcutSupport> getSortShortcutSupport() {
         return List.of(new SortShortcutSupport(this::minimalMapping, this::writeField, true));
     }
+
+    @Override
+    protected boolean supportsDocValuesSkippers() {
+        return false;
+    }
 }

plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java

@@ -709,4 +709,9 @@ protected IngestScriptSupport ingestScriptSupport() {
     protected List<SortShortcutSupport> getSortShortcutSupport() {
         return List.of();
     }
+
+    @Override
+    protected boolean supportsDocValuesSkippers() {
+        return false;
+    }
 }

plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java

@@ -153,4 +153,9 @@ protected IngestScriptSupport ingestScriptSupport() {
     protected List<SortShortcutSupport> getSortShortcutSupport() {
         return List.of(new SortShortcutSupport(this::minimalMapping, this::writeField, false));
     }
+
+    @Override
+    protected boolean supportsDocValuesSkippers() {
+        return false;
+    }
 }

rest-api-spec/build.gradle

@@ -110,6 +110,7 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.skipTest("logsdb/10_settings/override sort order settings", "we changed the error message")
   task.skipTest("tsdb/10_settings/set start_time and end_time without timeseries mode", "we don't validate for index_mode=tsdb when setting start_date/end_date anymore")
   task.skipTest("tsdb/10_settings/set start_time, end_time and routing_path via put settings api without time_series mode", "we don't validate for index_mode=tsdb when setting start_date/end_date anymore")
+  task.skipTest("search/140_pre_filter_search_shards/prefilter on non-indexed date fields", "prefiltering can now use skippers on dv-only fields")
   // Expected deprecation warning to compat yaml tests:
   task.addAllowedWarningRegex("Use of the \\[max_size\\] rollover condition has been deprecated in favour of the \\[max_primary_shard_size\\] condition and will be removed in a later version")
 })

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/140_pre_filter_search_shards.yml

@@ -240,8 +240,8 @@ setup:
 ---
 "prefilter on non-indexed date fields":
   - requires:
-      cluster_features: "gte_v8.1.0"
-      reason: "doc values search was added in 8.1.0"
+      cluster_features: [ "gte_v8.1.0", "mapper.skippers_on_unindexed_fields" ]
+      reason: "doc values search was added in 8.1.0; skippers allow pre-filtering"
 
   - do:
       index:
@@ -274,7 +274,7 @@ setup:
   - match: { _shards.failed: 0 }
   - match: { hits.total: 2 }
 
-  # this is a case where we would normally skip due to rewrite but we can't because we only have doc values
+  # this is a case where we can use docvalues skippers to pre-filter index_1
   - do:
       search:
         rest_total_hits_as_int: true
@@ -283,6 +283,6 @@ setup:
 
   - match: { _shards.total: 3 }
   - match: { _shards.successful: 3 }
-  - match: { _shards.skipped : 0 }
+  - match: { _shards.skipped : 1 }
   - match: { _shards.failed: 0 }
   - match: { hits.total: 2 }

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -197,6 +197,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion TIME_SERIES_ALL_FIELDS_USE_SKIPPERS = def(9_046_0_00, Version.LUCENE_10_3_1);
     public static final IndexVersion UPGRADE_TO_LUCENE_10_3_2 = def(9_047_0_00, Version.LUCENE_10_3_2);
     public static final IndexVersion SECURITY_MIGRATIONS_METADATA_FLATTENED_UPDATE = def(9_048_0_00, Version.LUCENE_10_3_2);
+    public static final IndexVersion STANDARD_INDEXES_USE_SKIPPERS = def(9_049_0_00, Version.LUCENE_10_3_2);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

@@ -149,6 +149,10 @@ private IndexType indexType() {
             if (docValues.get() == false) {
                 return IndexType.NONE;
             }
+            if (indexSettings.useDocValuesSkipper()
+                && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.STANDARD_INDEXES_USE_SKIPPERS)) {
+                return IndexType.skippers();
+            }
             return useTimeSeriesDocValuesSkippers(indexSettings, dimension.get()) ? IndexType.skippers() : IndexType.docValuesOnly();
         }
 

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -400,10 +400,15 @@ private Long parseNullValue(DateFieldType fieldType) {
         }
 
         private IndexType indexType(String fullFieldName) {
-            boolean hasDocValuesSkipper = shouldUseDocValuesSkipper(indexSettings, docValues.getValue(), fullFieldName);
-            if (hasDocValuesSkipper) {
+            if (shouldUseDocValuesSkipper(indexSettings, docValues.getValue(), fullFieldName)) {
                 return IndexType.skippers();
             }
+            if (index.get() == false && docValues.get()) {
+                if (indexSettings.useDocValuesSkipper()
+                    && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.STANDARD_INDEXES_USE_SKIPPERS)) {
+                    return IndexType.skippers();
+                }
+            }
             if (indexCreatedVersion.isLegacyIndexVersion()) {
                 return IndexType.archivedPoints();
             }

server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java

@@ -182,6 +182,12 @@ private IndexType indexType() {
             if (useTimeSeriesDocValuesSkippers(indexSettings, dimension.get())) {
                 return IndexType.skippers();
             }
+            if (indexed.get() == false && hasDocValues.get()) {
+                if (indexSettings.useDocValuesSkipper()
+                    && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.STANDARD_INDEXES_USE_SKIPPERS)) {
+                    return IndexType.skippers();
+                }
+            }
             return IndexType.points(indexed.get(), hasDocValues.get());
         }
 

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -441,7 +441,10 @@ public KeywordFieldMapper build(MapperBuilderContext context) {
 
         private FieldType resolveFieldType(final boolean forceDocValuesSkipper, final String fullFieldName) {
             FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE);
-            if (forceDocValuesSkipper || shouldUseHostnameSkipper(fullFieldName) || shouldUseTimeSeriesSkipper()) {
+            if (forceDocValuesSkipper
+                || shouldUseHostnameSkipper(fullFieldName)
+                || shouldUseTimeSeriesSkipper()
+                || shouldUseStandardSkipper()) {
                 fieldtype = new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES);
             }
             fieldtype.setOmitNorms(this.hasNorms.getValue() == false);
@@ -474,6 +477,13 @@ private boolean shouldUseHostnameSkipper(final String fullFieldName) {
                 && indexSortConfigByHostName(indexSettings.getIndexSortConfig());
         }
 
+        private boolean shouldUseStandardSkipper() {
+            return hasDocValues.get()
+                && indexed.get() == false
+                && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.STANDARD_INDEXES_USE_SKIPPERS)
+                && indexSettings.useDocValuesSkipper();
+        }
+
         private static boolean indexSortConfigByHostName(final IndexSortConfig indexSortConfig) {
             return indexSortConfig != null && indexSortConfig.hasIndexSort() && indexSortConfig.hasSortOnField(HOST_NAME);
         }

server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java

@@ -64,6 +64,7 @@ public class MapperFeatures implements FeatureSpecification {
     public static final NodeFeature GENERIC_VECTOR_FORMAT = new NodeFeature("mapper.vectors.generic_vector_format");
     public static final NodeFeature FIX_DENSE_VECTOR_WRONG_FIELDS = new NodeFeature("mapper.fix_dense_vector_wrong_fields");
     static final NodeFeature BBQ_DISK_STATS_SUPPORT = new NodeFeature("mapper.bbq_disk_stats_support");
+    static final NodeFeature SKIPPERS_ON_UNINDEXED_FIELDS = new NodeFeature("mapper.skippers_on_unindexed_fields");
     static final NodeFeature STORED_FIELDS_SPEC_MERGE_BUG = new NodeFeature("mapper.stored_fields_spec_merge_bug");
 
     @Override
@@ -110,6 +111,7 @@ public Set<NodeFeature> getTestFeatures() {
             BASE64_DENSE_VECTORS,
             FIX_DENSE_VECTOR_WRONG_FIELDS,
             BBQ_DISK_STATS_SUPPORT,
+            SKIPPERS_ON_UNINDEXED_FIELDS,
             STORED_FIELDS_SPEC_MERGE_BUG
         );
         if (ES93GenericFlatVectorsFormat.GENERIC_VECTOR_FORMAT.isEnabled()) {

server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java

@@ -223,8 +223,14 @@ private IndexType indexType() {
             if (indexSettings.getIndexVersionCreated().isLegacyIndexVersion()) {
                 return IndexType.archivedPoints();
             }
-            if (indexed.get() == false && hasDocValues.get() && useTimeSeriesDocValuesSkippers(indexSettings, dimension.get())) {
-                return IndexType.skippers();
+            if (indexed.get() == false && hasDocValues.get()) {
+                if (useTimeSeriesDocValuesSkippers(indexSettings, dimension.get())) {
+                    return IndexType.skippers();
+                }
+                if (indexSettings.useDocValuesSkipper()
+                    && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.STANDARD_INDEXES_USE_SKIPPERS)) {
+                    return IndexType.skippers();
+                }
             }
             return IndexType.points(indexed.get(), hasDocValues.get());
         }

server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java

@@ -268,4 +268,9 @@ protected IngestScriptSupport ingestScriptSupport() {
     protected List<SortShortcutSupport> getSortShortcutSupport() {
         return List.of();
     }
+
+    @Override
+    protected boolean supportsDocValuesSkippers() {
+        return false;
+    }
 }

server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java

@@ -990,4 +990,9 @@ protected IngestScriptSupport ingestScriptSupport() {
     protected List<SortShortcutSupport> getSortShortcutSupport() {
         return List.of();
     }
+
+    @Override
+    protected boolean supportsDocValuesSkippers() {
+        return false;
+    }
 }

server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java

@@ -724,4 +724,9 @@ protected IngestScriptSupport ingestScriptSupport() {
     protected List<SortShortcutSupport> getSortShortcutSupport() {
         return List.of();
     }
+
+    @Override
+    protected boolean supportsDocValuesSkippers() {
+        return false;
+    }
 }

server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java

@@ -50,6 +50,11 @@ protected boolean supportsIgnoreMalformed() {
         return false;
     }
 
+    @Override
+    protected boolean supportsDocValuesSkippers() {
+        return false;
+    }
+
     public void testExistsQueryDocValuesDisabled() throws IOException {
         MapperService mapperService = createMapperService(fieldMapping(b -> {
             minimalMapping(b);

server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java

@@ -1780,4 +1780,9 @@ public void testNormsEnabledWhenIndexModeIsTsdb_bwcCheck() throws IOException {
     protected List<SortShortcutSupport> getSortShortcutSupport() {
         return List.of();
     }
+
+    @Override
+    protected boolean supportsDocValuesSkippers() {
+        return false;
+    }
 }

server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java

@@ -1021,4 +1021,9 @@ public void assertStoredFieldsEquals(String info, IndexReader leftReader, IndexR
     protected List<SortShortcutSupport> getSortShortcutSupport() {
         return List.of(new SortShortcutSupport(this::minimalMapping, this::writeField, true));
     }
+
+    @Override
+    protected boolean supportsDocValuesSkippers() {
+        return false;
+    }
 }

server/src/test/java/org/elasticsearch/index/mapper/vectors/SyntheticVectorsMapperTestCase.java

@@ -271,4 +271,9 @@ private void assertSyntheticVectors(String mapping, BytesReference source, XCont
     protected List<SortShortcutSupport> getSortShortcutSupport() {
         return List.of();
     }
+
+    @Override
+    protected boolean supportsDocValuesSkippers() {
+        return false;
+    }
 }