Fix dropped ignore above fields (elastic#137394)

Currently, the CompositeSyntheticFieldLoaders for the various text-family
field mappers only load the stored fields containing the original
non-ignored values if ignore_above is set to a non-default value. But if a
value is ignored due to hitting a default ignore_above limit, it won't be
loaded by the field loader. This PR corrects that logic.

Fixes elastic#137360.

Co-authored-by: Felix Barnsteiner <[email protected]>

Author: jordan-powers

docs/changelog/137394.yaml

@@ -0,0 +1,6 @@
+pr: 137394
+summary: Fix dropped ignore above fields
+area: Mapping
+type: bug
+issues:
+ - 137360

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java

@@ -295,7 +295,8 @@ private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOExcepti
             String parentFieldName = searchExecutionContext.parentPath(name());
             var parent = searchExecutionContext.lookup().fieldType(parentFieldName);
 
-            if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent && keywordParent.ignoreAbove().isSet()) {
+            if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent
+                && keywordParent.ignoreAbove().valuesPotentiallyIgnored()) {
                 final String parentFallbackFieldName = keywordParent.syntheticSourceFallbackFieldName();
                 if (parent.isStored()) {
                     return storedFieldFetcher(parentFieldName, parentFallbackFieldName);
@@ -323,7 +324,7 @@ private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOExcepti
             final SearchExecutionContext searchExecutionContext,
             final KeywordFieldMapper.KeywordFieldType keywordDelegate
         ) {
-            if (keywordDelegate.ignoreAbove().isSet()) {
+            if (keywordDelegate.ignoreAbove().valuesPotentiallyIgnored()) {
                 // because we don't know whether the delegate field will be ignored during parsing, we must also check the current field
                 String fieldName = name();
                 String fallbackName = syntheticSourceFallbackFieldName();

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -1407,7 +1407,7 @@ protected BytesRef preserve(BytesRef value) {
 
         // if ignore_above is set, then there is a chance that this field will be ignored. In such cases, we save an
         // extra copy of the field for supporting synthetic source. This layer will check that copy.
-        if (fieldType().ignoreAbove.isSet()) {
+        if (fieldType().ignoreAbove.valuesPotentiallyIgnored()) {
             final String fieldName = fieldType().syntheticSourceFallbackFieldName();
             layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(fieldName) {
                 @Override

server/src/main/java/org/elasticsearch/index/mapper/Mapper.java

@@ -136,7 +136,7 @@ default boolean supportsVersion(IndexVersion indexCreatedVersion) {
      * This class models the ignore_above parameter in indices.
      */
     public static final class IgnoreAbove {
-
+        // We use Integer.MAX_VALUE to represent a no-op, accepting all values.
         public static final int IGNORE_ABOVE_DEFAULT_VALUE = Integer.MAX_VALUE;
         public static final int IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES = 8191;
 
@@ -172,6 +172,15 @@ public boolean isSet() {
             return Integer.valueOf(get()).equals(defaultValue) == false;
         }
 
+        /**
+         * Returns whether values are potentially ignored, either by an explicitly configured ignore_above or by the default value.
+         */
+        public boolean valuesPotentiallyIgnored() {
+            // We use Integer.MAX_VALUE to represent accepting all values. If the value is anything else, then either we have an
+            // explicitly configured ignore_above, or we have a non no-op default.
+            return get() != Integer.MAX_VALUE;
+        }
+
         /**
          * Returns whether the given string will be ignored.
          */

server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

@@ -1026,15 +1026,15 @@ public boolean isAggregatable() {
          * A delegate by definition must have doc_values or be stored so most of the time it can be used for loading.
          */
         public boolean canUseSyntheticSourceDelegateForLoading() {
-            return syntheticSourceDelegate.isPresent() && syntheticSourceDelegate.get().ignoreAbove().isSet() == false;
+            return syntheticSourceDelegate.isPresent() && syntheticSourceDelegate.get().ignoreAbove().valuesPotentiallyIgnored() == false;
         }
 
         /**
          * Returns true if the delegate sub-field can be used for querying only (ie. isSearchable must be true)
          */
         public boolean canUseSyntheticSourceDelegateForQuerying() {
             return syntheticSourceDelegate.isPresent()
-                && syntheticSourceDelegate.get().ignoreAbove().isSet() == false
+                && syntheticSourceDelegate.get().ignoreAbove().valuesPotentiallyIgnored() == false
                 && syntheticSourceDelegate.get().isSearchable();
         }
 

server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java

@@ -926,7 +926,7 @@ protected SyntheticSourceSupport syntheticSourceSupport() {
                 () -> new FlattenedSortedSetDocValuesSyntheticFieldLoader(
                     fullPath(),
                     fullPath() + KEYED_FIELD_SUFFIX,
-                    fieldType().ignoreAbove.isSet() ? fullPath() + KEYED_IGNORED_VALUES_FIELD_SUFFIX : null,
+                    fieldType().ignoreAbove.valuesPotentiallyIgnored() ? fullPath() + KEYED_IGNORED_VALUES_FIELD_SUFFIX : null,
                     leafName()
                 )
             );

server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java

@@ -25,6 +25,7 @@ public void test_ignore_above_with_value_and_index_mode_and_index_version() {
         // when/then
         assertEquals(123, ignoreAbove.get());
         assertTrue(ignoreAbove.isSet());
+        assertTrue(ignoreAbove.valuesPotentiallyIgnored());
     }
 
     public void test_ignore_above_with_value_only() {
@@ -34,6 +35,7 @@ public void test_ignore_above_with_value_only() {
         // when/then
         assertEquals(123, ignoreAbove.get());
         assertTrue(ignoreAbove.isSet());
+        assertTrue(ignoreAbove.valuesPotentiallyIgnored());
     }
 
     public void test_ignore_above_with_null_value_should_throw() {
@@ -52,6 +54,7 @@ public void test_ignore_above_with_null_value() {
         // when/then
         assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, ignoreAbove.get());
         assertFalse(ignoreAbove.isSet());
+        assertFalse(ignoreAbove.valuesPotentiallyIgnored());
     }
 
     public void test_ignore_above_with_null_value_and_logsdb_index_mode() {
@@ -61,6 +64,7 @@ public void test_ignore_above_with_null_value_and_logsdb_index_mode() {
         // when/then
         assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, ignoreAbove.get());
         assertFalse(ignoreAbove.isSet());
+        assertTrue(ignoreAbove.valuesPotentiallyIgnored());
     }
 
     public void test_ignore_above_with_null_everything() {
@@ -70,6 +74,7 @@ public void test_ignore_above_with_null_everything() {
         // when/then
         assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, ignoreAbove.get());
         assertFalse(ignoreAbove.isSet());
+        assertFalse(ignoreAbove.valuesPotentiallyIgnored());
     }
 
     public void test_ignore_above_default_for_standard_indices() {
@@ -79,6 +84,7 @@ public void test_ignore_above_default_for_standard_indices() {
         // when/then
         assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, ignoreAbove.get());
         assertFalse(ignoreAbove.isSet());
+        assertFalse(ignoreAbove.valuesPotentiallyIgnored());
     }
 
     public void test_ignore_above_default_for_logsdb_indices() {
@@ -88,6 +94,7 @@ public void test_ignore_above_default_for_logsdb_indices() {
         // when/then
         assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, ignoreAbove.get());
         assertFalse(ignoreAbove.isSet());
+        assertTrue(ignoreAbove.valuesPotentiallyIgnored());
     }
 
     public void test_string_isIgnored() {