[9.1] Fix dropped ignore above fields (elastic#137394) (elastic#137414)

* Fix dropped ignore above fields (elastic#137394)

Currently, the CompositeSyntheticFieldLoaders for the various text-family
field mappers only load the stored fields containing the original
non-ignored values if ignore_above is set to a non-default value. But if a
value is ignored due to hitting a default ignore_above limit, it won't be
loaded by the field loader. This PR corrects that logic.

Fixes elastic#137360.

Co-authored-by: Felix Barnsteiner <[email protected]>
(cherry picked from commit 61025ff)

# Conflicts:
#	modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java
#	server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
#	server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

* [CI] Auto commit changes from spotless

---------

Co-authored-by: elasticsearchmachine <[email protected]>

Author: jordan-powers

docs/changelog/137394.yaml

@@ -0,0 +1,6 @@
+pr: 137394
+summary: Fix dropped ignore above fields
+area: Mapping
+type: bug
+issues:
+ - 137360

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java

@@ -253,7 +253,8 @@ private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOExcepti
                 String parentField = searchExecutionContext.parentPath(name());
                 var parent = searchExecutionContext.lookup().fieldType(parentField);
 
-                if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent && keywordParent.ignoreAbove().isSet()) {
+                if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent
+                    && keywordParent.ignoreAbove().valuesPotentiallyIgnored()) {
                     if (parent.isStored()) {
                         return storedFieldFetcher(parentField, keywordParent.originalName());
                     } else if (parent.hasDocValues()) {
@@ -274,7 +275,7 @@ private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOExcepti
                 var kwd = textFieldType.syntheticSourceDelegate();
 
                 if (kwd != null) {
-                    if (kwd.ignoreAbove().isSet()) {
+                    if (kwd.ignoreAbove().valuesPotentiallyIgnored()) {
                         if (kwd.isStored()) {
                             return storedFieldFetcher(kwd.name(), kwd.originalName());
                         } else if (kwd.hasDocValues()) {

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -1326,7 +1326,7 @@ protected BytesRef preserve(BytesRef value) {
             }
         }
 
-        if (fieldType().ignoreAbove.isSet()) {
+        if (fieldType().ignoreAbove.valuesPotentiallyIgnored()) {
             layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName) {
                 @Override
                 protected void writeValue(Object value, XContentBuilder b) throws IOException {

server/src/main/java/org/elasticsearch/index/mapper/Mapper.java

@@ -136,7 +136,7 @@ default boolean supportsVersion(IndexVersion indexCreatedVersion) {
      * This class models the ignore_above parameter in indices.
      */
     public static final class IgnoreAbove {
-
+        // We use Integer.MAX_VALUE to represent a no-op, accepting all values.
         public static final int IGNORE_ABOVE_DEFAULT_VALUE = Integer.MAX_VALUE;
         public static final int IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES = 8191;
 
@@ -172,6 +172,15 @@ public boolean isSet() {
             return Integer.valueOf(get()).equals(defaultValue) == false;
         }
 
+        /**
+         * Returns whether values are potentially ignored, either by an explicitly configured ignore_above or by the default value.
+         */
+        public boolean valuesPotentiallyIgnored() {
+            // We use Integer.MAX_VALUE to represent accepting all values. If the value is anything else, then either we have an
+            // explicitly configured ignore_above, or we have a non no-op default.
+            return get() != Integer.MAX_VALUE;
+        }
+
         /**
          * Returns whether the given string will be ignored.
          */

server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

@@ -1017,15 +1017,15 @@ public boolean isAggregatable() {
          * A delegate by definition must have doc_values or be stored so most of the time it can be used for loading.
          */
         public boolean canUseSyntheticSourceDelegateForLoading() {
-            return syntheticSourceDelegate != null && syntheticSourceDelegate.ignoreAbove().isSet() == false;
+            return syntheticSourceDelegate != null && syntheticSourceDelegate.ignoreAbove().valuesPotentiallyIgnored() == false;
         }
 
         /**
          * Returns true if the delegate sub-field can be used for querying only (ie. isIndexed must be true)
          */
         public boolean canUseSyntheticSourceDelegateForQuerying() {
             return syntheticSourceDelegate != null
-                && syntheticSourceDelegate.ignoreAbove().isSet() == false
+                && syntheticSourceDelegate.ignoreAbove().valuesPotentiallyIgnored() == false
                 && syntheticSourceDelegate.isIndexed();
         }
 

server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java

@@ -921,7 +921,7 @@ protected SyntheticSourceSupport syntheticSourceSupport() {
                 () -> new FlattenedSortedSetDocValuesSyntheticFieldLoader(
                     fullPath(),
                     fullPath() + KEYED_FIELD_SUFFIX,
-                    fieldType().ignoreAbove.isSet() ? fullPath() + KEYED_IGNORED_VALUES_FIELD_SUFFIX : null,
+                    fieldType().ignoreAbove.valuesPotentiallyIgnored() ? fullPath() + KEYED_IGNORED_VALUES_FIELD_SUFFIX : null,
                     leafName()
                 )
             );

server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java

@@ -25,6 +25,7 @@ public void test_ignore_above_with_value_and_index_mode_and_index_version() {
         // when/then
         assertEquals(123, ignoreAbove.get());
         assertTrue(ignoreAbove.isSet());
+        assertTrue(ignoreAbove.valuesPotentiallyIgnored());
     }
 
     public void test_ignore_above_with_value_only() {
@@ -34,6 +35,7 @@ public void test_ignore_above_with_value_only() {
         // when/then
         assertEquals(123, ignoreAbove.get());
         assertTrue(ignoreAbove.isSet());
+        assertTrue(ignoreAbove.valuesPotentiallyIgnored());
     }
 
     public void test_ignore_above_with_null_value_should_throw() {
@@ -52,6 +54,7 @@ public void test_ignore_above_with_null_value() {
         // when/then
         assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, ignoreAbove.get());
         assertFalse(ignoreAbove.isSet());
+        assertFalse(ignoreAbove.valuesPotentiallyIgnored());
     }
 
     public void test_ignore_above_with_null_value_and_logsdb_index_mode() {
@@ -61,6 +64,7 @@ public void test_ignore_above_with_null_value_and_logsdb_index_mode() {
         // when/then
         assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, ignoreAbove.get());
         assertFalse(ignoreAbove.isSet());
+        assertTrue(ignoreAbove.valuesPotentiallyIgnored());
     }
 
     public void test_ignore_above_with_null_everything() {
@@ -70,6 +74,7 @@ public void test_ignore_above_with_null_everything() {
         // when/then
         assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, ignoreAbove.get());
         assertFalse(ignoreAbove.isSet());
+        assertFalse(ignoreAbove.valuesPotentiallyIgnored());
     }
 
     public void test_ignore_above_default_for_standard_indices() {
@@ -79,6 +84,7 @@ public void test_ignore_above_default_for_standard_indices() {
         // when/then
         assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, ignoreAbove.get());
         assertFalse(ignoreAbove.isSet());
+        assertFalse(ignoreAbove.valuesPotentiallyIgnored());
     }
 
     public void test_ignore_above_default_for_logsdb_indices() {
@@ -88,6 +94,7 @@ public void test_ignore_above_default_for_logsdb_indices() {
         // when/then
         assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, ignoreAbove.get());
         assertFalse(ignoreAbove.isSet());
+        assertTrue(ignoreAbove.valuesPotentiallyIgnored());
     }
 
     public void test_string_isIgnored() {