[8.x] Fix ignore_above for flattened fields (#112944) (#113308)

lkts · salvatore-campagna · web-flow · commit 9a6fcd6d627c · 2024-09-21T06:42:16.000+10:00
* Fix `ignore_above` for flattened fields (#112944) Flattened fields do no actually use `ignore_above`. When retrieving source, `ignore_above` behaves as expected. Anyway, it is ignored when fetching field values through the fields API. In that case values are returned no matter the `ignore_above` setting. Here we implement the filtering logic in the `ValueFecther` implementation of `RootFlattenedFieldType`. (cherry picked from commit 91674f8) * Fix --------- Co-authored-by: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com>
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml
@@ -0,0 +1,121 @@
+---
+flattened ignore_above single-value field:
+  - requires:
+      cluster_features: [ "flattened.ignore_above_support" ]
+      reason: introduce ignore_above support in flattened fields
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+                ignore_above: 5
+              flat:
+                type: flattened
+                ignore_above: 5
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        refresh: true
+        body:
+          keyword: "foo"
+          flat: { "value": "foo", "key": "foo key" }
+
+  - do:
+      index:
+        index: test
+        id: "2"
+        refresh: true
+        body:
+          keyword: "foo bar"
+          flat: { "value": "foo bar",  "key": "foo bar key"}
+
+  - do:
+      search:
+        index: test
+        body:
+          fields:
+            - keyword
+            - flat
+          query:
+            match_all: {}
+
+  - match: { hits.total.value: 2 }
+
+  - match: { hits.hits.0._source.keyword: "foo" }
+  - match: { hits.hits.0._source.flat.value: "foo" }
+  - match: { hits.hits.0._source.flat.key: "foo key" }
+  - match: { hits.hits.1._source.keyword: "foo bar" }
+  - match: { hits.hits.1._source.flat.value: "foo bar" }
+  - match: { hits.hits.1._source.flat.key: "foo bar key" }
+
+  - match: { hits.hits.0.fields.keyword.0: "foo" }
+  - match: { hits.hits.0.fields.flat.0.value: "foo" }
+  - match: { hits.hits.1.fields.keyword.0: null }
+  - match: { hits.hits.1.fields.flat.0.value: null }
+
+---
+flattened ignore_above multi-value field:
+  - requires:
+      cluster_features: [ "flattened.ignore_above_support" ]
+      reason: introduce ignore_above support in flattened fields
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+                ignore_above: 5
+              flat:
+                type: flattened
+                ignore_above: 5
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        refresh: true
+        body:
+          keyword: ["foo","bar"]
+          flat: { "value": ["foo", "bar"], "key": "foo bar array key" }
+
+  - do:
+      index:
+        index: test
+        id: "2"
+        refresh: true
+        body:
+          keyword: ["foobar", "foo", "bar"]
+          flat: { "value": ["foobar", "foo"], "key": ["foo key", "bar key"]}
+
+  - do:
+      search:
+        index: test
+        body:
+          fields:
+            - keyword
+            - flat
+          query:
+            match_all: { }
+
+  - match: { hits.total.value: 2 }
+
+  - match: { hits.hits.0._source.keyword: ["foo", "bar"] }
+  - match: { hits.hits.0._source.flat.value: ["foo", "bar"] }
+  - match: { hits.hits.0._source.flat.key: "foo bar array key" }
+  - match: { hits.hits.1._source.keyword: ["foobar", "foo", "bar"] }
+  - match: { hits.hits.1._source.flat.value: ["foobar", "foo"] }
+  - match: { hits.hits.1._source.flat.key: ["foo key", "bar key"] }
+
+  - match: { hits.hits.0.fields.keyword: [ "foo", "bar" ] }
+  - match: { hits.hits.0.fields.flat.0.value: ["foo", "bar"] }
+  - match: { hits.hits.0.fields.flat.0.key: null }
+  - match: { hits.hits.1.fields.keyword: [ "foo", "bar" ] }
+  - match: { hits.hits.1.fields.flat.0.value: "foo" }
+  - match: { hits.hits.1.fields.flat.0.key: null }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java
@@ -11,6 +11,7 @@
 
 import org.elasticsearch.features.FeatureSpecification;
 import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 
 import java.util.Set;
@@ -38,7 +39,8 @@ public Set<NodeFeature> getFeatures() {
             SourceFieldMapper.SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX,
             Mapper.SYNTHETIC_SOURCE_KEEP_FEATURE,
             SourceFieldMapper.SYNTHETIC_SOURCE_WITH_COPY_TO_AND_DOC_VALUES_FALSE_SUPPORT,
-            SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_FIX
+            SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_FIX,
+            FlattenedFieldMapper.IGNORE_ABOVE_SUPPORT
         );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java
@@ -38,6 +38,7 @@
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.FieldDataContext;
@@ -73,9 +74,12 @@
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 
 /**
@@ -105,6 +109,8 @@
  */
 public final class FlattenedFieldMapper extends FieldMapper {
 
+    public static final NodeFeature IGNORE_ABOVE_SUPPORT = new NodeFeature("flattened.ignore_above_support");
+
     public static final String CONTENT_TYPE = "flattened";
     public static final String KEYED_FIELD_SUFFIX = "._keyed";
     public static final String TIME_SERIES_DIMENSIONS_ARRAY_PARAM = "time_series_dimensions";
@@ -214,7 +220,8 @@ public FlattenedFieldMapper build(MapperBuilderContext context) {
                 meta.get(),
                 splitQueriesOnWhitespace.get(),
                 eagerGlobalOrdinals.get(),
-                dimensions.get()
+                dimensions.get(),
+                ignoreAbove.getValue()
             );
             return new FlattenedFieldMapper(leafName(), ft, builderParams(this, context), this);
         }
@@ -642,16 +649,18 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme
         private final boolean eagerGlobalOrdinals;
         private final List<String> dimensions;
         private final boolean isDimension;
+        private final int ignoreAbove;
 
         public RootFlattenedFieldType(
             String name,
             boolean indexed,
             boolean hasDocValues,
             Map<String, String> meta,
             boolean splitQueriesOnWhitespace,
-            boolean eagerGlobalOrdinals
+            boolean eagerGlobalOrdinals,
+            int ignoreAbove
         ) {
-            this(name, indexed, hasDocValues, meta, splitQueriesOnWhitespace, eagerGlobalOrdinals, Collections.emptyList());
+            this(name, indexed, hasDocValues, meta, splitQueriesOnWhitespace, eagerGlobalOrdinals, Collections.emptyList(), ignoreAbove);
         }
 
         public RootFlattenedFieldType(
@@ -661,7 +670,8 @@ public RootFlattenedFieldType(
             Map<String, String> meta,
             boolean splitQueriesOnWhitespace,
             boolean eagerGlobalOrdinals,
-            List<String> dimensions
+            List<String> dimensions,
+            int ignoreAbove
         ) {
             super(
                 name,
@@ -675,6 +685,7 @@ public RootFlattenedFieldType(
             this.eagerGlobalOrdinals = eagerGlobalOrdinals;
             this.dimensions = dimensions;
             this.isDimension = dimensions.isEmpty() == false;
+            this.ignoreAbove = ignoreAbove;
         }
 
         @Override
@@ -708,7 +719,67 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
 
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
-            return SourceValueFetcher.identity(name(), context, format);
+            if (format != null) {
+                throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
+            }
+            return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet());
+        }
+
+        private SourceValueFetcher sourceValueFetcher(Set<String> sourcePaths) {
+            return new SourceValueFetcher(sourcePaths, null) {
+                @Override
+                @SuppressWarnings("unchecked")
+                protected Object parseSourceValue(Object value) {
+                    if (value instanceof Map<?, ?> valueAsMap && valueAsMap.isEmpty() == false) {
+                        final Map<String, Object> result = filterIgnoredValues((Map<String, Object>) valueAsMap);
+                        return result.isEmpty() ? null : result;
+                    }
+                    if (value instanceof String valueAsString && valueAsString.length() <= ignoreAbove) {
+                        return valueAsString;
+                    }
+                    return null;
+                }
+
+                private Map<String, Object> filterIgnoredValues(final Map<String, Object> values) {
+                    final Map<String, Object> result = new HashMap<>();
+                    for (final Map.Entry<String, Object> entry : values.entrySet()) {
+                        Object value = filterIgnoredValues(entry.getValue());
+                        if (value != null) {
+                            result.put(entry.getKey(), value);
+                        }
+                    }
+                    return result;
+                }
+
+                private Object filterIgnoredValues(final Object entryValue) {
+                    if (entryValue instanceof List<?> valueAsList) {
+                        final List<Object> validValues = new ArrayList<>();
+                        for (Object value : valueAsList) {
+                            if (value instanceof String valueAsString) {
+                                if (valueAsString.length() <= ignoreAbove) {
+                                    validValues.add(valueAsString);
+                                }
+                            } else {
+                                validValues.add(value);
+                            }
+                        }
+                        if (validValues.isEmpty()) {
+                            return null;
+                        }
+                        if (validValues.size() == 1) {
+                            // NOTE: for single-value flattened fields do not return an array
+                            return validValues.get(0);
+                        }
+                        return validValues;
+                    } else if (entryValue instanceof String valueAsString) {
+                        if (valueAsString.length() <= ignoreAbove) {
+                            return valueAsString;
+                        }
+                        return null;
+                    }
+                    return entryValue;
+                }
+            };
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java
