Flattened field type onlt store keyed field.

Author: martijnvg

server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java

@@ -152,6 +152,7 @@ public static class Builder extends FieldMapper.Builder {
             }
         });
 
+        private final Parameter<Boolean> storeRoot = Parameter.boolParam("store_root", false, m -> builder(m).storeRoot.get(), false);
         private final Parameter<Boolean> indexed = Parameter.indexParam(m -> builder(m).indexed.get(), true);
         private final Parameter<Boolean> hasDocValues = Parameter.docValuesParam(m -> builder(m).hasDocValues.get(), true);
 
@@ -245,6 +246,7 @@ private Builder(
         protected Parameter<?>[] getParameters() {
             return new Parameter<?>[] {
                 indexed,
+                storeRoot,
                 hasDocValues,
                 depthLimit,
                 nullValue,
@@ -1105,7 +1107,8 @@ private FlattenedFieldMapper(String leafName, MappedFieldType mappedFieldType, B
             builder.depthLimit.get(),
             builder.ignoreAbove.get(),
             builder.nullValue.get(),
-            builder.usesBinaryDocValues
+            builder.usesBinaryDocValues,
+            builder.storeRoot.get()
         );
     }
 

server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java

@@ -42,6 +42,7 @@ class FlattenedFieldParser {
     private final String nullValue;
 
     private final boolean usesBinaryDocValues;
+    private final boolean storeRoot;
 
     FlattenedFieldParser(
         String rootFieldFullPath,
@@ -51,7 +52,8 @@ class FlattenedFieldParser {
         int depthLimit,
         int ignoreAbove,
         String nullValue,
-        boolean usesBinaryDocValues
+        boolean usesBinaryDocValues,
+        boolean storeRoot
     ) {
         this.rootFieldFullPath = rootFieldFullPath;
         this.keyedFieldFullPath = keyedFieldFullPath;
@@ -61,6 +63,7 @@ class FlattenedFieldParser {
         this.ignoreAbove = ignoreAbove;
         this.nullValue = nullValue;
         this.usesBinaryDocValues = usesBinaryDocValues;
+        this.storeRoot = storeRoot;
     }
 
     public void parse(final DocumentParserContext documentParserContext) throws IOException {
@@ -136,7 +139,8 @@ private void addField(Context context, ContentPath path, String currentName, Str
         String keyedValue = createKeyedValue(key, value);
         BytesRef bytesKeyedValue = new BytesRef(keyedValue);
 
-        if (value.length() > ignoreAbove) {
+        boolean ignoreRelevant = usesBinaryDocValues == false || fieldType.indexType().hasDenseIndex();
+        if (ignoreRelevant && value.length() > ignoreAbove) {
             if (context.documentParserContext().mappingLookup().isSourceSynthetic()) {
                 context.documentParserContext.doc().add(new StoredField(keyedIgnoredValuesFieldFullPath, bytesKeyedValue));
             }
@@ -145,7 +149,7 @@ private void addField(Context context, ContentPath path, String currentName, Str
 
         // check the keyed value doesn't exceed the IndexWriter.MAX_TERM_LENGTH limit enforced by Lucene at index time
         // in that case we can already throw a more user friendly exception here which includes the offending fields key and value lengths
-        if (bytesKeyedValue.length > IndexWriter.MAX_TERM_LENGTH) {
+        if (ignoreRelevant && bytesKeyedValue.length > IndexWriter.MAX_TERM_LENGTH) {
             String msg = "Flattened field ["
                 + rootFieldFullPath
                 + "] contains one immense field"
@@ -162,24 +166,30 @@ private void addField(Context context, ContentPath path, String currentName, Str
         }
         BytesRef bytesValue = new BytesRef(value);
         if (fieldType.indexType().hasTerms()) {
-            context.documentParserContext.doc().add(new StringField(rootFieldFullPath, bytesValue, Field.Store.NO));
+            if (storeRoot) {
+                context.documentParserContext.doc().add(new StringField(rootFieldFullPath, bytesValue, Field.Store.NO));
+            }
             context.documentParserContext.doc().add(new StringField(keyedFieldFullPath, bytesKeyedValue, Field.Store.NO));
         }
 
         if (fieldType.hasDocValues()) {
             if (usesBinaryDocValues) {
-                MultiValuedBinaryDocValuesField.SeparateCount.addToSeparateCountMultiBinaryFieldInDoc(
-                    context.documentParserContext.doc(),
-                    rootFieldFullPath,
-                    bytesValue
-                );
+                if (storeRoot) {
+                    MultiValuedBinaryDocValuesField.SeparateCount.addToSeparateCountMultiBinaryFieldInDoc(
+                        context.documentParserContext.doc(),
+                        rootFieldFullPath,
+                        bytesValue
+                    );
+                }
                 MultiValuedBinaryDocValuesField.SeparateCount.addToSeparateCountMultiBinaryFieldInDoc(
                     context.documentParserContext.doc(),
                     keyedFieldFullPath,
                     bytesKeyedValue
                 );
             } else {
-                context.documentParserContext.doc().add(new SortedSetDocValuesField(rootFieldFullPath, bytesValue));
+                if (storeRoot) {
+                    context.documentParserContext.doc().add(new SortedSetDocValuesField(rootFieldFullPath, bytesValue));
+                }
                 context.documentParserContext.doc().add(new SortedSetDocValuesField(keyedFieldFullPath, bytesKeyedValue));
             }
 

server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java

@@ -63,7 +63,7 @@ public class FlattenedFieldMapperTests extends MapperTestCase {
 
     @Override
     protected void minimalMapping(XContentBuilder b) throws IOException {
-        b.field("type", "flattened");
+        b.field("type", "flattened").field("store_root", true);
     }
 
     @Override
@@ -144,6 +144,33 @@ public void testDefaults() throws Exception {
         assertEquals(0, fieldNamesFields.size());
     }
 
+    public void testDefaults2() throws Exception {
+        DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "flattened")));
+        ParsedDocument parsedDoc = mapper.parse(source(b -> b.startObject("field").field("key", "value").endObject()));
+
+        // Check the root fields.
+        List<IndexableField> fields = parsedDoc.rootDoc().getFields("field");
+        assertEquals(0, fields.size());
+
+        // Check the keyed fields.
+        List<IndexableField> keyedFields = parsedDoc.rootDoc().getFields("field._keyed");
+        assertEquals(2, keyedFields.size());
+
+        assertEquals("field._keyed", keyedFields.get(0).name());
+        assertEquals(new BytesRef("key\0value"), keyedFields.get(0).binaryValue());
+        assertFalse(keyedFields.get(0).fieldType().stored());
+        assertTrue(keyedFields.get(0).fieldType().omitNorms());
+        assertEquals(DocValuesType.NONE, keyedFields.get(0).fieldType().docValuesType());
+
+        assertEquals("field._keyed", keyedFields.get(1).name());
+        assertEquals(new BytesRef("key\0value"), keyedFields.get(1).binaryValue());
+        assertEquals(DocValuesType.SORTED_SET, keyedFields.get(1).fieldType().docValuesType());
+
+        // Check that there is no 'field names' field.
+        List<IndexableField> fieldNamesFields = parsedDoc.rootDoc().getFields(FieldNamesFieldMapper.NAME);
+        assertEquals(0, fieldNamesFields.size());
+    }
+
     public void testBinaryDocValuesType() throws Exception {
         DocumentMapper mapper = createMapperService(
             Settings.builder().put(IndexSettings.USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING.getKey(), true).build(),
@@ -258,6 +285,7 @@ public void testDisableIndex() throws Exception {
         DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
             b.field("type", "flattened");
             b.field("index", false);
+            b.field("store_root", true);
         }));
         ParsedDocument parsedDoc = mapper.parse(source(b -> b.startObject("field").field("key", "value").endObject()));
 
@@ -276,6 +304,7 @@ public void testDisableIndexBinary() throws Exception {
             fieldMapping(b -> {
                 b.field("type", "flattened");
                 b.field("index", false);
+                b.field("store_root", true);
             })
         ).documentMapper();
         ParsedDocument parsedDoc = mapper.parse(source(b -> b.startObject("field").field("key", "value").endObject()));
@@ -294,6 +323,7 @@ public void testDisableDocValues() throws Exception {
         DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
             b.field("type", "flattened");
             b.field("doc_values", false);
+            b.field("store_root", true);
         }));
         ParsedDocument parsedDoc = mapper.parse(source(b -> b.startObject("field").field("key", "value").endObject()));
 
@@ -359,6 +389,7 @@ public void testMixOfOrdinaryAndFlattenedFields() throws Exception {
                     .startObject("properties")
                     .startObject("field")
                     .field("type", "flattened")
+                    .field("store_root", true)
                     .endObject()
                     .startObject("a")
                     .field("type", "object")
@@ -471,6 +502,7 @@ public void testDepthLimit() throws IOException {
         // Set a lower value for depth_limit and check that the field is rejected.
         merge(mapperService, fieldMapping(b -> {
             b.field("type", "flattened");
+            b.field("store_root", true);
             b.field("depth_limit", 2);
         }));
 
@@ -668,11 +700,12 @@ public void testImmenseKeyedTermException() throws IOException {
 
     public void testNullValues() throws Exception {
         DocumentMapper mapper = createDocumentMapper(mapping(b -> {
-            b.startObject("field").field("type", "flattened").endObject();
+            b.startObject("field").field("type", "flattened").field("store_root", true).endObject();
             b.startObject("other_field");
             {
                 b.field("type", "flattened");
                 b.field("null_value", "placeholder");
+                b.field("store_root", true);
             }
             b.endObject();
         }));
@@ -732,6 +765,7 @@ public void testDynamicTemplateAndDottedPaths() throws IOException {
             b.field("match_mapping_type", "object");
             b.startObject("mapping");
             b.field("type", "flattened");
+            b.field("store_root", true);
             b.endObject();
             b.endObject();
             b.endObject();

server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParserTests.java

@@ -41,7 +41,8 @@ public void setUp() throws Exception {
             Integer.MAX_VALUE,
             Integer.MAX_VALUE,
             null,
-            false
+            false,
+            true
         );
     }
 
@@ -305,7 +306,8 @@ public void testDepthLimit() throws Exception {
             2,
             Integer.MAX_VALUE,
             null,
-            false
+            false,
+            true
         );
 
         TestDocumentParserContext context = new TestDocumentParserContext(xContentParser);
@@ -330,7 +332,8 @@ public void testDepthLimitBoundary() throws Exception {
             3,
             Integer.MAX_VALUE,
             null,
-            false
+            false,
+            true
         );
 
         TestDocumentParserContext context = new TestDocumentParserContext(xContentParser);
@@ -350,7 +353,8 @@ public void testIgnoreAbove() throws Exception {
             Integer.MAX_VALUE,
             10,
             null,
-            false
+            false,
+            true
         );
 
         TestDocumentParserContext context = new TestDocumentParserContext(xContentParser);
@@ -376,7 +380,8 @@ public void testNullValues() throws Exception {
             Integer.MAX_VALUE,
             Integer.MAX_VALUE,
             "placeholder",
-            false
+            false,
+            true
         );
 
         TestDocumentParserContext configuredContext = new TestDocumentParserContext(createXContentParser(input));