Improve storage footprint for text field under derived source and handle ig…

server/src/main/java/org/opensearch/index/mapper/CompositeFieldValueFetcher.java

@@ -0,0 +1,55 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.apache.lucene.index.LeafReader;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Composite fetcher that tries multiple sources and returns already-converted values from the available source
+ * with the highest priority
+ *
+ * @opensearch.internal
+ */
+public class CompositeFieldValueFetcher extends FieldValueFetcher {
+
+    private final List<FieldValueFetcher> fieldValueFetchers;
+
+    public CompositeFieldValueFetcher(String simpleName, List<FieldValueFetcher> fieldValueFetchers) {
+        super(simpleName);
+        this.fieldValueFetchers = fieldValueFetchers;
+    }
+
+    @Override
+    public List<Object> fetch(LeafReader reader, int docId) throws IOException {
+        // Try fetching values from various fetchers as per priority
+        for (final FieldValueFetcher fieldValueFetcher : fieldValueFetchers) {
+            List<Object> values = fieldValueFetcher.fetch(reader, docId);
+
+            // Convert values immediately after fetching
+            if (values != null && !values.isEmpty()) {
+                List<Object> convertedValues = new ArrayList<>(values.size());
+                for (Object value : values) {
+                    convertedValues.add(fieldValueFetcher.convert(value));
+                }
+                return convertedValues;
+            }
+        }
+        return null;
+    }
+
+    @Override
+    Object convert(Object value) {
+        // Values are already converted, return as-is
+        return value;
+    }
+}

server/src/main/java/org/opensearch/index/mapper/FieldValueFetcher.java

@@ -51,7 +51,7 @@ Object convert(Object value) {
      * @param builder - builder to store the field value(s) in
      */
     void write(XContentBuilder builder, List<Object> values) throws IOException {
-        if (values.isEmpty()) {
+        if (values == null || values.isEmpty()) {
             return;
         }
         if (values.size() == 1) {

server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java

@@ -37,6 +37,7 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BoostQuery;
@@ -271,10 +272,8 @@ public Optional<DimensionType> getSupportedDataCubeDimensionType() {
 
     @Override
     protected void canDeriveSourceInternal() {
-        if (this.ignoreAbove != Integer.MAX_VALUE || !Objects.equals(this.normalizerName, "default")) {
-            throw new UnsupportedOperationException(
-                "Unable to derive source for [" + name() + "] with " + "ignore_above and/or normalizer set"
-            );
+        if (!(fieldType().normalizer() == null || Lucene.KEYWORD_ANALYZER.equals(fieldType().normalizer()))) {
+            throw new UnsupportedOperationException("Unable to derive source for [" + name() + "] with " + "normalizer set");
         }
         checkStoredAndDocValuesForDerivedSource();
     }
@@ -295,11 +294,18 @@ protected void canDeriveSourceInternal() {
      */
     @Override
     protected DerivedFieldGenerator derivedFieldGenerator() {
-        return new DerivedFieldGenerator(
-            mappedFieldType,
-            new SortedSetDocValuesFetcher(mappedFieldType, simpleName()),
-            new StoredFieldFetcher(mappedFieldType, simpleName())
+        final FieldValueFetcher primaryFieldValueFetcher = KeywordFieldMapper.DerivedSourceHelper.getPrimaryFieldValueFetcher(this);
+        final FieldValueFetcher fallbackFieldValueFetcher = KeywordFieldMapper.DerivedSourceHelper.getFallbackFieldValueFetcher(this);
+        final FieldValueFetcher compositeFieldValueFetcher = new CompositeFieldValueFetcher(
+            simpleName(),
+            List.of(primaryFieldValueFetcher, fallbackFieldValueFetcher)
         );
+        return new DerivedFieldGenerator(mappedFieldType, compositeFieldValueFetcher, null) {
+            @Override
+            public FieldValueType getDerivedFieldPreference() {
+                return FieldValueType.DOC_VALUES;
+            }
+        };
     }
 
     /**
@@ -872,11 +878,21 @@ protected void parseCreateField(ParseContext context) throws IOException {
             }
         }
 
-        if (value == null || value.length() > ignoreAbove) {
+        if (value == null) {
             return;
         }
 
         NamedAnalyzer normalizer = fieldType().normalizer();
+
+        if (value.length() > ignoreAbove) {
+            if ((normalizer == null || Lucene.KEYWORD_ANALYZER.equals(normalizer))
+                && context.indexSettings().isDerivedSourceEnabled()
+                && context.isWithinMultiFields() == false) {
+                final BytesRef binaryValue = new BytesRef(value);
+                context.doc().add(new StoredField(fieldType().derivedSourceIgnoreFieldName(), binaryValue));
+            }
+            return;
+        }
         if (normalizer != null) {
             value = normalizeValue(normalizer, name(), value);
         }
@@ -936,4 +952,51 @@ protected String contentType() {
     public ParametrizedFieldMapper.Builder getMergeBuilder() {
         return new Builder(simpleName(), indexAnalyzers).init(this);
     }
+
+    private static final class DerivedSourceHelper {
+
+        private static FieldValueFetcher getPrimaryFieldValueFetcher(KeywordFieldMapper mapper) {
+            return mapper.fieldType().hasDocValues()
+                ? new SortedSetDocValuesFetcher(mapper.fieldType(), mapper.simpleName())
+                : new StoredFieldFetcher(mapper.fieldType(), mapper.simpleName());
+        }
+
+        private static FieldValueFetcher getFallbackFieldValueFetcher(KeywordFieldMapper mapper) {
+            // Override to read from the special ignored value field
+            final MappedFieldType ignoredFieldType = new MappedFieldType(
+                mapper.fieldType().derivedSourceIgnoreFieldName(),
+                false,  // not searchable
+                true,   // stored
+                false,  // no doc values
+                TextSearchInfo.NONE,
+                Collections.emptyMap()
+            ) {
+                @Override
+                public String typeName() {
+                    return "keyword";
+                }
+
+                @Override
+                public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) {
+                    return null;
+                }
+
+                @Override
+                public Query termQuery(Object value, QueryShardContext context) {
+                    return null;
+                }
+
+                @Override
+                public Object valueForDisplay(Object value) {
+                    if (value == null) {
+                        return null;
+                    }
+                    // keywords are internally stored as utf8 bytes
+                    BytesRef binaryValue = (BytesRef) value;
+                    return binaryValue.utf8ToString();
+                }
+            };
+            return new StoredFieldFetcher(ignoredFieldType, mapper.simpleName());
+        }
+    }
 }

server/src/main/java/org/opensearch/index/mapper/MatchOnlyTextFieldMapper.java

@@ -139,21 +139,22 @@ public Builder(String name, Version indexCreatedVersion, IndexAnalyzers indexAna
         @Override
         public MatchOnlyTextFieldMapper build(BuilderContext context) {
             FieldType fieldType = TextParams.buildFieldType(index, store, indexOptions, norms, termVectors);
-            MatchOnlyTextFieldType tft = buildFieldType(fieldType, context);
+            MultiFields multiFields = multiFieldsBuilder.build(this, context);
+            MatchOnlyTextFieldType tft = buildFieldType(fieldType, multiFields, context);
             return new MatchOnlyTextFieldMapper(
                 name,
                 fieldType,
                 tft,
                 buildPrefixMapper(context, fieldType, tft),
                 buildPhraseMapper(fieldType, tft),
-                multiFieldsBuilder.build(this, context),
+                multiFields,
                 copyTo.build(),
                 this
             );
         }
 
         @Override
-        protected MatchOnlyTextFieldType buildFieldType(FieldType fieldType, BuilderContext context) {
+        protected MatchOnlyTextFieldType buildFieldType(FieldType fieldType, MultiFields multiFields, BuilderContext context) {
             NamedAnalyzer indexAnalyzer = analyzers.getIndexAnalyzer();
             NamedAnalyzer searchAnalyzer = analyzers.getSearchAnalyzer();
             NamedAnalyzer searchQuoteAnalyzer = analyzers.getSearchQuoteAnalyzer();

server/src/main/java/org/opensearch/index/mapper/StringFieldType.java

@@ -66,6 +66,7 @@
  */
 public abstract class StringFieldType extends TermBasedFieldType {
 
+    private static final String IGNORED_VALUE_FIELD_SUFFIX = ".ignored_value";
     private static final Pattern WILDCARD_PATTERN = Pattern.compile("(\\\\.)|([?*]+)");
 
     public StringFieldType(
@@ -255,4 +256,8 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower
             includeUpper
         );
     }
+
+    public String derivedSourceIgnoreFieldName() {
+        return name() + IGNORED_VALUE_FIELD_SUFFIX;
+    }
 }