Use ignored source for text fields

Author: Kubik42

plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java

@@ -22,7 +22,6 @@
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.analysis.AnalyzerScope;
@@ -552,14 +551,14 @@ public Map<String, NamedAnalyzer> indexAnalyzers() {
 
     @Override
     protected void parseCreateField(DocumentParserContext context) throws IOException {
-        final var value = context.parser().optimizedTextOrNull();
+        final String value = context.parser().textOrNull();
 
         if (value == null) {
             return;
         }
 
         if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
-            Field field = new Field(mappedFieldType.name(), value.string(), fieldType);
+            Field field = new Field(mappedFieldType.name(), value, fieldType);
             context.doc().add(field);
             if (fieldType.omitNorms()) {
                 context.addToFieldNames(fieldType().name());
@@ -569,15 +568,13 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
             // else to support synthetic source
 
             // if we can rely on the synthetic source delegate for synthetic source, then return
-            if (fieldType().canUseSyntheticSourceDelegateForSyntheticSource(value.string())) {
+            if (fieldType().canUseSyntheticSourceDelegateForSyntheticSource(value)) {
                 return;
             }
 
             // otherwise, we need to store a copy of this value so that synthetic source can load it
-            var utfBytes = value.bytes();
-            var bytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length());
             final String fieldName = fieldType().syntheticSourceFallbackFieldName();
-            context.doc().add(new StoredField(fieldName, bytesRef));
+            context.doc().add(new StoredField(fieldName, value));
         }
     }
 

server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

@@ -21,7 +21,6 @@
 import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queries.intervals.Intervals;
@@ -1434,8 +1433,14 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
                 return;
             }
 
-            final String fieldName = fieldType().syntheticSourceFallbackFieldName();
-            context.doc().add(new StoredField(fieldName, value));
+            // record this field's value in _ignored_source - synthetic source will fallback to it automatically
+            var utfBytes = context.parser().optimizedTextOrNull().bytes();
+            var valuesBytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length());
+            context.addIgnoredField(
+                new IgnoredSourceFieldMapper.NameValue(fullPath(), fullPath().lastIndexOf(leafName()), valuesBytesRef, context.doc())
+            );
+            // final String fieldName = fieldType().syntheticSourceFallbackFieldName();
+            // context.doc().add(new StoredField(fieldName, value));
         }
     }
 
@@ -1603,6 +1608,7 @@ private boolean isIndexed() {
 
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport() {
+        // if we stored this field in Lucene, then use that for synthetic source
         if (store) {
             return new SyntheticSourceSupport.Native(() -> new StringStoredFieldFieldLoader(fullPath(), leafName()) {
                 @Override
@@ -1612,36 +1618,15 @@ protected void write(XContentBuilder b, Object value) throws IOException {
             });
         }
 
-        if (isIndexed()) {
-            return super.syntheticSourceSupport();
-        }
-
-        return new SyntheticSourceSupport.Native(() -> syntheticFieldLoader(fullPath(), leafName()));
-    }
-
-    private SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fullFieldName, String leafFieldName) {
-        // since we don't know whether the delegate field loader can be used for synthetic source until parsing, we
-        // need to check both this field and the delegate
-
-        // first field loader, representing this field
-        final String fieldName = fieldType().syntheticSourceFallbackFieldName();
-        final var thisFieldLayer = new CompositeSyntheticFieldLoader.StoredFieldLayer(fieldName) {
-            @Override
-            protected void writeValue(Object value, XContentBuilder b) throws IOException {
-                b.value(value.toString());
-            }
-        };
-
-        final CompositeSyntheticFieldLoader fieldLoader = new CompositeSyntheticFieldLoader(leafFieldName, fullFieldName, thisFieldLayer);
-
-        // second loader, representing a delegate field, if one exists
+        // otherwise, use the delegate
+        // note, the delegate itself might not be stored in Lucene due to various reasons (ex. it tripped ignore_above), in such cases, we
+        // should've added this field to ignored_source
         var kwd = TextFieldMapper.SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(this);
         if (kwd != null) {
-            // merge the two field loaders into one
-            return fieldLoader.mergedWith(kwd.syntheticFieldLoader(fullPath(), leafName()));
+            return new SyntheticSourceSupport.Native(() -> kwd.syntheticFieldLoader(fullPath(), leafName()));
         }
 
-        return fieldLoader;
+        return super.syntheticSourceSupport();
     }
 
     public static class SyntheticSourceHelper {