Reverted ignore source changes

Author: Kubik42

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java

@@ -13,6 +13,7 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
@@ -682,11 +683,13 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
             // if the delegate can't support synthetic source for the given value, then store a copy of said value so
             // that synthetic source can load it
             if (fieldType().textFieldType.canUseSyntheticSourceDelegateForSyntheticSource(value.string()) == false) {
+                final String fieldName = fieldType().syntheticSourceFallbackFieldName();
                 if (storedFieldInBinaryFormat) {
                     final var bytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length());
-                    context.storeFieldForSyntheticSource(fullPath(), leafName(), bytesRef, context.doc());
+                    context.doc().add(new StoredField(fieldName, bytesRef));
                 } else {
-                    context.storeFieldForSyntheticSource(fullPath(), leafName(), context.encodeFlattenedToken(), context.doc());
+
+                    context.doc().add(new StoredField(fieldName, value.string()));
                 }
             }
         }
@@ -704,13 +707,7 @@ public MatchOnlyTextFieldType fieldType() {
 
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport() {
-        var kwd = TextFieldMapper.SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(this);
-        if (kwd != null) {
-            // merge the two field loaders into one
-            return new SyntheticSourceSupport.Native(() -> kwd.syntheticFieldLoader(fullPath(), leafName()));
-        }
-
-        return super.syntheticSourceSupport();
+        return new SyntheticSourceSupport.Native(() -> syntheticFieldLoader(fullPath(), leafName()));
     }
 
     private SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fullFieldName, String leafFieldName) {

server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java

@@ -385,7 +385,7 @@ public final DocumentParserContext addIgnoredFieldFromContext(IgnoredSourceField
      * For instance: { "a.b": "b", "a.c": "c" } => { "a": { "b": "b" }, "a": { "c": "c" } }
      * This can happen when storing parts of document source that are not indexed (e.g. disabled objects).
      */
-    public BytesRef encodeFlattenedToken() throws IOException {
+    BytesRef encodeFlattenedToken() throws IOException {
         boolean old = path().isWithinLeafObject();
         path().setWithinLeafObject(true);
         BytesRef encoded = XContentDataHelper.encodeToken(parser());

server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

@@ -21,6 +21,7 @@
 import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queries.intervals.Intervals;
@@ -1459,7 +1460,8 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
             }
 
             // otherwise, store this field in Lucene so that synthetic source can load it
-            context.storeFieldForSyntheticSource(fullPath(), leafName(), context.encodeFlattenedToken(), context.doc());
+            final String fieldName = fieldType().syntheticSourceFallbackFieldName();
+            context.doc().add(new StoredField(fieldName, value));
         }
     }
 
@@ -1637,15 +1639,32 @@ protected void write(XContentBuilder b, Object value) throws IOException {
             });
         }
 
-        // otherwise, use the delegate
-        // note, the delegate itself might not be stored in Lucene due to various reasons (ex. it tripped ignore_above), in such cases, we
-        // should've added this field to ignored_source
+        return new SyntheticSourceSupport.Native(() -> syntheticFieldLoader(fullPath(), leafName()));
+    }
+
+    private SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fullFieldName, String leafFieldName) {
+        // since we don't know whether the delegate field loader can be used for synthetic source until parsing, we
+        // need to check both this field and the delegate
+
+        // first field loader, representing this field
+        final String fieldName = fieldType().syntheticSourceFallbackFieldName();
+        final var thisFieldLayer = new CompositeSyntheticFieldLoader.StoredFieldLayer(fieldName) {
+            @Override
+            protected void writeValue(Object value, XContentBuilder b) throws IOException {
+                b.value(value.toString());
+            }
+        };
+
+        final CompositeSyntheticFieldLoader fieldLoader = new CompositeSyntheticFieldLoader(leafFieldName, fullFieldName, thisFieldLayer);
+
+        // second loader, representing a delegate field, if one exists
         var kwd = TextFieldMapper.SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(this);
         if (kwd != null) {
-            return new SyntheticSourceSupport.Native(() -> kwd.syntheticFieldLoader(fullPath(), leafName()));
+            // merge the two field loaders into one
+            return fieldLoader.mergedWith(kwd.syntheticFieldLoader(fullPath(), leafName()));
         }
 
-        return super.syntheticSourceSupport();
+        return fieldLoader;
     }
 
     public static class SyntheticSourceHelper {