Add index version for match_only_text stored field in binary format

Author: jordan-powers

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java

@@ -35,6 +35,7 @@
 import org.elasticsearch.common.text.UTF8DecodingReader;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.FieldDataContext;
@@ -105,8 +106,15 @@ public static class Builder extends FieldMapper.Builder {
 
         private final TextParams.Analyzers analyzers;
         private final boolean withinMultiField;
+        private final boolean storedFieldInBinaryFormat;
 
-        public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers, boolean withinMultiField) {
+        public Builder(
+            String name,
+            IndexVersion indexCreatedVersion,
+            IndexAnalyzers indexAnalyzers,
+            boolean withinMultiField,
+            boolean storedFieldInBinaryFormat
+        ) {
             super(name);
             this.indexCreatedVersion = indexCreatedVersion;
             this.analyzers = new TextParams.Analyzers(
@@ -116,6 +124,7 @@ public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers ind
                 indexCreatedVersion
             );
             this.withinMultiField = withinMultiField;
+            this.storedFieldInBinaryFormat = storedFieldInBinaryFormat;
         }
 
         @Override
@@ -135,7 +144,8 @@ private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context) {
                 context.isSourceSynthetic(),
                 meta.getValue(),
                 withinMultiField,
-                multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField()
+                multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField(),
+                storedFieldInBinaryFormat
             );
             return ft;
         }
@@ -156,7 +166,13 @@ public MatchOnlyTextFieldMapper build(MapperBuilderContext context) {
     }
 
     public static final TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), c.isWithinMultiField())
+        (n, c) -> new Builder(
+            n,
+            c.indexVersionCreated(),
+            c.getIndexAnalyzers(),
+            c.isWithinMultiField(),
+            c.indexVersionCreated().onOrAfter(IndexVersions.MATCH_ONLY_TEXT_STORED_AS_BYTES)
+        )
     );
 
     public static class MatchOnlyTextFieldType extends StringFieldType {
@@ -167,6 +183,7 @@ public static class MatchOnlyTextFieldType extends StringFieldType {
 
         private final boolean withinMultiField;
         private final boolean hasCompatibleMultiFields;
+        private final boolean storedFieldInBinaryFormat;
 
         public MatchOnlyTextFieldType(
             String name,
@@ -175,14 +192,16 @@ public MatchOnlyTextFieldType(
             boolean isSyntheticSource,
             Map<String, String> meta,
             boolean withinMultiField,
-            boolean hasCompatibleMultiFields
+            boolean hasCompatibleMultiFields,
+            boolean storedFieldInBinaryFormat
         ) {
             super(name, true, false, false, tsi, meta);
             this.indexAnalyzer = Objects.requireNonNull(indexAnalyzer);
             this.textFieldType = new TextFieldType(name, isSyntheticSource);
             this.originalName = isSyntheticSource ? name + "._original" : null;
             this.withinMultiField = withinMultiField;
             this.hasCompatibleMultiFields = hasCompatibleMultiFields;
+            this.storedFieldInBinaryFormat = storedFieldInBinaryFormat;
         }
 
         public MatchOnlyTextFieldType(String name) {
@@ -193,6 +212,7 @@ public MatchOnlyTextFieldType(String name) {
                 false,
                 Collections.emptyMap(),
                 false,
+                false,
                 false
             );
         }
@@ -451,7 +471,11 @@ protected BytesRef toBytesRef(Object v) {
         @Override
         public BlockLoader blockLoader(BlockLoaderContext blContext) {
             if (textFieldType.isSyntheticSource()) {
-                return new BytesFromMixedStringsBytesRefBlockLoader(storedFieldNameForSyntheticSource());
+                if (storedFieldInBinaryFormat) {
+                    return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(storedFieldNameForSyntheticSource());
+                } else {
+                    return new BytesFromMixedStringsBytesRefBlockLoader(storedFieldNameForSyntheticSource());
+                }
             }
             SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()));
             // MatchOnlyText never has norms, so we have to use the field names field
@@ -502,6 +526,7 @@ private String storedFieldNameForSyntheticSource() {
     private final boolean storeSource;
     private final FieldType fieldType;
     private final boolean withinMultiField;
+    private final boolean storedFieldInBinaryFormat;
 
     private MatchOnlyTextFieldMapper(
         String simpleName,
@@ -521,6 +546,7 @@ private MatchOnlyTextFieldMapper(
         this.positionIncrementGap = builder.analyzers.positionIncrementGap.getValue();
         this.storeSource = storeSource;
         this.withinMultiField = builder.withinMultiField;
+        this.storedFieldInBinaryFormat = builder.storedFieldInBinaryFormat;
     }
 
     @Override
@@ -530,7 +556,7 @@ public Map<String, NamedAnalyzer> indexAnalyzers() {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, withinMultiField).init(this);
+        return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, withinMultiField, storedFieldInBinaryFormat).init(this);
     }
 
     @Override
@@ -547,8 +573,12 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
         context.addToFieldNames(fieldType().name());
 
         if (storeSource) {
-            final var bytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length());
-            context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), bytesRef));
+            if (storedFieldInBinaryFormat) {
+                final var bytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length());
+                context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), bytesRef));
+            } else {
+                context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), value.string()));
+            }
         }
     }
 

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -178,6 +178,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion UPGRADE_TO_LUCENE_10_2_2 = def(9_030_0_00, Version.LUCENE_10_2_2);
     public static final IndexVersion SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT = def(9_031_0_00, Version.LUCENE_10_2_2);
     public static final IndexVersion DEFAULT_DENSE_VECTOR_TO_BBQ_HNSW = def(9_032_0_00, Version.LUCENE_10_2_2);
+    public static final IndexVersion MATCH_ONLY_TEXT_STORED_AS_BYTES = def(9_033_0_00, Version.LUCENE_10_2_2);
 
     /*
      * STOP! READ THIS FIRST! No, really,