Skip to content

Commit 4ff2157

Browse files
jordan-powersjordan-powers
authored
[8.19] Add index version for match_only_text stored field in binary format (elastic#130363) (elastic#130416)
* Add index version for match_only_text stored field in binary format (elastic#130363) Follow-up to elastic#130049 to gate using the binary format for the stored field in match_only_text fields behind an index version. (cherry picked from commit a69c484) # Conflicts: # server/src/main/java/org/elasticsearch/index/IndexVersions.java * Fix IndexVersion in testLoadSyntheticSourceFromStringOrBytesRef * Trigger build for auto-merge
1 parent bf4d88f commit 4ff2157

File tree

3 files changed

+50
-10
lines changed

3 files changed

+50
-10
lines changed

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java‎

Lines changed: 41 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -104,8 +104,15 @@ public static class Builder extends FieldMapper.Builder {
104104

105105
private final TextParams.Analyzers analyzers;
106106
private final boolean withinMultiField;
107+
private final boolean storedFieldInBinaryFormat;
107108

108-
public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers, boolean withinMultiField) {
109+
public Builder(
110+
String name,
111+
IndexVersion indexCreatedVersion,
112+
IndexAnalyzers indexAnalyzers,
113+
boolean withinMultiField,
114+
boolean storedFieldInBinaryFormat
115+
) {
109116
super(name);
110117
this.indexCreatedVersion = indexCreatedVersion;
111118
this.analyzers = new TextParams.Analyzers(
@@ -115,6 +122,7 @@ public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers ind
115122
indexCreatedVersion
116123
);
117124
this.withinMultiField = withinMultiField;
125+
this.storedFieldInBinaryFormat = storedFieldInBinaryFormat;
118126
}
119127

120128
@Override
@@ -134,7 +142,8 @@ private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context) {
134142
context.isSourceSynthetic(),
135143
meta.getValue(),
136144
withinMultiField,
137-
multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField()
145+
multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField(),
146+
storedFieldInBinaryFormat
138147
);
139148
return ft;
140149
}
@@ -154,8 +163,18 @@ public MatchOnlyTextFieldMapper build(MapperBuilderContext context) {
154163
}
155164
}
156165

166+
private static boolean isSyntheticSourceStoredFieldInBinaryFormat(IndexVersion indexCreatedVersion) {
167+
return indexCreatedVersion.onOrAfter(IndexVersions.MATCH_ONLY_TEXT_STORED_AS_BYTES_BACKPORT_8_X);
168+
}
169+
157170
public static final TypeParser PARSER = new TypeParser(
158-
(n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), c.isWithinMultiField())
171+
(n, c) -> new Builder(
172+
n,
173+
c.indexVersionCreated(),
174+
c.getIndexAnalyzers(),
175+
c.isWithinMultiField(),
176+
isSyntheticSourceStoredFieldInBinaryFormat(c.indexVersionCreated())
177+
)
159178
);
160179

161180
public static class MatchOnlyTextFieldType extends StringFieldType {
@@ -166,6 +185,7 @@ public static class MatchOnlyTextFieldType extends StringFieldType {
166185

167186
private final boolean withinMultiField;
168187
private final boolean hasCompatibleMultiFields;
188+
private final boolean storedFieldInBinaryFormat;
169189

170190
public MatchOnlyTextFieldType(
171191
String name,
@@ -174,14 +194,16 @@ public MatchOnlyTextFieldType(
174194
boolean isSyntheticSource,
175195
Map<String, String> meta,
176196
boolean withinMultiField,
177-
boolean hasCompatibleMultiFields
197+
boolean hasCompatibleMultiFields,
198+
boolean storedFieldInBinaryFormat
178199
) {
179200
super(name, true, false, false, tsi, meta);
180201
this.indexAnalyzer = Objects.requireNonNull(indexAnalyzer);
181202
this.textFieldType = new TextFieldType(name, isSyntheticSource);
182203
this.originalName = isSyntheticSource ? name() + "._original" : null;
183204
this.withinMultiField = withinMultiField;
184205
this.hasCompatibleMultiFields = hasCompatibleMultiFields;
206+
this.storedFieldInBinaryFormat = storedFieldInBinaryFormat;
185207
}
186208

187209
public MatchOnlyTextFieldType(String name) {
@@ -192,6 +214,7 @@ public MatchOnlyTextFieldType(String name) {
192214
false,
193215
Collections.emptyMap(),
194216
false,
217+
false,
195218
false
196219
);
197220
}
@@ -450,7 +473,11 @@ protected BytesRef toBytesRef(Object v) {
450473
@Override
451474
public BlockLoader blockLoader(BlockLoaderContext blContext) {
452475
if (textFieldType.isSyntheticSource()) {
453-
return new BytesFromMixedStringsBytesRefBlockLoader(storedFieldNameForSyntheticSource());
476+
if (storedFieldInBinaryFormat) {
477+
return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(storedFieldNameForSyntheticSource());
478+
} else {
479+
return new BytesFromMixedStringsBytesRefBlockLoader(storedFieldNameForSyntheticSource());
480+
}
454481
}
455482
SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()));
456483
// MatchOnlyText never has norms, so we have to use the field names field
@@ -501,6 +528,7 @@ private String storedFieldNameForSyntheticSource() {
501528
private final boolean storeSource;
502529
private final FieldType fieldType;
503530
private final boolean withinMultiField;
531+
private final boolean storedFieldInBinaryFormat;
504532

505533
private MatchOnlyTextFieldMapper(
506534
String simpleName,
@@ -520,6 +548,7 @@ private MatchOnlyTextFieldMapper(
520548
this.positionIncrementGap = builder.analyzers.positionIncrementGap.getValue();
521549
this.storeSource = storeSource;
522550
this.withinMultiField = builder.withinMultiField;
551+
this.storedFieldInBinaryFormat = builder.storedFieldInBinaryFormat;
523552
}
524553

525554
@Override
@@ -529,7 +558,7 @@ public Map<String, NamedAnalyzer> indexAnalyzers() {
529558

530559
@Override
531560
public FieldMapper.Builder getMergeBuilder() {
532-
return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, withinMultiField).init(this);
561+
return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, withinMultiField, storedFieldInBinaryFormat).init(this);
533562
}
534563

535564
@Override
@@ -546,8 +575,12 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
546575
context.addToFieldNames(fieldType().name());
547576

548577
if (storeSource) {
549-
final var bytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length());
550-
context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), bytesRef));
578+
if (storedFieldInBinaryFormat) {
579+
final var bytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length());
580+
context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), bytesRef));
581+
} else {
582+
context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), value.string()));
583+
}
551584
}
552585
}
553586

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,10 @@
2626
import org.apache.lucene.tests.index.RandomIndexWriter;
2727
import org.apache.lucene.util.BytesRef;
2828
import org.elasticsearch.common.Strings;
29+
import org.elasticsearch.common.settings.Settings;
2930
import org.elasticsearch.core.Tuple;
3031
import org.elasticsearch.index.IndexSettings;
32+
import org.elasticsearch.index.IndexVersions;
3133
import org.elasticsearch.index.mapper.DocumentMapper;
3234
import org.elasticsearch.index.mapper.KeywordFieldMapper;
3335
import org.elasticsearch.index.mapper.LuceneDocument;
@@ -356,10 +358,14 @@ public void testStoreParameterDefaultsSyntheticSourceTextFieldIsMultiField() thr
356358
}
357359

358360
public void testLoadSyntheticSourceFromStringOrBytesRef() throws IOException {
359-
DocumentMapper mapper = createSytheticSourceMapperService(mapping(b -> {
361+
var mappings = mapping(b -> {
360362
b.startObject("field1").field("type", "match_only_text").endObject();
361363
b.startObject("field2").field("type", "match_only_text").endObject();
362-
})).documentMapper();
364+
});
365+
var settings = Settings.builder().put("index.mapping.source.mode", "synthetic").build();
366+
DocumentMapper mapper = createMapperService(IndexVersions.UPGRADE_TO_LUCENE_9_12_2, settings, () -> true, mappings)
367+
.documentMapper();
368+
363369
try (Directory directory = newDirectory()) {
364370
RandomIndexWriter iw = indexWriterForSyntheticSource(directory);
365371

‎server/src/main/java/org/elasticsearch/index/IndexVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ private static IndexVersion def(int id, Version luceneVersion) {
136136
public static final IndexVersion MAPPER_TEXT_MATCH_ONLY_MULTI_FIELDS_DEFAULT_NOT_STORED_8_19 = def(8_533_0_00, Version.LUCENE_9_12_1);
137137
public static final IndexVersion UPGRADE_TO_LUCENE_9_12_2 = def(8_534_0_00, Version.LUCENE_9_12_2);
138138
public static final IndexVersion SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT_BACKPORT_8_X = def(8_535_0_00, Version.LUCENE_9_12_2);
139+
public static final IndexVersion MATCH_ONLY_TEXT_STORED_AS_BYTES_BACKPORT_8_X = def(8_536_0_00, Version.LUCENE_9_12_2);
139140

140141
/*
141142
* STOP! READ THIS FIRST! No, really,

0 commit comments

Comments
 (0)