Skip to content

Commit bc5d50b

Browse files
Kubik42Kubik42
committed
bwc checks
1 parent e7d380b commit bc5d50b

File tree

8 files changed

+297
-6
lines changed

8 files changed

+297
-6
lines changed

‎plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,13 +87,13 @@ public static class Builder extends FieldMapper.Builder {
8787
final Parameter<String> indexOptions = TextParams.textIndexOptions(m -> builder(m).indexOptions.getValue());
8888
final Parameter<Boolean> norms = Parameter.normsParam(m -> builder(m).norms.getValue(), true);
8989
final Parameter<String> termVectors = TextParams.termVectors(m -> builder(m).termVectors.getValue());
90-
private final Parameter<Boolean> store = Parameter.storeParam(m -> builder(m).store.getValue(), false);
9190

9291
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
9392

9493
private final IndexVersion indexCreatedVersion;
9594
private final TextParams.Analyzers analyzers;
9695
private final boolean isWithinMultiField;
96+
private final Parameter<Boolean> store;
9797

9898
private boolean isSyntheticSourceEnabled;
9999

@@ -107,6 +107,12 @@ public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers ind
107107
m -> builder(m).analyzers.positionIncrementGap.getValue(),
108108
indexCreatedVersion
109109
);
110+
this.store = Parameter.storeParam(m -> builder(m).store.getValue(), () -> {
111+
if (keywordMultiFieldsNotStoredWhenIgnored_indexVersionCheck(indexCreatedVersion)) {
112+
return false;
113+
}
114+
return isSyntheticSourceEnabled && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false;
115+
});
110116
}
111117

112118
@Override

‎plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@
2929
import org.elasticsearch.common.Strings;
3030
import org.elasticsearch.index.IndexMode;
3131
import org.elasticsearch.index.IndexSettings;
32+
import org.elasticsearch.index.IndexVersion;
33+
import org.elasticsearch.index.IndexVersions;
3234
import org.elasticsearch.index.analysis.AnalyzerScope;
3335
import org.elasticsearch.index.analysis.CharFilterFactory;
3436
import org.elasticsearch.index.analysis.CustomAnalyzer;
@@ -291,7 +293,19 @@ public void testEnableStore() throws IOException {
291293
assertTrue(fields.get(0).fieldType().stored());
292294
}
293295

294-
public void testStoreParameterDefaults() throws IOException {
296+
public void test_store_parameter_defaults_to_false_in_latest_index_version_when_synthetic_source_is_enabled() throws IOException {
297+
// given
298+
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { b.field("type", "annotated_text"); }));
299+
300+
// when
301+
ParsedDocument doc = mapper.parse(source(b -> b.field("field", "1234")));
302+
303+
// then
304+
List<IndexableField> fields = doc.rootDoc().getFields("field");
305+
assertFalse(fields.getFirst().fieldType().stored());
306+
}
307+
308+
public void testStoreParameterDefaultsBwc() throws IOException {
295309
var timeSeriesIndexMode = randomBoolean();
296310
var isStored = randomBoolean();
297311
var hasKeywordFieldForSyntheticSource = randomBoolean();
@@ -330,7 +344,9 @@ public void testStoreParameterDefaults() throws IOException {
330344
b.endObject();
331345
}
332346
});
333-
DocumentMapper mapper = createMapperService(getVersion(), indexSettings, () -> true, mapping).documentMapper();
347+
348+
IndexVersion bwcIndexVersion = IndexVersions.MAPPER_TEXT_MATCH_ONLY_MULTI_FIELDS_DEFAULT_NOT_STORED;
349+
DocumentMapper mapper = createMapperService(bwcIndexVersion, indexSettings, () -> true, mapping).documentMapper();
334350

335351
var source = source(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, b -> {
336352
b.field("field", "1234");

‎server/src/main/java/org/elasticsearch/index/IndexVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,7 @@ private static Version parseUnchecked(String version) {
182182
public static final IndexVersion MATCH_ONLY_TEXT_STORED_AS_BYTES = def(9_033_0_00, Version.LUCENE_10_2_2);
183183
public static final IndexVersion IGNORED_SOURCE_FIELDS_PER_ENTRY_WITH_FF = def(9_034_0_00, Version.LUCENE_10_2_2);
184184
public static final IndexVersion EXCLUDE_SOURCE_VECTORS_DEFAULT = def(9_035_0_00, Version.LUCENE_10_2_2);
185+
public static final IndexVersion KEYWORD_MULTI_FIELDS_NOT_STORED_WHEN_IGNORED = def(9_036_0_00, Version.LUCENE_10_2_2);
185186

186187
/*
187188
* STOP! READ THIS FIRST! No, really,

‎server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -610,13 +610,28 @@ public static class Builder {
610610

611611
private final Map<String, Function<MapperBuilderContext, FieldMapper>> mapperBuilders = new HashMap<>();
612612

613+
private boolean hasSyntheticSourceCompatibleKeywordField;
614+
613615
public Builder add(FieldMapper.Builder builder) {
614616
mapperBuilders.put(builder.leafName(), builder::build);
617+
618+
if (builder instanceof KeywordFieldMapper.Builder kwd) {
619+
if (kwd.hasNormalizer() == false && (kwd.hasDocValues() || kwd.isStored())) {
620+
hasSyntheticSourceCompatibleKeywordField = true;
621+
}
622+
}
623+
615624
return this;
616625
}
617626

618627
private void add(FieldMapper mapper) {
619628
mapperBuilders.put(mapper.leafName(), context -> mapper);
629+
630+
if (mapper instanceof KeywordFieldMapper kwd) {
631+
if (kwd.hasNormalizer() == false && (kwd.fieldType().hasDocValues() || kwd.fieldType().isStored())) {
632+
hasSyntheticSourceCompatibleKeywordField = true;
633+
}
634+
}
620635
}
621636

622637
private void update(FieldMapper toMerge, MapperMergeContext context) {
@@ -634,6 +649,10 @@ public boolean hasMultiFields() {
634649
return mapperBuilders.isEmpty() == false;
635650
}
636651

652+
public boolean hasSyntheticSourceCompatibleKeywordField() {
653+
return hasSyntheticSourceCompatibleKeywordField;
654+
}
655+
637656
public MultiFields build(Mapper.Builder mainFieldBuilder, MapperBuilderContext context) {
638657
if (mapperBuilders.isEmpty()) {
639658
return empty();

‎server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldMapper.java‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,19 +40,34 @@ protected TextFamilyFieldMapper(
4040
* Returns whether this field mapper needs to support synthetic source.
4141
*/
4242
protected boolean needsToSupportSyntheticSource() {
43-
if (multiFieldsNotStoredByDefaultIndexVersionCheck()) {
43+
if (multiFieldsNotStoredByDefault_indexVersionCheck()) {
4444
// if we're within a multi field, then supporting synthetic source isn't necessary as that's the responsibility of the parent
4545
return isSyntheticSourceEnabled && isWithinMultiField == false;
4646
}
4747
return isSyntheticSourceEnabled;
4848
}
4949

50-
private boolean multiFieldsNotStoredByDefaultIndexVersionCheck() {
50+
private boolean multiFieldsNotStoredByDefault_indexVersionCheck() {
51+
return TextFamilyFieldMapper.multiFieldsNotStoredByDefault_indexVersionCheck(indexCreatedVersion);
52+
}
53+
54+
/**
55+
* Returns whether the current index version supports not storing fields by default when they're multi fields.
56+
*/
57+
protected static boolean multiFieldsNotStoredByDefault_indexVersionCheck(final IndexVersion indexCreatedVersion) {
5158
return indexCreatedVersion.onOrAfter(IndexVersions.MAPPER_TEXT_MATCH_ONLY_MULTI_FIELDS_DEFAULT_NOT_STORED)
5259
|| indexCreatedVersion.between(
5360
IndexVersions.MAPPER_TEXT_MATCH_ONLY_MULTI_FIELDS_DEFAULT_NOT_STORED_8_19,
5461
IndexVersions.UPGRADE_TO_LUCENE_10_0_0
5562
);
5663
}
5764

65+
/**
66+
* Returns whether the current index version supports not storing keyword multi fields when they trip ignore_above. The consequence
67+
* of this check is that the store parameter will be simplified and defaulted to false.
68+
*/
69+
protected static boolean keywordMultiFieldsNotStoredWhenIgnored_indexVersionCheck(final IndexVersion indexCreatedVersion) {
70+
return indexCreatedVersion.onOrAfter(IndexVersions.KEYWORD_MULTI_FIELDS_NOT_STORED_WHEN_IGNORED);
71+
}
72+
5873
}

‎server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,7 @@ public static class Builder extends FieldMapper.Builder {
245245
private final IndexMode indexMode;
246246

247247
private final Parameter<Boolean> index = Parameter.indexParam(m -> ((TextFieldMapper) m).index, true);
248-
private final Parameter<Boolean> store = Parameter.storeParam(m -> ((TextFieldMapper) m).store, false);
248+
private Parameter<Boolean> store = Parameter.storeParam(m -> ((TextFieldMapper) m).store, false);
249249

250250
final Parameter<SimilarityProvider> similarity = TextParams.similarity(m -> ((TextFieldMapper) m).similarity);
251251

@@ -471,6 +471,19 @@ private SubFieldInfo buildPhraseInfo(FieldType fieldType, TextFieldType parent)
471471
public TextFieldMapper build(MapperBuilderContext context) {
472472
this.isSyntheticSourceEnabled = context.isSourceSynthetic();
473473

474+
// backwards compatibility checks
475+
if (keywordMultiFieldsNotStoredWhenIgnored_indexVersionCheck(indexCreatedVersion) == false) {
476+
this.store = Parameter.storeParam(m -> ((TextFieldMapper) m).store, () -> {
477+
if (multiFieldsNotStoredByDefault_indexVersionCheck(indexCreatedVersion)) {
478+
return isSyntheticSourceEnabled
479+
&& isWithinMultiField == false
480+
&& multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false;
481+
} else {
482+
return isSyntheticSourceEnabled && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false;
483+
}
484+
});
485+
}
486+
474487
FieldType fieldType = TextParams.buildFieldType(
475488
index,
476489
store,

‎server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper;
11+
12+
import org.elasticsearch.common.lucene.Lucene;
13+
import org.elasticsearch.index.IndexVersion;
14+
import org.elasticsearch.index.analysis.IndexAnalyzers;
15+
import org.elasticsearch.script.ScriptCompiler;
16+
import org.elasticsearch.test.ESTestCase;
17+
18+
import java.util.Map;
19+
20+
import static org.elasticsearch.index.mapper.MapperService.MergeReason.MAPPING_UPDATE;
21+
22+
public class MultiFieldsTests extends ESTestCase {
23+
24+
public void testMultiFieldsBuilderHasSyntheticSourceCompatibleKeywordField() {
25+
var isStored = randomBoolean();
26+
var hasNormalizer = randomBoolean();
27+
28+
var builder = new FieldMapper.MultiFields.Builder();
29+
assertFalse(builder.hasSyntheticSourceCompatibleKeywordField());
30+
31+
var keywordFieldMapperBuilder = getKeywordFieldMapperBuilder(isStored, hasNormalizer);
32+
builder.add(keywordFieldMapperBuilder);
33+
34+
var expected = hasNormalizer == false;
35+
assertEquals(expected, builder.hasSyntheticSourceCompatibleKeywordField());
36+
}
37+
38+
public void testMultiFieldsBuilderHasSyntheticSourceCompatibleKeywordFieldDuringMerge() {
39+
var isStored = randomBoolean();
40+
var hasNormalizer = randomBoolean();
41+
42+
var builder = new TextFieldMapper.Builder("text_field", createDefaultIndexAnalyzers());
43+
assertFalse(builder.multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField());
44+
45+
var keywordFieldMapperBuilder = getKeywordFieldMapperBuilder(isStored, hasNormalizer);
46+
47+
var newField = new TextFieldMapper.Builder("text_field", createDefaultIndexAnalyzers()).addMultiField(keywordFieldMapperBuilder)
48+
.build(MapperBuilderContext.root(false, false));
49+
50+
builder.merge(
51+
newField,
52+
new FieldMapper.Conflicts("TextFieldMapper"),
53+
MapperMergeContext.root(false, false, MAPPING_UPDATE, Long.MAX_VALUE)
54+
);
55+
56+
var expected = hasNormalizer == false;
57+
assertEquals(expected, builder.multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField());
58+
}
59+
60+
private KeywordFieldMapper.Builder getKeywordFieldMapperBuilder(boolean isStored, boolean hasNormalizer) {
61+
var keywordFieldMapperBuilder = new KeywordFieldMapper.Builder(
62+
"field",
63+
IndexAnalyzers.of(Map.of(), Map.of("normalizer", Lucene.STANDARD_ANALYZER), Map.of()),
64+
ScriptCompiler.NONE,
65+
Integer.MAX_VALUE,
66+
IndexVersion.current(),
67+
Mapper.SourceKeepMode.NONE,
68+
false
69+
);
70+
if (isStored) {
71+
keywordFieldMapperBuilder.stored(true);
72+
if (randomBoolean()) {
73+
keywordFieldMapperBuilder.docValues(false);
74+
}
75+
}
76+
if (hasNormalizer) {
77+
keywordFieldMapperBuilder.normalizer("normalizer");
78+
}
79+
return keywordFieldMapperBuilder;
80+
}
81+
}

0 commit comments

Comments
 (0)