Skip to content

Commit 4e3c548

Browse files
romseygeekromseygeek
committed
WIP: TextFieldMapper failures
1 parent cad52e7 commit 4e3c548

File tree

2 files changed

+152
-98
lines changed

2 files changed

+152
-98
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java‎

Lines changed: 35 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,8 @@ public static class Builder extends FieldMapper.Builder {
274274

275275
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
276276

277-
final TextParams.Analyzers analyzers;
277+
final TextParams.AnalyzerParameters analyzers;
278+
final IndexAnalyzers indexAnalyzers;
278279

279280
public Builder(String name, IndexAnalyzers indexAnalyzers) {
280281
this(name, Version.CURRENT, indexAnalyzers);
@@ -283,10 +284,9 @@ public Builder(String name, IndexAnalyzers indexAnalyzers) {
283284
public Builder(String name, Version indexCreatedVersion, IndexAnalyzers indexAnalyzers) {
284285
super(name);
285286
this.indexCreatedVersion = indexCreatedVersion;
286-
this.analyzers = new TextParams.Analyzers(
287-
indexAnalyzers,
288-
m -> ((TextFieldMapper) m).indexAnalyzer,
289-
m -> (((TextFieldMapper) m).positionIncrementGap),
287+
this.indexAnalyzers = indexAnalyzers;
288+
this.analyzers = new TextParams.AnalyzerParameters(
289+
m -> ((TextFieldMapper) m).analyzerConfiguration,
290290
indexCreatedVersion
291291
);
292292
}
@@ -341,17 +341,11 @@ private TextFieldType buildFieldType(
341341
FieldType fieldType,
342342
MultiFields multiFields,
343343
MapperBuilderContext context,
344-
Version indexCreatedVersion
344+
Version indexCreatedVersion,
345+
TextParams.Analyzers analyzers
345346
) {
346-
NamedAnalyzer searchAnalyzer = analyzers.getSearchAnalyzer();
347-
NamedAnalyzer searchQuoteAnalyzer = analyzers.getSearchQuoteAnalyzer();
348-
if (analyzers.positionIncrementGap.isConfigured()) {
349-
if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
350-
throw new IllegalArgumentException(
351-
"Cannot set position_increment_gap on field [" + name + "] without positions enabled"
352-
);
353-
}
354-
}
347+
NamedAnalyzer searchAnalyzer = analyzers.searchAnalyzer();
348+
NamedAnalyzer searchQuoteAnalyzer = analyzers.searchQuoteAnalyzer();
355349
TextSearchInfo tsi = new TextSearchInfo(fieldType, similarity.getValue(), searchAnalyzer, searchQuoteAnalyzer);
356350
TextFieldType ft;
357351
if (indexCreatedVersion.isLegacyIndexVersion()) {
@@ -390,7 +384,7 @@ private KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate(FieldType fi
390384
return null;
391385
}
392386

393-
private SubFieldInfo buildPrefixInfo(MapperBuilderContext context, FieldType fieldType, TextFieldType tft) {
387+
private SubFieldInfo buildPrefixInfo(MapperBuilderContext context, TextParams.Analyzers analyzers, FieldType fieldType, TextFieldType tft) {
394388
if (indexPrefixes.get() == null) {
395389
return null;
396390
}
@@ -423,15 +417,15 @@ private SubFieldInfo buildPrefixInfo(MapperBuilderContext context, FieldType fie
423417
fullName + "._index_prefix",
424418
pft,
425419
new PrefixWrappedAnalyzer(
426-
analyzers.getIndexAnalyzer().analyzer(),
427-
analyzers.positionIncrementGap.get(),
420+
analyzers.indexAnalyzer().analyzer(),
421+
analyzers.configuration().posIncrementGap(),
428422
indexPrefixes.get().minChars,
429423
indexPrefixes.get().maxChars
430424
)
431425
);
432426
}
433427

434-
private SubFieldInfo buildPhraseInfo(FieldType fieldType, TextFieldType parent) {
428+
private SubFieldInfo buildPhraseInfo(TextParams.Analyzers analyzers, FieldType fieldType, TextFieldType parent) {
435429
if (indexPhrases.get() == false) {
436430
return null;
437431
}
@@ -444,15 +438,14 @@ private SubFieldInfo buildPhraseInfo(FieldType fieldType, TextFieldType parent)
444438
FieldType phraseFieldType = new FieldType(fieldType);
445439
parent.setIndexPhrases();
446440
PhraseWrappedAnalyzer a = new PhraseWrappedAnalyzer(
447-
analyzers.getIndexAnalyzer().analyzer(),
448-
analyzers.positionIncrementGap.get()
441+
analyzers.indexAnalyzer().analyzer(),
442+
analyzers.configuration().posIncrementGap()
449443
);
450444
return new SubFieldInfo(parent.name() + FAST_PHRASE_SUFFIX, phraseFieldType, a);
451445
}
452446

453-
public Map<String, NamedAnalyzer> indexAnalyzers(String name, SubFieldInfo phraseFieldInfo, SubFieldInfo prefixFieldInfo) {
447+
private Map<String, NamedAnalyzer> indexAnalyzers(String name, NamedAnalyzer main, SubFieldInfo phraseFieldInfo, SubFieldInfo prefixFieldInfo) {
454448
Map<String, NamedAnalyzer> analyzers = new HashMap<>();
455-
NamedAnalyzer main = this.analyzers.getIndexAnalyzer();
456449
analyzers.put(name, main);
457450
if (phraseFieldInfo != null) {
458451
analyzers.put(
@@ -471,6 +464,7 @@ public Map<String, NamedAnalyzer> indexAnalyzers(String name, SubFieldInfo phras
471464

472465
@Override
473466
public TextFieldMapper build(MapperBuilderContext context) {
467+
TextParams.Analyzers analyzers = this.analyzers.buildAnalyzers(indexAnalyzers);
474468
MultiFields multiFields = multiFieldsBuilder.build(this, context);
475469
FieldType fieldType = TextParams.buildFieldType(
476470
index,
@@ -480,9 +474,16 @@ public TextFieldMapper build(MapperBuilderContext context) {
480474
indexCreatedVersion.isLegacyIndexVersion() ? () -> false : norms,
481475
termVectors
482476
);
483-
TextFieldType tft = buildFieldType(fieldType, multiFields, context, indexCreatedVersion);
484-
SubFieldInfo phraseFieldInfo = buildPhraseInfo(fieldType, tft);
485-
SubFieldInfo prefixFieldInfo = buildPrefixInfo(context, fieldType, tft);
477+
if (this.analyzers.positionIncrementGap.isConfigured()) {
478+
if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
479+
throw new IllegalArgumentException(
480+
"Cannot set position_increment_gap on field [" + name + "] without positions enabled"
481+
);
482+
}
483+
}
484+
TextFieldType tft = buildFieldType(fieldType, multiFields, context, indexCreatedVersion, analyzers);
485+
SubFieldInfo phraseFieldInfo = buildPhraseInfo(analyzers, fieldType, tft);
486+
SubFieldInfo prefixFieldInfo = buildPrefixInfo(context, analyzers, fieldType, tft);
486487
for (Mapper mapper : multiFields) {
487488
if (mapper.name().endsWith(FAST_PHRASE_SUFFIX) || mapper.name().endsWith(FAST_PREFIX_SUFFIX)) {
488489
throw new MapperParsingException("Cannot use reserved field name [" + mapper.name() + "]");
@@ -492,9 +493,10 @@ public TextFieldMapper build(MapperBuilderContext context) {
492493
name,
493494
fieldType,
494495
tft,
495-
indexAnalyzers(tft.name(), phraseFieldInfo, prefixFieldInfo),
496+
indexAnalyzers(tft.name(), analyzers.indexAnalyzer(), phraseFieldInfo, prefixFieldInfo),
496497
prefixFieldInfo,
497498
phraseFieldInfo,
499+
analyzers,
498500
multiFields,
499501
copyTo.build(),
500502
this
@@ -1150,16 +1152,18 @@ public Query existsQuery(SearchExecutionContext context) {
11501152
private final FieldType fieldType;
11511153
private final SubFieldInfo prefixFieldInfo;
11521154
private final SubFieldInfo phraseFieldInfo;
1155+
private final TextParams.AnalyzerConfiguration analyzerConfiguration;
11531156

11541157
private final Map<String, NamedAnalyzer> indexAnalyzerMap;
11551158

1156-
protected TextFieldMapper(
1159+
private TextFieldMapper(
11571160
String simpleName,
11581161
FieldType fieldType,
11591162
TextFieldType mappedFieldType,
11601163
Map<String, NamedAnalyzer> indexAnalyzers,
11611164
SubFieldInfo prefixFieldInfo,
11621165
SubFieldInfo phraseFieldInfo,
1166+
TextParams.Analyzers analyzers,
11631167
MultiFields multiFields,
11641168
CopyTo copyTo,
11651169
Builder builder
@@ -1174,8 +1178,8 @@ protected TextFieldMapper(
11741178
this.prefixFieldInfo = prefixFieldInfo;
11751179
this.phraseFieldInfo = phraseFieldInfo;
11761180
this.indexCreatedVersion = builder.indexCreatedVersion;
1177-
this.indexAnalyzer = builder.analyzers.getIndexAnalyzer();
1178-
this.indexAnalyzers = builder.analyzers.indexAnalyzers;
1181+
this.indexAnalyzer = analyzers.indexAnalyzer();
1182+
this.indexAnalyzers = builder.indexAnalyzers;
11791183
this.positionIncrementGap = builder.analyzers.positionIncrementGap.getValue();
11801184
this.index = builder.index.getValue();
11811185
this.store = builder.store.getValue();
@@ -1189,6 +1193,7 @@ protected TextFieldMapper(
11891193
this.fieldData = builder.fieldData.get();
11901194
this.indexPhrases = builder.indexPhrases.getValue();
11911195
this.indexAnalyzerMap = Map.copyOf(indexAnalyzers);
1196+
this.analyzerConfiguration = analyzers.configuration();
11921197
}
11931198

11941199
@Override

‎server/src/main/java/org/elasticsearch/index/mapper/TextParams.java‎

Lines changed: 117 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
import org.apache.lucene.index.IndexOptions;
1313
import org.elasticsearch.Version;
1414
import org.elasticsearch.index.analysis.AnalysisMode;
15-
import org.elasticsearch.index.analysis.AnalysisRegistry;
1615
import org.elasticsearch.index.analysis.IndexAnalyzers;
1716
import org.elasticsearch.index.analysis.NamedAnalyzer;
1817
import org.elasticsearch.index.mapper.FieldMapper.Parameter;
@@ -29,96 +28,146 @@ public final class TextParams {
2928

3029
private TextParams() {}
3130

32-
public static final class Analyzers {
33-
public final Parameter<NamedAnalyzer> indexAnalyzer;
34-
public final Parameter<NamedAnalyzer> searchAnalyzer;
35-
public final Parameter<NamedAnalyzer> searchQuoteAnalyzer;
31+
/*
32+
* i configured s configured sq configured return
33+
* i y i
34+
* i n default index analyzer
35+
* s * y s
36+
* s y n i
37+
* s n n default search analyzer
38+
* sq * * y sq
39+
* sq * y n s
40+
* sq y n n i
41+
* sq n n n default search quote analyzer
42+
*
43+
*
44+
*
45+
* */
46+
47+
public record AnalyzerConfiguration(String indexAnalyzer, String searchAnalyzer, String searchQuoteAnalyzer, int posIncrementGap) {
48+
49+
public Analyzers buildAnalyzers(IndexAnalyzers indexAnalyzers) {
50+
return new Analyzers(
51+
wrap(buildIndexAnalyzer(indexAnalyzers), AnalysisMode.INDEX_TIME),
52+
wrap(buildSearchAnalyzer(indexAnalyzers), AnalysisMode.SEARCH_TIME),
53+
wrap(buildSearchQuoteAnalyzer(indexAnalyzers), AnalysisMode.SEARCH_TIME),
54+
this
55+
);
56+
}
57+
58+
private NamedAnalyzer buildIndexAnalyzer(IndexAnalyzers indexAnalyzers) {
59+
if (this.indexAnalyzer == null) {
60+
return indexAnalyzers.getDefaultIndexAnalyzer();
61+
}
62+
NamedAnalyzer a = indexAnalyzers.get(this.indexAnalyzer);
63+
if (a == null) {
64+
throw new IllegalArgumentException("Unknown analyzer [" + this.indexAnalyzer + "]");
65+
}
66+
return a;
67+
}
68+
69+
private NamedAnalyzer buildSearchAnalyzer(IndexAnalyzers indexAnalyzers) {
70+
if (this.searchAnalyzer == null) {
71+
if (this.indexAnalyzer == null) {
72+
return indexAnalyzers.getDefaultSearchAnalyzer();
73+
}
74+
return indexAnalyzers.get(this.indexAnalyzer); // null check will already have happened in buildIndexAnalyzer
75+
}
76+
NamedAnalyzer a = indexAnalyzers.get(this.searchAnalyzer);
77+
if (a == null) {
78+
throw new IllegalArgumentException("Unknown analyzer [" + this.searchAnalyzer + "]");
79+
}
80+
return a;
81+
}
82+
83+
private NamedAnalyzer buildSearchQuoteAnalyzer(IndexAnalyzers indexAnalyzers) {
84+
if (this.searchQuoteAnalyzer == null) {
85+
if (this.searchAnalyzer == null) {
86+
if (this.indexAnalyzer == null) {
87+
return indexAnalyzers.getDefaultSearchQuoteAnalyzer();
88+
}
89+
return indexAnalyzers.get(this.indexAnalyzer); // null checked already in buildIndexAnalyzer
90+
}
91+
return indexAnalyzers.get(this.searchAnalyzer); // null checked already in buildSearchAnalyzer
92+
}
93+
NamedAnalyzer a = indexAnalyzers.get(this.searchQuoteAnalyzer);
94+
if (a == null) {
95+
throw new IllegalArgumentException("Unknown analyzer [" + this.searchQuoteAnalyzer + "]");
96+
}
97+
return a;
98+
}
99+
100+
private NamedAnalyzer wrap(NamedAnalyzer in, AnalysisMode analysisMode) {
101+
in.checkAllowedInMode(analysisMode);
102+
if (in.getPositionIncrementGap("") != posIncrementGap) {
103+
return new NamedAnalyzer(in, posIncrementGap);
104+
}
105+
return in;
106+
}
107+
108+
}
109+
110+
public record Analyzers(
111+
NamedAnalyzer indexAnalyzer,
112+
NamedAnalyzer searchAnalyzer,
113+
NamedAnalyzer searchQuoteAnalyzer,
114+
AnalyzerConfiguration configuration
115+
) {}
116+
117+
public static final class AnalyzerParameters {
118+
public final Parameter<String> indexAnalyzer;
119+
public final Parameter<String> searchAnalyzer;
120+
public final Parameter<String> searchQuoteAnalyzer;
36121
public final Parameter<Integer> positionIncrementGap;
37-
public final IndexAnalyzers indexAnalyzers;
38122

39-
public Analyzers(
40-
IndexAnalyzers indexAnalyzers,
41-
Function<FieldMapper, NamedAnalyzer> analyzerInitFunction,
42-
Function<FieldMapper, Integer> positionGapInitFunction,
123+
public AnalyzerParameters(
124+
Function<FieldMapper, AnalyzerConfiguration> analyzerInitFunction,
43125
Version indexCreatedVersion
44126
) {
45127

46-
this.indexAnalyzer = Parameter.analyzerParam(
128+
this.indexAnalyzer = Parameter.stringParam(
47129
"analyzer",
48130
indexCreatedVersion.isLegacyIndexVersion(),
49-
analyzerInitFunction,
50-
indexAnalyzers::getDefaultIndexAnalyzer,
51-
indexCreatedVersion
131+
mapper -> analyzerInitFunction.apply(mapper).indexAnalyzer,
132+
null
52133
)
53-
.setSerializerCheck(
54-
(id, ic, a) -> id
55-
|| ic
56-
|| Objects.equals(a, getSearchAnalyzer()) == false
57-
|| Objects.equals(a, getSearchQuoteAnalyzer()) == false
58-
)
59-
.addValidator(a -> a.checkAllowedInMode(AnalysisMode.INDEX_TIME));
60-
this.searchAnalyzer = Parameter.analyzerParam(
134+
.setSerializerCheck((includeDefaults, isConfigured, value) -> value != null)
135+
.setMergeValidator(
136+
// special case - we allow 'default' to be merged in to an unconfigured analyzer
137+
(previous, toMerge, conflicts) -> Objects.equals(previous, toMerge) || (previous == null && "default".equals(toMerge))
138+
);
139+
this.searchAnalyzer = Parameter.stringParam(
61140
"search_analyzer",
62141
true,
63-
m -> m.fieldType().getTextSearchInfo().searchAnalyzer(),
64-
() -> {
65-
if (indexAnalyzer.isConfigured() == false) {
66-
NamedAnalyzer defaultAnalyzer = indexAnalyzers.get(AnalysisRegistry.DEFAULT_SEARCH_ANALYZER_NAME);
67-
if (defaultAnalyzer != null) {
68-
return defaultAnalyzer;
69-
}
70-
}
71-
return indexAnalyzer.get();
72-
},
73-
indexCreatedVersion
74-
)
75-
.setSerializerCheck((id, ic, a) -> id || ic || Objects.equals(a, getSearchQuoteAnalyzer()) == false)
76-
.addValidator(a -> a.checkAllowedInMode(AnalysisMode.SEARCH_TIME));
77-
this.searchQuoteAnalyzer = Parameter.analyzerParam(
142+
mapper -> analyzerInitFunction.apply(mapper).searchAnalyzer,
143+
null
144+
).setSerializerCheck((includeDefaults, isConfigured, value) -> value != null);
145+
this.searchQuoteAnalyzer = Parameter.stringParam(
78146
"search_quote_analyzer",
79147
true,
80-
m -> m.fieldType().getTextSearchInfo().searchQuoteAnalyzer(),
81-
() -> {
82-
if (searchAnalyzer.isConfigured() == false && indexAnalyzer.isConfigured() == false) {
83-
NamedAnalyzer defaultAnalyzer = indexAnalyzers.get(AnalysisRegistry.DEFAULT_SEARCH_QUOTED_ANALYZER_NAME);
84-
if (defaultAnalyzer != null) {
85-
return defaultAnalyzer;
86-
}
87-
}
88-
return searchAnalyzer.get();
89-
},
90-
indexCreatedVersion
91-
).addValidator(a -> a.checkAllowedInMode(AnalysisMode.SEARCH_TIME));
148+
mapper -> analyzerInitFunction.apply(mapper).searchQuoteAnalyzer,
149+
null
150+
).setSerializerCheck((includeDefaults, isConfigured, value) -> value != null);
92151
this.positionIncrementGap = Parameter.intParam(
93152
"position_increment_gap",
94153
false,
95-
positionGapInitFunction,
154+
mapper -> analyzerInitFunction.apply(mapper).posIncrementGap,
96155
TextFieldMapper.Defaults.POSITION_INCREMENT_GAP
97156
).addValidator(v -> {
98157
if (v < 0) {
99158
throw new MapperParsingException("[position_increment_gap] must be positive, got [" + v + "]");
100159
}
101160
});
102-
this.indexAnalyzers = indexAnalyzers;
103-
}
104-
105-
public NamedAnalyzer getIndexAnalyzer() {
106-
return wrapAnalyzer(indexAnalyzer.getValue());
107-
}
108-
109-
public NamedAnalyzer getSearchAnalyzer() {
110-
return wrapAnalyzer(searchAnalyzer.getValue());
111-
}
112-
113-
public NamedAnalyzer getSearchQuoteAnalyzer() {
114-
return wrapAnalyzer(searchQuoteAnalyzer.getValue());
115161
}
116162

117-
private NamedAnalyzer wrapAnalyzer(NamedAnalyzer a) {
118-
if (positionIncrementGap.isConfigured() == false) {
119-
return a;
120-
}
121-
return new NamedAnalyzer(a, positionIncrementGap.get());
163+
public Analyzers buildAnalyzers(IndexAnalyzers indexAnalyzers) {
164+
AnalyzerConfiguration config = new AnalyzerConfiguration(
165+
indexAnalyzer.get(),
166+
searchAnalyzer.get(),
167+
searchQuoteAnalyzer.get(),
168+
positionIncrementGap.get()
169+
);
170+
return config.buildAnalyzers(indexAnalyzers);
122171
}
123172
}
124173

0 commit comments

Comments
 (0)