Skip to content

Move analyzer resolution in mappings from parse to build #94985

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
romseygeek wants to merge 6 commits into from
Closed

Move analyzer resolution in mappings from parse to build #94985

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4e3c548
WIP: TextFieldMapper failures
romseygeek Mar 23, 2023
07a3629
Merge remote-tracking branch 'origin/main' into mapper/analyzer-params
romseygeek Mar 30, 2023
ddd666c
more wipp, some :server:test tests failing
romseygeek Mar 30, 2023
077daa7
Merge remote-tracking branch 'origin/main' into mapper/analyzer-params
romseygeek Apr 3, 2023
eaa1f62
tests
romseygeek Apr 3, 2023
319c446
annotated text
romseygeek Apr 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 18 additions & 16 deletions ...-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,8 @@ public static class Builder extends FieldMapper.Builder {

private final Parameter<Map<String, String>> meta = Parameter.metaParam();

private final TextParams.Analyzers analyzers;
final TextParams.AnalyzerParameters analyzers;
final IndexAnalyzers indexAnalyzers;

public Builder(String name, IndexAnalyzers indexAnalyzers) {
this(name, Version.CURRENT, indexAnalyzers);
Expand All @@ -104,10 +105,9 @@ public Builder(String name, IndexAnalyzers indexAnalyzers) {
public Builder(String name, Version indexCreatedVersion, IndexAnalyzers indexAnalyzers) {
super(name);
this.indexCreatedVersion = indexCreatedVersion;
this.analyzers = new TextParams.Analyzers(
indexAnalyzers,
m -> ((MatchOnlyTextFieldMapper) m).indexAnalyzer,
m -> ((MatchOnlyTextFieldMapper) m).positionIncrementGap,
this.indexAnalyzers = indexAnalyzers;
this.analyzers = new TextParams.AnalyzerParameters(
m -> ((MatchOnlyTextFieldMapper) m).analyzerConfiguration,
indexCreatedVersion
);
}
Expand All @@ -117,24 +117,24 @@ protected Parameter<?>[] getParameters() {
return new Parameter<?>[] { meta };
}

private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context) {
NamedAnalyzer searchAnalyzer = analyzers.getSearchAnalyzer();
NamedAnalyzer searchQuoteAnalyzer = analyzers.getSearchQuoteAnalyzer();
NamedAnalyzer indexAnalyzer = analyzers.getIndexAnalyzer();
private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context, TextParams.Analyzers analyzers) {
NamedAnalyzer searchAnalyzer = analyzers.searchAnalyzer();
NamedAnalyzer searchQuoteAnalyzer = analyzers.searchQuoteAnalyzer();
NamedAnalyzer indexAnalyzer = analyzers.indexAnalyzer();
TextSearchInfo tsi = new TextSearchInfo(Defaults.FIELD_TYPE, null, searchAnalyzer, searchQuoteAnalyzer);
MatchOnlyTextFieldType ft = new MatchOnlyTextFieldType(
return new MatchOnlyTextFieldType(
context.buildFullName(name),
tsi,
indexAnalyzer,
context.isSourceSynthetic(),
meta.getValue()
);
return ft;
}

@Override
public MatchOnlyTextFieldMapper build(MapperBuilderContext context) {
MatchOnlyTextFieldType tft = buildFieldType(context);
TextParams.Analyzers analyzers = this.analyzers.buildAnalyzers(this.indexAnalyzers);
MatchOnlyTextFieldType tft = buildFieldType(context, analyzers);
MultiFields multiFields = multiFieldsBuilder.build(this, context);
return new MatchOnlyTextFieldMapper(
name,
Expand All @@ -143,6 +143,7 @@ public MatchOnlyTextFieldMapper build(MapperBuilderContext context) {
multiFields,
copyTo.build(),
context.isSourceSynthetic(),
analyzers,
this
);
}
Expand Down Expand Up @@ -359,9 +360,9 @@ private String storedFieldNameForSyntheticSource() {
private final Version indexCreatedVersion;
private final IndexAnalyzers indexAnalyzers;
private final NamedAnalyzer indexAnalyzer;
private final int positionIncrementGap;
private final boolean storeSource;
private final FieldType fieldType;
private final TextParams.AnalyzerConfiguration analyzerConfiguration;

private MatchOnlyTextFieldMapper(
String simpleName,
Expand All @@ -370,16 +371,17 @@ private MatchOnlyTextFieldMapper(
MultiFields multiFields,
CopyTo copyTo,
boolean storeSource,
TextParams.Analyzers analyzers,
Builder builder
) {
super(simpleName, mappedFieldType, multiFields, copyTo, false, null);
assert mappedFieldType.getTextSearchInfo().isTokenized();
assert mappedFieldType.hasDocValues() == false;
this.fieldType = fieldType;
this.indexCreatedVersion = builder.indexCreatedVersion;
this.indexAnalyzers = builder.analyzers.indexAnalyzers;
this.indexAnalyzer = builder.analyzers.getIndexAnalyzer();
this.positionIncrementGap = builder.analyzers.positionIncrementGap.getValue();
this.indexAnalyzers = builder.indexAnalyzers;
this.indexAnalyzer = analyzers.indexAnalyzer();
this.analyzerConfiguration = analyzers.configuration();
this.storeSource = storeSource;
}

Expand Down
25 changes: 15 additions & 10 deletions ...xtras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,8 @@ public static class Builder extends FieldMapper.Builder {
}
}).alwaysSerialize();

final TextParams.Analyzers analyzers;
final IndexAnalyzers indexAnalyzers;
final TextParams.AnalyzerParameters analyzers;
final Parameter<SimilarityProvider> similarity = TextParams.similarity(m -> builder(m).similarity.get());

final Parameter<String> indexOptions = TextParams.textIndexOptions(m -> builder(m).indexOptions.get());
Expand All @@ -147,10 +148,9 @@ public static class Builder extends FieldMapper.Builder {
public Builder(String name, Version indexCreatedVersion, IndexAnalyzers indexAnalyzers) {
super(name);
this.indexCreatedVersion = indexCreatedVersion;
this.analyzers = new TextParams.Analyzers(
indexAnalyzers,
m -> builder(m).analyzers.getIndexAnalyzer(),
m -> builder(m).analyzers.positionIncrementGap.getValue(),
this.indexAnalyzers = indexAnalyzers;
this.analyzers = new TextParams.AnalyzerParameters(
m -> ((SearchAsYouTypeFieldMapper) m).analyzerConfiguration,
indexCreatedVersion
);
}
Expand Down Expand Up @@ -183,15 +183,16 @@ public SearchAsYouTypeFieldMapper build(MapperBuilderContext context) {

Map<String, NamedAnalyzer> indexAnalyzers = new HashMap<>();

NamedAnalyzer indexAnalyzer = analyzers.getIndexAnalyzer();
NamedAnalyzer searchAnalyzer = analyzers.getSearchAnalyzer();
TextParams.Analyzers analyzers = this.analyzers.buildAnalyzers(this.indexAnalyzers);
NamedAnalyzer indexAnalyzer = analyzers.indexAnalyzer();
NamedAnalyzer searchAnalyzer = analyzers.searchAnalyzer();

SearchAsYouTypeFieldType ft = new SearchAsYouTypeFieldType(
context.buildFullName(name),
fieldType,
similarity.getValue(),
analyzers.getSearchAnalyzer(),
analyzers.getSearchQuoteAnalyzer(),
analyzers.searchAnalyzer(),
analyzers.searchQuoteAnalyzer(),
meta.getValue()
);

Expand Down Expand Up @@ -267,6 +268,7 @@ public SearchAsYouTypeFieldMapper build(MapperBuilderContext context) {
prefixFieldMapper,
shingleFieldMappers,
multiFieldsBuilder.build(this, context),
analyzers.configuration(),
this
);
}
Expand Down Expand Up @@ -666,6 +668,7 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew
private final PrefixFieldMapper prefixField;
private final ShingleFieldMapper[] shingleFields;
private final Builder builder;
private final TextParams.AnalyzerConfiguration analyzerConfiguration;

private final Map<String, NamedAnalyzer> indexAnalyzers;

Expand All @@ -677,6 +680,7 @@ public SearchAsYouTypeFieldMapper(
PrefixFieldMapper prefixField,
ShingleFieldMapper[] shingleFields,
MultiFields multiFields,
TextParams.AnalyzerConfiguration analyzerConfiguration,
Builder builder
) {
super(simpleName, mappedFieldType, multiFields, copyTo, false, null);
Expand All @@ -685,6 +689,7 @@ public SearchAsYouTypeFieldMapper(
this.maxShingleSize = builder.maxShingleSize.getValue();
this.builder = builder;
this.indexAnalyzers = Map.copyOf(indexAnalyzers);
this.analyzerConfiguration = analyzerConfiguration;
}

@Override
Expand Down Expand Up @@ -721,7 +726,7 @@ protected String contentType() {
}

public FieldMapper.Builder getMergeBuilder() {
return new Builder(simpleName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzers).init(this);
return new Builder(simpleName(), builder.indexCreatedVersion, builder.indexAnalyzers).init(this);
}

public static String getShingleFieldName(String parentField, int shingleSize) {
Expand Down
30 changes: 18 additions & 12 deletions .../src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,9 @@ public static class Builder extends FieldMapper.Builder {

private final Parameter<Boolean> store = Parameter.storeParam(m -> builder(m).store.getValue(), false);

final TextParams.Analyzers analyzers;
final TextParams.AnalyzerParameters analyzers;
final IndexAnalyzers indexAnalyzers;

final Parameter<SimilarityProvider> similarity = TextParams.similarity(m -> builder(m).similarity.getValue());

final Parameter<String> indexOptions = TextParams.textIndexOptions(m -> builder(m).indexOptions.getValue());
Expand All @@ -91,10 +93,9 @@ public static class Builder extends FieldMapper.Builder {
public Builder(String name, Version indexCreatedVersion, IndexAnalyzers indexAnalyzers) {
super(name);
this.indexCreatedVersion = indexCreatedVersion;
this.analyzers = new TextParams.Analyzers(
indexAnalyzers,
m -> builder(m).analyzers.getIndexAnalyzer(),
m -> builder(m).analyzers.positionIncrementGap.getValue(),
this.indexAnalyzers = indexAnalyzers;
this.analyzers = new TextParams.AnalyzerParameters(
m -> ((AnnotatedTextFieldMapper) m).analyzerConfiguration,
indexCreatedVersion
);
}
Expand All @@ -114,12 +115,12 @@ protected Parameter<?>[] getParameters() {
meta };
}

private AnnotatedTextFieldType buildFieldType(FieldType fieldType, MapperBuilderContext context) {
private AnnotatedTextFieldType buildFieldType(FieldType fieldType, MapperBuilderContext context, TextParams.Analyzers analyzers) {
TextSearchInfo tsi = new TextSearchInfo(
fieldType,
similarity.get(),
wrapAnalyzer(analyzers.getSearchAnalyzer()),
wrapAnalyzer(analyzers.getSearchQuoteAnalyzer())
wrapAnalyzer(analyzers.searchAnalyzer()),
wrapAnalyzer(analyzers.searchQuoteAnalyzer())
);
return new AnnotatedTextFieldType(
context.buildFullName(name),
Expand All @@ -132,11 +133,12 @@ private AnnotatedTextFieldType buildFieldType(FieldType fieldType, MapperBuilder

@Override
public AnnotatedTextFieldMapper build(MapperBuilderContext context) {
TextParams.Analyzers analyzers = this.analyzers.buildAnalyzers(indexAnalyzers);
FieldType fieldType = TextParams.buildFieldType(() -> true, store, indexOptions, norms, termVectors);
if (fieldType.indexOptions() == IndexOptions.NONE) {
throw new IllegalArgumentException("[" + CONTENT_TYPE + "] fields must be indexed");
}
if (analyzers.positionIncrementGap.isConfigured()) {
if (this.analyzers.positionIncrementGap.isConfigured()) {
if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
throw new IllegalArgumentException(
"Cannot set position_increment_gap on field [" + name + "] without positions enabled"
Expand All @@ -146,9 +148,10 @@ public AnnotatedTextFieldMapper build(MapperBuilderContext context) {
return new AnnotatedTextFieldMapper(
name,
fieldType,
buildFieldType(fieldType, context),
buildFieldType(fieldType, context, analyzers),
multiFieldsBuilder.build(this, context),
copyTo.build(),
analyzers,
this
);
}
Expand Down Expand Up @@ -494,6 +497,7 @@ public String typeName() {
}

private final FieldType fieldType;
private final TextParams.AnalyzerConfiguration analyzerConfiguration;
private final Builder builder;

private final NamedAnalyzer indexAnalyzer;
Expand All @@ -504,13 +508,15 @@ protected AnnotatedTextFieldMapper(
AnnotatedTextFieldType mappedFieldType,
MultiFields multiFields,
CopyTo copyTo,
TextParams.Analyzers analyzers,
Builder builder
) {
super(simpleName, mappedFieldType, multiFields, copyTo);
assert fieldType.tokenized();
this.fieldType = fieldType;
this.builder = builder;
this.indexAnalyzer = wrapAnalyzer(builder.analyzers.getIndexAnalyzer());
this.indexAnalyzer = wrapAnalyzer(analyzers.indexAnalyzer());
this.analyzerConfiguration = analyzers.configuration();
}

@Override
Expand Down Expand Up @@ -542,6 +548,6 @@ protected String contentType() {

@Override
public FieldMapper.Builder getMergeBuilder() {
return new Builder(simpleName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzers).init(this);
return new Builder(simpleName(), builder.indexCreatedVersion, builder.indexAnalyzers).init(this);
}
}
Loading