diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index faea13dac4e31..fed5090e2155b 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -45,14 +45,16 @@ import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.BlockSourceReader; import org.elasticsearch.index.mapper.BlockStoredFieldsReader; +import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.StringFieldType; -import org.elasticsearch.index.mapper.StringStoredFieldFieldLoader; +import org.elasticsearch.index.mapper.TextFamilyFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType; import org.elasticsearch.index.mapper.TextParams; @@ -73,14 +75,12 @@ import java.util.Objects; import java.util.Set; -import static org.elasticsearch.index.mapper.TextFieldMapper.Builder.multiFieldsNotStoredByDefaultIndexVersionCheck; - /** * A {@link FieldMapper} for full-text fields that only indexes * {@link IndexOptions#DOCS} and runs positional queries by looking at the * _source. */ -public class MatchOnlyTextFieldMapper extends FieldMapper { +public class MatchOnlyTextFieldMapper extends TextFamilyFieldMapper { public static final String CONTENT_TYPE = "match_only_text"; @@ -106,15 +106,17 @@ public static class Builder extends FieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); private final TextParams.Analyzers analyzers; - private final boolean withinMultiField; private final boolean storedFieldInBinaryFormat; + private final boolean isWithinMultiField; + + private boolean isSyntheticSourceEnabled; public Builder( String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers, - boolean withinMultiField, - boolean storedFieldInBinaryFormat + boolean storedFieldInBinaryFormat, + boolean isWithinMultiField ) { super(name); this.indexCreatedVersion = indexCreatedVersion; @@ -124,8 +126,8 @@ public Builder( m -> ((MatchOnlyTextFieldMapper) m).positionIncrementGap, indexCreatedVersion ); - this.withinMultiField = withinMultiField; this.storedFieldInBinaryFormat = storedFieldInBinaryFormat; + this.isWithinMultiField = isWithinMultiField; } @Override @@ -133,36 +135,40 @@ protected Parameter[] getParameters() { return new Parameter[] { meta }; } - private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context) { + private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context, MultiFields multiFields) { NamedAnalyzer searchAnalyzer = analyzers.getSearchAnalyzer(); NamedAnalyzer searchQuoteAnalyzer = analyzers.getSearchQuoteAnalyzer(); NamedAnalyzer indexAnalyzer = analyzers.getIndexAnalyzer(); TextSearchInfo tsi = new TextSearchInfo(Defaults.FIELD_TYPE, null, searchAnalyzer, searchQuoteAnalyzer); - MatchOnlyTextFieldType ft = new MatchOnlyTextFieldType( + return new MatchOnlyTextFieldType( context.buildFullName(leafName()), tsi, indexAnalyzer, context.isSourceSynthetic(), meta.getValue(), - withinMultiField, - multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField(), - storedFieldInBinaryFormat + isWithinMultiField, + storedFieldInBinaryFormat, + TextFieldMapper.SyntheticSourceHelper.syntheticSourceDelegate(getFieldType(), multiFields) ); - return ft; + } + + /** + * This is a helper function that's useful in TextFieldMapper.SyntheticSourceHelper.syntheticSourceDelegate() + */ + private FieldType getFieldType() { + FieldType fieldType = new FieldType(); + // by definition, match_only_text fields are not stored + fieldType.setStored(false); + return fieldType; } @Override public MatchOnlyTextFieldMapper build(MapperBuilderContext context) { - MatchOnlyTextFieldType tft = buildFieldType(context); - final boolean storeSource; - if (multiFieldsNotStoredByDefaultIndexVersionCheck(indexCreatedVersion)) { - storeSource = context.isSourceSynthetic() - && withinMultiField == false - && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false; - } else { - storeSource = context.isSourceSynthetic(); - } - return new MatchOnlyTextFieldMapper(leafName(), Defaults.FIELD_TYPE, tft, builderParams(this, context), storeSource, this); + this.isSyntheticSourceEnabled = context.isSourceSynthetic(); + + BuilderParams builderParams = builderParams(this, context); + MatchOnlyTextFieldType tft = buildFieldType(context, builderParams.multiFields()); + return new MatchOnlyTextFieldMapper(leafName(), Defaults.FIELD_TYPE, tft, builderParams, this); } } @@ -179,8 +185,8 @@ private static boolean isSyntheticSourceStoredFieldInBinaryFormat(IndexVersion i n, c.indexVersionCreated(), c.getIndexAnalyzers(), - c.isWithinMultiField(), - isSyntheticSourceStoredFieldInBinaryFormat(c.indexVersionCreated()) + isSyntheticSourceStoredFieldInBinaryFormat(c.indexVersionCreated()), + c.isWithinMultiField() ) ); @@ -188,10 +194,6 @@ public static class MatchOnlyTextFieldType extends StringFieldType { private final Analyzer indexAnalyzer; private final TextFieldType textFieldType; - private final String originalName; - - private final boolean withinMultiField; - private final boolean hasCompatibleMultiFields; private final boolean storedFieldInBinaryFormat; public MatchOnlyTextFieldType( @@ -201,15 +203,12 @@ public MatchOnlyTextFieldType( boolean isSyntheticSource, Map meta, boolean withinMultiField, - boolean hasCompatibleMultiFields, - boolean storedFieldInBinaryFormat + boolean storedFieldInBinaryFormat, + KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate ) { - super(name, true, false, false, tsi, meta); + super(name, true, false, false, tsi, meta, isSyntheticSource, withinMultiField); this.indexAnalyzer = Objects.requireNonNull(indexAnalyzer); - this.textFieldType = new TextFieldType(name, isSyntheticSource); - this.originalName = isSyntheticSource ? name + "._original" : null; - this.withinMultiField = withinMultiField; - this.hasCompatibleMultiFields = hasCompatibleMultiFields; + this.textFieldType = new TextFieldType(name, isSyntheticSource, withinMultiField, syntheticSourceDelegate); this.storedFieldInBinaryFormat = storedFieldInBinaryFormat; } @@ -222,7 +221,7 @@ public MatchOnlyTextFieldType(String name) { Collections.emptyMap(), false, false, - false + null ); } @@ -249,59 +248,31 @@ private IOFunction, IOExcepti "Field [" + name() + "] of type [" + CONTENT_TYPE + "] cannot run positional queries since [_source] is disabled." ); } - if (searchExecutionContext.isSourceSynthetic() && withinMultiField) { - String parentField = searchExecutionContext.parentPath(name()); - var parent = searchExecutionContext.lookup().fieldType(parentField); - - if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent - && keywordParent.ignoreAbove() != Integer.MAX_VALUE) { - if (parent.isStored()) { - return storedFieldFetcher(parentField, keywordParent.originalName()); - } else if (parent.hasDocValues()) { - var ifd = searchExecutionContext.getForField(parent, MappedFieldType.FielddataOperation.SEARCH); - return combineFieldFetchers(docValuesFieldFetcher(ifd), storedFieldFetcher(keywordParent.originalName())); - } - } - if (parent.isStored()) { - return storedFieldFetcher(parentField); - } else if (parent.hasDocValues()) { - var ifd = searchExecutionContext.getForField(parent, MappedFieldType.FielddataOperation.SEARCH); - return docValuesFieldFetcher(ifd); + // if synthetic source is enabled, then fetch the value from one of the valid source providers + if (searchExecutionContext.isSourceSynthetic()) { + if (isWithinMultiField) { + // fetch the value from parent + return parentFieldValueFetcher(searchExecutionContext); + } else if (textFieldType.syntheticSourceDelegate().isPresent()) { + // otherwise, if there is a delegate field, fetch the value from it + return delegateFieldValueFetcher(searchExecutionContext, textFieldType.syntheticSourceDelegate().get()); } else { - assert false : "parent field should either be stored or have doc values"; + // otherwise, fetch the value from self + return storedFieldFetcher(name(), syntheticSourceFallbackFieldName()); } - } else if (searchExecutionContext.isSourceSynthetic() && hasCompatibleMultiFields) { - var mapper = (MatchOnlyTextFieldMapper) searchExecutionContext.getMappingLookup().getMapper(name()); - var kwd = TextFieldMapper.SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(mapper); - - if (kwd != null) { - var fieldType = kwd.fieldType(); - - if (fieldType.ignoreAbove() != Integer.MAX_VALUE) { - if (fieldType.isStored()) { - return storedFieldFetcher(fieldType.name(), fieldType.originalName()); - } else if (fieldType.hasDocValues()) { - var ifd = searchExecutionContext.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); - return combineFieldFetchers(docValuesFieldFetcher(ifd), storedFieldFetcher(fieldType.originalName())); - } - } - - if (fieldType.isStored()) { - return storedFieldFetcher(fieldType.name()); - } else if (fieldType.hasDocValues()) { - var ifd = searchExecutionContext.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); - return docValuesFieldFetcher(ifd); - } else { - assert false : "multi field should either be stored or have doc values"; - } - } else { - assert false : "multi field of type keyword should exist"; - } - } else if (searchExecutionContext.isSourceSynthetic()) { - String name = storedFieldNameForSyntheticSource(); - return storedFieldFetcher(name); } + + // otherwise, synthetic source must be disabled, so fetch the value directly from _source + return sourceFieldValueFetcher(searchExecutionContext); + } + + /** + * Returns a function that will fetch values directly from _source. + */ + private IOFunction, IOException>> sourceFieldValueFetcher( + final SearchExecutionContext searchExecutionContext + ) { return context -> { ValueFetcher valueFetcher = valueFetcher(searchExecutionContext, null); SourceProvider sourceProvider = searchExecutionContext.lookup(); @@ -316,6 +287,67 @@ private IOFunction, IOExcepti }; } + /** + * Returns a function that will fetch value from a parent field. + */ + private IOFunction, IOException>> parentFieldValueFetcher( + final SearchExecutionContext searchExecutionContext + ) { + assert searchExecutionContext.isSourceSynthetic() : "Synthetic source should be enabled"; + + String parentField = searchExecutionContext.parentPath(name()); + var parent = searchExecutionContext.lookup().fieldType(parentField); + + if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent && keywordParent.isIgnoreAboveSet()) { + final String parentFieldName = keywordParent.syntheticSourceFallbackFieldName(); + if (parent.isStored()) { + return storedFieldFetcher(parentField, parentFieldName); + } else if (parent.hasDocValues()) { + var ifd = searchExecutionContext.getForField(parent, MappedFieldType.FielddataOperation.SEARCH); + return combineFieldFetchers(docValuesFieldFetcher(ifd), storedFieldFetcher(parentFieldName)); + } + } + + if (parent.isStored()) { + return storedFieldFetcher(parentField); + } else if (parent.hasDocValues()) { + var ifd = searchExecutionContext.getForField(parent, MappedFieldType.FielddataOperation.SEARCH); + return docValuesFieldFetcher(ifd); + } else { + assert false : "parent field should either be stored or have doc values"; + return sourceFieldValueFetcher(searchExecutionContext); + } + } + + /** + * Returns a function that will fetch values from a synthetic source delegate. + */ + private IOFunction, IOException>> delegateFieldValueFetcher( + final SearchExecutionContext searchExecutionContext, + final KeywordFieldMapper.KeywordFieldType keywordDelegate + ) { + // because we don't know whether the delegate field will be ignored during parsing, we must also check the current field + String fieldName = name(); + String fallbackName = syntheticSourceFallbackFieldName(); + + // delegate field names + String delegateFieldName = keywordDelegate.name(); + String delegateFieldFallbackName = keywordDelegate.syntheticSourceFallbackFieldName(); + + if (keywordDelegate.isStored()) { + return storedFieldFetcher(delegateFieldName, delegateFieldFallbackName, fieldName, fallbackName); + } else if (keywordDelegate.hasDocValues()) { + var ifd = searchExecutionContext.getForField(keywordDelegate, MappedFieldType.FielddataOperation.SEARCH); + return combineFieldFetchers( + docValuesFieldFetcher(ifd), + storedFieldFetcher(delegateFieldFallbackName, fieldName, fallbackName) + ); + } else { + assert false : "multi field should either be stored or have doc values"; + return sourceFieldValueFetcher(searchExecutionContext); + } + } + private static IOFunction, IOException>> docValuesFieldFetcher( IndexFieldData ifd ) { @@ -543,11 +575,12 @@ protected BytesRef toBytesRef(Object v) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - if (textFieldType.isSyntheticSource()) { + if (textFieldType.isSyntheticSourceEnabled()) { + final String fieldName = syntheticSourceFallbackFieldName(); if (storedFieldInBinaryFormat) { - return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(storedFieldNameForSyntheticSource()); + return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(fieldName); } else { - return new BytesFromMixedStringsBytesRefBlockLoader(storedFieldNameForSyntheticSource()); + return new BytesFromMixedStringsBytesRefBlockLoader(fieldName); } } SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name())); @@ -561,9 +594,9 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext if (fieldDataContext.fielddataOperation() != FielddataOperation.SCRIPT) { throw new IllegalArgumentException(CONTENT_TYPE + " fields do not support sorting and aggregations"); } - if (textFieldType.isSyntheticSource()) { + if (textFieldType.isSyntheticSourceEnabled()) { return (cache, breaker) -> new StoredFieldSortedBinaryIndexFieldData( - storedFieldNameForSyntheticSource(), + syntheticSourceFallbackFieldName(), CoreValuesSourceType.KEYWORD, TextDocValuesField::new ) { @@ -586,19 +619,13 @@ protected BytesRef storedToBytesRef(Object stored) { TextDocValuesField::new ); } - - private String storedFieldNameForSyntheticSource() { - return originalName; - } } private final IndexVersion indexCreatedVersion; private final IndexAnalyzers indexAnalyzers; private final NamedAnalyzer indexAnalyzer; private final int positionIncrementGap; - private final boolean storeSource; private final FieldType fieldType; - private final boolean withinMultiField; private final boolean storedFieldInBinaryFormat; private MatchOnlyTextFieldMapper( @@ -606,19 +633,25 @@ private MatchOnlyTextFieldMapper( FieldType fieldType, MatchOnlyTextFieldType mappedFieldType, BuilderParams builderParams, - boolean storeSource, Builder builder ) { - super(simpleName, mappedFieldType, builderParams); + super( + simpleName, + builder.indexCreatedVersion, + builder.isSyntheticSourceEnabled, + builder.isWithinMultiField, + mappedFieldType, + builderParams + ); + assert mappedFieldType.getTextSearchInfo().isTokenized(); assert mappedFieldType.hasDocValues() == false; + this.fieldType = freezeAndDeduplicateFieldType(fieldType); this.indexCreatedVersion = builder.indexCreatedVersion; this.indexAnalyzers = builder.analyzers.indexAnalyzers; this.indexAnalyzer = builder.analyzers.getIndexAnalyzer(); this.positionIncrementGap = builder.analyzers.positionIncrementGap.getValue(); - this.storeSource = storeSource; - this.withinMultiField = builder.withinMultiField; this.storedFieldInBinaryFormat = builder.storedFieldInBinaryFormat; } @@ -629,7 +662,7 @@ public Map indexAnalyzers() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, withinMultiField, storedFieldInBinaryFormat).init(this); + return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, storedFieldInBinaryFormat, isWithinMultiField).init(this); } @Override @@ -645,12 +678,18 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio context.doc().add(field); context.addToFieldNames(fieldType().name()); - if (storeSource) { - if (storedFieldInBinaryFormat) { - final var bytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length()); - context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), bytesRef)); - } else { - context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), value.string())); + // match_only_text isn't stored, so if synthetic source needs to be supported, we must do something about it + if (needsToSupportSyntheticSource()) { + // if the delegate can't support synthetic source for the given value, then store a copy of said value so + // that synthetic source can load it + if (fieldType().textFieldType.canUseSyntheticSourceDelegateForSyntheticSource(value.string()) == false) { + final String fieldName = fieldType().syntheticSourceFallbackFieldName(); + if (storedFieldInBinaryFormat) { + final var bytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length()); + context.doc().add(new StoredField(fieldName, bytesRef)); + } else { + context.doc().add(new StoredField(fieldName, value.string())); + } } } } @@ -667,27 +706,37 @@ public MatchOnlyTextFieldType fieldType() { @Override protected SyntheticSourceSupport syntheticSourceSupport() { - if (storeSource) { - return new SyntheticSourceSupport.Native( - () -> new StringStoredFieldFieldLoader(fieldType().storedFieldNameForSyntheticSource(), fieldType().name(), leafName()) { - @Override - protected void write(XContentBuilder b, Object value) throws IOException { - if (value instanceof BytesRef valueBytes) { - b.value(valueBytes.utf8ToString()); - } else { - assert value instanceof String; - b.value(value.toString()); - } - } + return new SyntheticSourceSupport.Native(() -> syntheticFieldLoader(fullPath(), leafName())); + } + + private SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fullFieldName, String leafFieldName) { + // match_only_text is not stored for space efficiency, except when synthetic source is enabled as synthetic source + // needs something to load to reconstruct source. In such cases, we *might* store this field or we *might* rely + // on a delegate field if one exists. Because of this uncertainty, we need multiple field loaders. + + // first field loader, representing this field + final String fieldName = fieldType().syntheticSourceFallbackFieldName(); + final var thisFieldLayer = new CompositeSyntheticFieldLoader.StoredFieldLayer(fieldName) { + @Override + protected void writeValue(Object value, XContentBuilder b) throws IOException { + if (value instanceof BytesRef valueBytes) { + b.value(valueBytes.utf8ToString()); + } else { + assert value instanceof String; + b.value(value.toString()); } - ); - } else { - var kwd = TextFieldMapper.SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(this); - if (kwd != null) { - return new SyntheticSourceSupport.Native(() -> kwd.syntheticFieldLoader(fullPath(), leafName())); } - assert false : "there should be a suite field mapper with native synthetic source support"; - return super.syntheticSourceSupport(); + }; + + final CompositeSyntheticFieldLoader fieldLoader = new CompositeSyntheticFieldLoader(leafFieldName, fullFieldName, thisFieldLayer); + + // second loader, representing a delegate field, if one exists + var kwd = TextFieldMapper.SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(this); + if (kwd != null) { + // merge the two field loaders into one + return fieldLoader.mergedWith(kwd.syntheticFieldLoader(fullPath(), leafName())); } + + return fieldLoader; } } diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 8ee639ffc8431..2a0fea17efb25 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -20,18 +20,21 @@ import org.apache.lucene.analysis.tokenattributes.TypeAttribute; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexOptions; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MapperBuilderContext; -import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.StringStoredFieldFieldLoader; +import org.elasticsearch.index.mapper.TextFamilyFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TextParams; import org.elasticsearch.index.mapper.TextSearchInfo; @@ -61,7 +64,7 @@ * This code is largely a copy of TextFieldMapper which is less than ideal - * my attempts to subclass TextFieldMapper failed but we can revisit this. **/ -public class AnnotatedTextFieldMapper extends FieldMapper { +public class AnnotatedTextFieldMapper extends TextFamilyFieldMapper { public static final String CONTENT_TYPE = "annotated_text"; @@ -84,28 +87,26 @@ public static class Builder extends FieldMapper.Builder { final Parameter indexOptions = TextParams.textIndexOptions(m -> builder(m).indexOptions.getValue()); final Parameter norms = TextParams.norms(true, m -> builder(m).norms.getValue()); final Parameter termVectors = TextParams.termVectors(m -> builder(m).termVectors.getValue()); + private final Parameter store = Parameter.storeParam(m -> builder(m).store.getValue(), false); private final Parameter> meta = Parameter.metaParam(); private final IndexVersion indexCreatedVersion; private final TextParams.Analyzers analyzers; - private final boolean isSyntheticSourceEnabled; - private final Parameter store; + private final boolean isWithinMultiField; - public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers, boolean isSyntheticSourceEnabled) { + private boolean isSyntheticSourceEnabled; + + public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers, boolean isWithinMultiField) { super(name); this.indexCreatedVersion = indexCreatedVersion; + this.isWithinMultiField = isWithinMultiField; this.analyzers = new TextParams.Analyzers( indexAnalyzers, m -> builder(m).analyzers.getIndexAnalyzer(), m -> builder(m).analyzers.positionIncrementGap.getValue(), indexCreatedVersion ); - this.isSyntheticSourceEnabled = isSyntheticSourceEnabled; - this.store = Parameter.storeParam( - m -> builder(m).store.getValue(), - () -> isSyntheticSourceEnabled && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false - ); } @Override @@ -135,6 +136,7 @@ private AnnotatedTextFieldType buildFieldType(FieldType fieldType, MapperBuilder store.getValue(), tsi, context.isSourceSynthetic(), + isWithinMultiField, TextFieldMapper.SyntheticSourceHelper.syntheticSourceDelegate(fieldType, multiFields), meta.getValue() ); @@ -154,6 +156,7 @@ public AnnotatedTextFieldMapper build(MapperBuilderContext context) { } } BuilderParams builderParams = builderParams(this, context); + this.isSyntheticSourceEnabled = context.isSourceSynthetic(); return new AnnotatedTextFieldMapper( leafName(), fieldType, @@ -165,7 +168,7 @@ public AnnotatedTextFieldMapper build(MapperBuilderContext context) { } public static final TypeParser PARSER = new TypeParser( - (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), SourceFieldMapper.isSynthetic(c.getIndexSettings())) + (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), c.isWithinMultiField()) ); /** @@ -484,15 +487,17 @@ private void emitAnnotation(int firstSpannedTextPosInc, int annotationPosLen) th } public static final class AnnotatedTextFieldType extends TextFieldMapper.TextFieldType { + private AnnotatedTextFieldType( String name, boolean store, TextSearchInfo tsi, boolean isSyntheticSource, + boolean isWithinMultiField, KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate, Map meta ) { - super(name, true, store, tsi, isSyntheticSource, syntheticSourceDelegate, meta, false, false); + super(name, true, store, tsi, isSyntheticSource, isWithinMultiField, syntheticSourceDelegate, meta, false, false); } public AnnotatedTextFieldType(String name, Map meta) { @@ -505,9 +510,9 @@ public String typeName() { } } + private final IndexVersion indexCreatedVersion; private final FieldType fieldType; private final Builder builder; - private final NamedAnalyzer indexAnalyzer; protected AnnotatedTextFieldMapper( @@ -517,11 +522,26 @@ protected AnnotatedTextFieldMapper( BuilderParams builderParams, Builder builder ) { - super(simpleName, mappedFieldType, builderParams); + super( + simpleName, + builder.indexCreatedVersion, + builder.isSyntheticSourceEnabled, + builder.isWithinMultiField, + mappedFieldType, + builderParams + ); + assert fieldType.tokenized(); + this.fieldType = freezeAndDeduplicateFieldType(fieldType); this.builder = builder; this.indexAnalyzer = wrapAnalyzer(builder.analyzers.getIndexAnalyzer()); + this.indexCreatedVersion = builder.indexCreatedVersion; + } + + @Override + public AnnotatedTextFieldType fieldType() { + return (AnnotatedTextFieldType) super.fieldType(); } @Override @@ -543,6 +563,18 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio if (fieldType.omitNorms()) { context.addToFieldNames(fieldType().name()); } + } else if (needsToSupportSyntheticSource() && fieldType.stored() == false) { + // if synthetic source needs to be supported, yet the field isn't stored, then we need to rely on something + // else to support synthetic source + + // if we can rely on the synthetic source delegate for synthetic source, then return + if (fieldType().canUseSyntheticSourceDelegateForSyntheticSource(value)) { + return; + } + + // otherwise, we need to store a copy of this value so that synthetic source can load it + final String fieldName = fieldType().syntheticSourceFallbackFieldName(); + context.doc().add(new StoredField(fieldName, value)); } } @@ -553,8 +585,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzers, builder.isSyntheticSourceEnabled) - .init(this); + return new Builder(leafName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzers, isWithinMultiField).init(this); } @Override @@ -568,11 +599,31 @@ protected void write(XContentBuilder b, Object value) throws IOException { }); } + return new SyntheticSourceSupport.Native(() -> syntheticFieldLoader(fullPath(), leafName())); + } + + private SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fullFieldName, String leafFieldName) { + // since we don't know whether the delegate field loader can be used for synthetic source until parsing, we + // need to check both this field and the delegate + + // first field loader, representing this field + final String fieldName = fieldType().syntheticSourceFallbackFieldName(); + final var thisFieldLayer = new CompositeSyntheticFieldLoader.StoredFieldLayer(fieldName) { + @Override + protected void writeValue(Object value, XContentBuilder b) throws IOException { + b.value(value.toString()); + } + }; + + final CompositeSyntheticFieldLoader fieldLoader = new CompositeSyntheticFieldLoader(leafFieldName, fullFieldName, thisFieldLayer); + + // second loader, representing a delegate field, if one exists var kwd = TextFieldMapper.SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(this); if (kwd != null) { - return new SyntheticSourceSupport.Native(() -> kwd.syntheticFieldLoader(fullPath(), leafName())); + // merge the two field loaders into one + return fieldLoader.mergedWith(kwd.syntheticFieldLoader(fullPath(), leafName())); } - return super.syntheticSourceSupport(); + return fieldLoader; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompositeSyntheticFieldLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/CompositeSyntheticFieldLoader.java index 4f0afeb77fb9a..8dff9b4f4d08b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompositeSyntheticFieldLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompositeSyntheticFieldLoader.java @@ -132,6 +132,18 @@ public String fieldName() { return this.fullFieldName; } + /** + * Returns a new {@link CompositeSyntheticFieldLoader} that merges this field loader with the given one. + */ + public CompositeSyntheticFieldLoader mergedWith(CompositeSyntheticFieldLoader other) { + if (other == null) { + return new CompositeSyntheticFieldLoader(leafFieldName, fullFieldName, List.copyOf(parts)); + } + List mergedParts = new ArrayList<>(parts); + mergedParts.addAll(other.parts); + return new CompositeSyntheticFieldLoader(leafFieldName, fullFieldName, mergedParts); + } + /** * Represents one layer of loading synthetic source values for a field * as a part of {@link CompositeSyntheticFieldLoader}. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index b77c0426c23d4..9d67b15a0df62 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -335,6 +335,20 @@ public final void addIgnoredField(IgnoredSourceFieldMapper.NameValue values) { } } + /** + * This function acts as a more "readable" wrapper around adding ignored fields. + * + * This is useful when we want to reuse the existing logic that {@link IgnoredSourceFieldMapper} provides for synthetic source, without + * explicitly calling addIgnoredField(). Without this, it's a bit confusing why fields that are not meant to be ignored, are being + * added to ignored source. + */ + public final void storeFieldForSyntheticSource(String fullPath, String leafName, BytesRef valueBytes, LuceneDocument doc) { + if (canAddIgnoredField()) { + var fieldData = new IgnoredSourceFieldMapper.NameValue(fullPath, fullPath.lastIndexOf(leafName), valueBytes, doc); + ignoredFieldValues.add(fieldData); + } + } + final void removeLastIgnoredField(String name) { if (ignoredFieldValues.isEmpty() == false && ignoredFieldValues.getLast().name().equals(name)) { ignoredFieldValues.removeLast(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java index fc0b0d864547b..7d4574fe981d0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java @@ -335,10 +335,9 @@ public boolean newDynamicStringField(DocumentParserContext context, String name) ); } else { return createDynamicField( - new TextFieldMapper.Builder(name, context.indexAnalyzers(), SourceFieldMapper.isSynthetic(context.indexSettings())) - .addMultiField( - new KeywordFieldMapper.Builder("keyword", context.indexSettings().getIndexVersionCreated()).ignoreAbove(256) - ), + new TextFieldMapper.Builder(name, context.indexAnalyzers()).addMultiField( + new KeywordFieldMapper.Builder("keyword", context.indexSettings().getIndexVersionCreated(), true).ignoreAbove(256) + ), context ); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index a43575b8f990c..90fc818283ada 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -610,28 +610,13 @@ public static class Builder { private final Map> mapperBuilders = new HashMap<>(); - private boolean hasSyntheticSourceCompatibleKeywordField; - public Builder add(FieldMapper.Builder builder) { mapperBuilders.put(builder.leafName(), builder::build); - - if (builder instanceof KeywordFieldMapper.Builder kwd) { - if (kwd.hasNormalizer() == false && (kwd.hasDocValues() || kwd.isStored())) { - hasSyntheticSourceCompatibleKeywordField = true; - } - } - return this; } private void add(FieldMapper mapper) { mapperBuilders.put(mapper.leafName(), context -> mapper); - - if (mapper instanceof KeywordFieldMapper kwd) { - if (kwd.hasNormalizer() == false && (kwd.fieldType().hasDocValues() || kwd.fieldType().isStored())) { - hasSyntheticSourceCompatibleKeywordField = true; - } - } } private void update(FieldMapper toMerge, MapperMergeContext context) { @@ -649,10 +634,6 @@ public boolean hasMultiFields() { return mapperBuilders.isEmpty() == false; } - public boolean hasSyntheticSourceCompatibleKeywordField() { - return hasSyntheticSourceCompatibleKeywordField; - } - public MultiFields build(Mapper.Builder mainFieldBuilder, MapperBuilderContext context) { if (mapperBuilders.isEmpty()) { return empty(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index e26c969f7d495..c5bab79002663 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -212,6 +212,7 @@ public static final class Builder extends FieldMapper.DimensionBuilder { private final boolean enableDocValuesSkipper; private final boolean forceDocValuesSkipper; private final SourceKeepMode indexSourceKeepMode; + private final boolean isWithinMultiField; public Builder(final String name, final MappingParserContext mappingParserContext) { this( @@ -224,7 +225,8 @@ public Builder(final String name, final MappingParserContext mappingParserContex mappingParserContext.getIndexSettings().getIndexSortConfig(), USE_DOC_VALUES_SKIPPER.get(mappingParserContext.getSettings()), false, - mappingParserContext.getIndexSettings().sourceKeepMode() + mappingParserContext.getIndexSettings().sourceKeepMode(), + mappingParserContext.isWithinMultiField() ); } @@ -234,7 +236,8 @@ public Builder(final String name, final MappingParserContext mappingParserContex ScriptCompiler scriptCompiler, int ignoreAboveDefault, IndexVersion indexCreatedVersion, - SourceKeepMode sourceKeepMode + SourceKeepMode sourceKeepMode, + boolean isWithinMultiField ) { this( name, @@ -246,7 +249,8 @@ public Builder(final String name, final MappingParserContext mappingParserContex null, false, false, - sourceKeepMode + sourceKeepMode, + isWithinMultiField ); } @@ -260,7 +264,8 @@ private Builder( IndexSortConfig indexSortConfig, boolean enableDocValuesSkipper, boolean forceDocValuesSkipper, - SourceKeepMode indexSourceKeepMode + SourceKeepMode indexSourceKeepMode, + boolean isWithinMultiField ) { super(name); this.indexAnalyzers = indexAnalyzers; @@ -300,17 +305,23 @@ private Builder( this.enableDocValuesSkipper = enableDocValuesSkipper; this.forceDocValuesSkipper = forceDocValuesSkipper; this.indexSourceKeepMode = indexSourceKeepMode; + this.isWithinMultiField = isWithinMultiField; } public Builder(String name, IndexVersion indexCreatedVersion) { - this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexCreatedVersion, SourceKeepMode.NONE); + this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexCreatedVersion, SourceKeepMode.NONE, false); + } + + public Builder(String name, IndexVersion indexCreatedVersion, boolean isWithinMultiField) { + this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexCreatedVersion, SourceKeepMode.NONE, isWithinMultiField); } public static Builder buildWithDocValuesSkipper( String name, IndexMode indexMode, IndexVersion indexCreatedVersion, - boolean enableDocValuesSkipper + boolean enableDocValuesSkipper, + boolean isWithinMultiField ) { return new Builder( name, @@ -323,7 +334,8 @@ public static Builder buildWithDocValuesSkipper( null, enableDocValuesSkipper, true, - SourceKeepMode.NONE + SourceKeepMode.NONE, + isWithinMultiField ); } @@ -487,7 +499,6 @@ public KeywordFieldMapper build(MapperBuilderContext context) { fieldtype, buildFieldType(context, fieldtype), builderParams(this, context), - context.isSourceSynthetic(), this, offsetsFieldName, indexSourceKeepMode @@ -543,11 +554,9 @@ public static final class KeywordFieldType extends StringFieldType { private final boolean eagerGlobalOrdinals; private final FieldValues scriptValues; private final boolean isDimension; - private final boolean isSyntheticSource; private final IndexMode indexMode; private final IndexSortConfig indexSortConfig; private final boolean hasDocValuesSkipper; - private final String originalName; public KeywordFieldType( String name, @@ -564,7 +573,9 @@ public KeywordFieldType( fieldType.stored(), builder.hasDocValues.getValue(), textSearchInfo(fieldType, builder.similarity.getValue(), searchAnalyzer, quoteAnalyzer), - builder.meta.getValue() + builder.meta.getValue(), + isSyntheticSource, + builder.isWithinMultiField ); this.eagerGlobalOrdinals = builder.eagerGlobalOrdinals.getValue(); this.normalizer = normalizer; @@ -572,26 +583,22 @@ public KeywordFieldType( this.nullValue = builder.nullValue.getValue(); this.scriptValues = builder.scriptValues(); this.isDimension = builder.dimension.getValue(); - this.isSyntheticSource = isSyntheticSource; this.indexMode = builder.indexMode; this.indexSortConfig = builder.indexSortConfig; this.hasDocValuesSkipper = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; - this.originalName = isSyntheticSource ? name + "._original" : null; } public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map meta) { - super(name, isIndexed, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + super(name, isIndexed, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta, false, false); this.normalizer = Lucene.KEYWORD_ANALYZER; this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; - this.isSyntheticSource = false; this.indexMode = IndexMode.STANDARD; this.indexSortConfig = null; this.hasDocValuesSkipper = false; - this.originalName = null; } public KeywordFieldType(String name) { @@ -605,7 +612,9 @@ public KeywordFieldType(String name, FieldType fieldType) { false, false, textSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), - Collections.emptyMap() + Collections.emptyMap(), + false, + false ); this.normalizer = Lucene.KEYWORD_ANALYZER; this.ignoreAbove = Integer.MAX_VALUE; @@ -613,26 +622,31 @@ public KeywordFieldType(String name, FieldType fieldType) { this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; - this.isSyntheticSource = false; this.indexMode = IndexMode.STANDARD; this.indexSortConfig = null; this.hasDocValuesSkipper = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; - this.originalName = null; } public KeywordFieldType(String name, NamedAnalyzer analyzer) { - super(name, true, false, true, textSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap()); + super( + name, + true, + false, + true, + textSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), + Collections.emptyMap(), + false, + false + ); this.normalizer = Lucene.KEYWORD_ANALYZER; this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; - this.isSyntheticSource = false; this.indexMode = IndexMode.STANDARD; this.indexSortConfig = null; this.hasDocValuesSkipper = false; - this.originalName = null; } @Override @@ -793,7 +807,7 @@ NamedAnalyzer normalizer() { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - if (hasDocValues() && (blContext.fieldExtractPreference() != FieldExtractPreference.STORED || isSyntheticSource)) { + if (hasDocValues() && (blContext.fieldExtractPreference() != FieldExtractPreference.STORED || isSyntheticSourceEnabled)) { return new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(name()); } if (isStored()) { @@ -801,7 +815,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { } // Multi fields don't have fallback synthetic source. - if (isSyntheticSource && blContext.parentField(name()) == null) { + if (isSyntheticSourceEnabled && blContext.parentField(name()) == null) { return new FallbackSyntheticSourceBlockLoader(fallbackSyntheticSourceBlockLoaderReader(), name()) { @Override public Builder builder(BlockFactory factory, int expectedCount) { @@ -814,6 +828,10 @@ public Builder builder(BlockFactory factory, int expectedCount) { return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext)); } + public boolean isIgnoreAboveSet() { + return ignoreAbove != Integer.MAX_VALUE; + } + private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { var nullValueBytes = nullValue != null ? new BytesRef(nullValue) : null; return new FallbackSyntheticSourceBlockLoader.SingleValueReader(nullValueBytes) { @@ -874,7 +892,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext if (hasDocValues()) { return fieldDataFromDocValues(); } - if (isSyntheticSource) { + if (isSyntheticSourceEnabled) { if (false == isStored()) { throw new IllegalStateException( "keyword field [" @@ -1098,12 +1116,11 @@ public Query automatonQuery( } /** - * The name used to store "original" that have been ignored - * by {@link KeywordFieldType#ignoreAbove()} so that they can be rebuilt - * for synthetic source. + * Returns whether the value for this field will be ignored (not indexed/stored) during parsing. */ - public String originalName() { - return originalName; + boolean isIgnored(final String value) { + // all values that exceed ignore_above limits that are multi fields are ignored + return value.length() > ignoreAbove && isWithinMultiField; } } @@ -1116,7 +1133,6 @@ public String originalName() { private final Script script; private final ScriptCompiler scriptCompiler; private final IndexVersion indexCreatedVersion; - private final boolean isSyntheticSource; private final IndexAnalyzers indexAnalyzers; private final int ignoreAboveDefault; @@ -1126,14 +1142,14 @@ public String originalName() { private final boolean forceDocValuesSkipper; private final String offsetsFieldName; private final SourceKeepMode indexSourceKeepMode; - private final String originalName; + + private final boolean isWithinMultiField; private KeywordFieldMapper( String simpleName, FieldType fieldType, KeywordFieldType mappedFieldType, BuilderParams builderParams, - boolean isSyntheticSource, Builder builder, String offsetsFieldName, SourceKeepMode indexSourceKeepMode @@ -1150,7 +1166,6 @@ private KeywordFieldMapper( this.indexAnalyzers = builder.indexAnalyzers; this.scriptCompiler = builder.scriptCompiler; this.indexCreatedVersion = builder.indexCreatedVersion; - this.isSyntheticSource = isSyntheticSource; this.ignoreAboveDefault = builder.ignoreAboveDefault; this.indexMode = builder.indexMode; this.indexSortConfig = builder.indexSortConfig; @@ -1158,7 +1173,7 @@ private KeywordFieldMapper( this.forceDocValuesSkipper = builder.forceDocValuesSkipper; this.offsetsFieldName = offsetsFieldName; this.indexSourceKeepMode = indexSourceKeepMode; - this.originalName = mappedFieldType.originalName(); + this.isWithinMultiField = builder.isWithinMultiField; } @Override @@ -1202,7 +1217,16 @@ private boolean indexValue(DocumentParserContext context, String value) { return indexValue(context, new Text(value)); } + /** + * Returns whether this field should be stored separately as a {@link StoredField} for supporting synthetic source. + */ + private boolean storeIgnoredValuesForSyntheticSource() { + // skip all fields that are multi-fields + return fieldType().isSyntheticSourceEnabled && fieldType().isWithinMultiField == false; + } + private boolean indexValue(DocumentParserContext context, XContentString value) { + // nothing to index if (value == null) { return false; } @@ -1212,13 +1236,15 @@ private boolean indexValue(DocumentParserContext context, XContentString value) return false; } + // if the value's length exceeds ignore_above, then don't index it if (value.stringLength() > fieldType().ignoreAbove()) { context.addIgnoredField(fullPath()); - if (isSyntheticSource) { - // Save a copy of the field so synthetic source can load it + // unless, synthetic source is enabled, then store a copy of the value so that synthetic source be load it + if (storeIgnoredValuesForSyntheticSource()) { var utfBytes = value.bytes(); var bytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length()); - context.doc().add(new StoredField(originalName, bytesRef)); + final String fieldName = fieldType().syntheticSourceFallbackFieldName(); + context.doc().add(new StoredField(fieldName, bytesRef)); } return false; } @@ -1313,7 +1339,8 @@ public FieldMapper.Builder getMergeBuilder() { indexSortConfig, enableDocValuesSkipper, forceDocValuesSkipper, - indexSourceKeepMode + indexSourceKeepMode, + isWithinMultiField ).dimension(fieldType().isDimension()).init(this); } @@ -1349,7 +1376,7 @@ protected SyntheticSourceSupport syntheticSourceSupport() { return super.syntheticSourceSupport(); } - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fullFieldName, String leafFieldName) { + public CompositeSyntheticFieldLoader syntheticFieldLoader(String fullFieldName, String leafFieldName) { assert fieldType.stored() || hasDocValues; var layers = new ArrayList(2); @@ -1381,8 +1408,11 @@ protected BytesRef preserve(BytesRef value) { } } - if (fieldType().ignoreAbove != Integer.MAX_VALUE) { - layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName) { + // if ignore_above is set, then there is a chance that this field will be ignored. In such cases, we save an + // extra copy of the field for supporting synthetic source. This layer will check that copy. + if (fieldType().isIgnoreAboveSet()) { + final String fieldName = fieldType().syntheticSourceFallbackFieldName(); + layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(fieldName) { @Override protected void writeValue(Object value, XContentBuilder b) throws IOException { BytesRef ref = (BytesRef) value; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java index 63a3e1cb13284..e6af09710212c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java @@ -24,11 +24,15 @@ public class MapperBuilderContext { * The root context, to be used when building a tree of mappers */ public static MapperBuilderContext root(boolean isSourceSynthetic, boolean isDataStream) { - return root(isSourceSynthetic, isDataStream, MergeReason.MAPPING_UPDATE); + return MapperBuilderContext.builder().isSourceSynthetic(isSourceSynthetic).isDataStream(isDataStream).build(); } public static MapperBuilderContext root(boolean isSourceSynthetic, boolean isDataStream, MergeReason mergeReason) { - return new MapperBuilderContext(null, isSourceSynthetic, isDataStream, false, ObjectMapper.Defaults.DYNAMIC, mergeReason, false); + return MapperBuilderContext.builder() + .isSourceSynthetic(isSourceSynthetic) + .isDataStream(isDataStream) + .mergeReason(mergeReason) + .build(); } private final String path; @@ -146,4 +150,66 @@ public MergeReason getMergeReason() { public boolean isInNestedContext() { return inNestedContext; } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + + private String path = null; + private boolean isSourceSynthetic; + private boolean isDataStream; + private boolean parentObjectContainsDimensions = false; + private ObjectMapper.Dynamic dynamic = ObjectMapper.Defaults.DYNAMIC; + private MergeReason mergeReason = MergeReason.MAPPING_UPDATE; + private boolean inNestedContext = false; + + public Builder path(String path) { + this.path = path; + return this; + } + + public Builder isSourceSynthetic(boolean isSourceSynthetic) { + this.isSourceSynthetic = isSourceSynthetic; + return this; + } + + public Builder isDataStream(boolean isDataStream) { + this.isDataStream = isDataStream; + return this; + } + + public Builder parentObjectContainsDimensions(boolean parentObjectContainsDimensions) { + this.parentObjectContainsDimensions = parentObjectContainsDimensions; + return this; + } + + public Builder dynamic(ObjectMapper.Dynamic dynamic) { + this.dynamic = dynamic; + return this; + } + + public Builder mergeReason(MergeReason mergeReason) { + this.mergeReason = mergeReason; + return this; + } + + public Builder inNestedContext(boolean inNestedContext) { + this.inNestedContext = inNestedContext; + return this; + } + + public MapperBuilderContext build() { + return new MapperBuilderContext( + path, + isSourceSynthetic, + isDataStream, + parentObjectContainsDimensions, + dynamic, + mergeReason, + inNestedContext + ); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java index ceb96b87a0983..fa87f6f2b35b6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java @@ -43,6 +43,10 @@ public abstract class StringFieldType extends TermBasedFieldType { private static final Pattern WILDCARD_PATTERN = Pattern.compile("(\\\\.)|([?*]+)"); + // using Boolean instead of boolean is deliberate as not all children of this class care about these two fields + protected final Boolean isSyntheticSourceEnabled; + protected final Boolean isWithinMultiField; + public StringFieldType( String name, boolean isIndexed, @@ -50,8 +54,23 @@ public StringFieldType( boolean hasDocValues, TextSearchInfo textSearchInfo, Map meta + ) { + this(name, isIndexed, isStored, hasDocValues, textSearchInfo, meta, null, null); + } + + public StringFieldType( + String name, + boolean isIndexed, + boolean isStored, + boolean hasDocValues, + TextSearchInfo textSearchInfo, + Map meta, + Boolean isSyntheticSourceEnabled, + Boolean isWithinMultiField ) { super(name, isIndexed, isStored, hasDocValues, textSearchInfo, meta); + this.isSyntheticSourceEnabled = isSyntheticSourceEnabled; + this.isWithinMultiField = isWithinMultiField; } @Override @@ -224,4 +243,12 @@ public Query rangeQuery( includeUpper ); } + + /** + * Returns the name of the "fallback" field that can be used for synthetic source when the "main" field was not + * stored for whatever reason. + */ + public String syntheticSourceFallbackFieldName() { + return isSyntheticSourceEnabled ? name() + "._original" : null; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldMapper.java new file mode 100644 index 0000000000000..bc4c019fd048f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldMapper.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; + +/** + * This class is meant to contain common functionality that is needed by the Text family of field mappers. Namely + * {@link TextFieldMapper} and anything strongly related to it. + */ +public abstract class TextFamilyFieldMapper extends FieldMapper { + + protected final IndexVersion indexCreatedVersion; + protected final boolean isSyntheticSourceEnabled; + protected final boolean isWithinMultiField; + + protected TextFamilyFieldMapper( + String name, + IndexVersion indexCreatedVersion, + boolean isSyntheticSourceEnabled, + boolean isWithinMultiField, + MappedFieldType mappedFieldType, + BuilderParams params + ) { + super(name, mappedFieldType, params); + this.indexCreatedVersion = indexCreatedVersion; + this.isSyntheticSourceEnabled = isSyntheticSourceEnabled; + this.isWithinMultiField = isWithinMultiField; + } + + /** + * Returns whether this field mapper needs to support synthetic source. + */ + protected boolean needsToSupportSyntheticSource() { + if (multiFieldsNotStoredByDefaultIndexVersionCheck()) { + // if we're within a multi field, then supporting synthetic source isn't necessary as that's the responsibility of the parent + return isSyntheticSourceEnabled && isWithinMultiField == false; + } + return isSyntheticSourceEnabled; + } + + private boolean multiFieldsNotStoredByDefaultIndexVersionCheck() { + return indexCreatedVersion.onOrAfter(IndexVersions.MAPPER_TEXT_MATCH_ONLY_MULTI_FIELDS_DEFAULT_NOT_STORED) + || indexCreatedVersion.between( + IndexVersions.MAPPER_TEXT_MATCH_ONLY_MULTI_FIELDS_DEFAULT_NOT_STORED_8_19, + IndexVersions.UPGRADE_TO_LUCENE_10_0_0 + ); + } + +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 3d2b89f5a1d48..00c83a4fdf289 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -84,12 +84,13 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.function.IntPredicate; import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; /** A {@link FieldMapper} for full-text fields. */ -public final class TextFieldMapper extends FieldMapper { +public final class TextFieldMapper extends TextFamilyFieldMapper { public static final String CONTENT_TYPE = "text"; private static final String FAST_PHRASE_SUFFIX = "._index_phrase"; @@ -238,11 +239,9 @@ private static FielddataFrequencyFilter parseFrequencyFilter(String name, Mappin public static class Builder extends FieldMapper.Builder { private final IndexVersion indexCreatedVersion; - private final Parameter store; - - private final boolean isSyntheticSourceEnabled; private final Parameter index = Parameter.indexParam(m -> ((TextFieldMapper) m).index, true); + private final Parameter store = Parameter.storeParam(m -> ((TextFieldMapper) m).store, false); final Parameter similarity = TextParams.similarity(m -> ((TextFieldMapper) m).similarity); @@ -287,54 +286,24 @@ public static class Builder extends FieldMapper.Builder { final TextParams.Analyzers analyzers; - private final boolean withinMultiField; + private boolean isSyntheticSourceEnabled; + private final boolean isWithinMultiField; - public Builder(String name, IndexAnalyzers indexAnalyzers, boolean isSyntheticSourceEnabled) { - this(name, IndexVersion.current(), indexAnalyzers, isSyntheticSourceEnabled, false); + public Builder(String name, IndexAnalyzers indexAnalyzers) { + this(name, IndexVersion.current(), indexAnalyzers, false); } - public Builder( - String name, - IndexVersion indexCreatedVersion, - IndexAnalyzers indexAnalyzers, - boolean isSyntheticSourceEnabled, - boolean withinMultiField - ) { + public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers, boolean isWithinMultiField) { super(name); - // If synthetic source is used we need to either store this field - // to recreate the source or use keyword multi-fields for that. - // So if there are no suitable multi-fields we will default to - // storing the field without requiring users to explicitly set 'store'. - // - // If 'store' parameter was explicitly provided we'll reject the request. - // Note that if current builder is a multi field, then we don't need to store, given that responsibility lies with parent field - this.withinMultiField = withinMultiField; - this.store = Parameter.storeParam(m -> ((TextFieldMapper) m).store, () -> { - if (multiFieldsNotStoredByDefaultIndexVersionCheck(indexCreatedVersion)) { - return isSyntheticSourceEnabled - && this.withinMultiField == false - && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false; - } else { - return isSyntheticSourceEnabled && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false; - } - }); this.indexCreatedVersion = indexCreatedVersion; + this.isWithinMultiField = isWithinMultiField; this.analyzers = new TextParams.Analyzers( indexAnalyzers, m -> ((TextFieldMapper) m).indexAnalyzer, m -> (((TextFieldMapper) m).positionIncrementGap), indexCreatedVersion ); - this.isSyntheticSourceEnabled = isSyntheticSourceEnabled; - } - - public static boolean multiFieldsNotStoredByDefaultIndexVersionCheck(IndexVersion indexCreatedVersion) { - return indexCreatedVersion.onOrAfter(IndexVersions.MAPPER_TEXT_MATCH_ONLY_MULTI_FIELDS_DEFAULT_NOT_STORED) - || indexCreatedVersion.between( - IndexVersions.MAPPER_TEXT_MATCH_ONLY_MULTI_FIELDS_DEFAULT_NOT_STORED_8_19, - IndexVersions.UPGRADE_TO_LUCENE_10_0_0 - ); } public Builder index(boolean index) { @@ -410,6 +379,7 @@ private TextFieldType buildFieldType( store.getValue(), tsi, context.isSourceSynthetic(), + isWithinMultiField, SyntheticSourceHelper.syntheticSourceDelegate(fieldType, multiFields), meta.getValue(), eagerGlobalOrdinals.getValue(), @@ -483,6 +453,8 @@ private SubFieldInfo buildPhraseInfo(FieldType fieldType, TextFieldType parent) @Override public TextFieldMapper build(MapperBuilderContext context) { + this.isSyntheticSourceEnabled = context.isSourceSynthetic(); + FieldType fieldType = TextParams.buildFieldType( index, store, @@ -505,13 +477,7 @@ public TextFieldMapper build(MapperBuilderContext context) { } public static final TypeParser PARSER = createTypeParserWithLegacySupport( - (n, c) -> new Builder( - n, - c.indexVersionCreated(), - c.getIndexAnalyzers(), - SourceFieldMapper.isSynthetic(c.getIndexSettings()), - c.isWithinMultiField() - ) + (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), c.isWithinMultiField()) ); private static class PhraseWrappedAnalyzer extends AnalyzerWrapper { @@ -686,13 +652,12 @@ public static class TextFieldType extends StringFieldType { private PrefixFieldType prefixFieldType; private final boolean indexPhrases; private final boolean eagerGlobalOrdinals; - private final boolean isSyntheticSource; /** * In some configurations text fields use a sub-keyword field to provide - * their values for synthetic source. This is that field. Or null if we're + * their values for synthetic source. This is that field. Or empty if we're * not running in synthetic _source or synthetic source doesn't need it. */ - private final KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate; + private final Optional syntheticSourceDelegate; public TextFieldType( String name, @@ -700,16 +665,16 @@ public TextFieldType( boolean stored, TextSearchInfo tsi, boolean isSyntheticSource, + boolean isWithinMultiField, KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate, Map meta, boolean eagerGlobalOrdinals, boolean indexPhrases ) { - super(name, indexed, stored, false, tsi, meta); + super(name, indexed, stored, false, tsi, meta, isSyntheticSource, isWithinMultiField); fielddata = false; - this.isSyntheticSource = isSyntheticSource; // TODO block loader could use a "fast loading" delegate which isn't always the same - but frequently is. - this.syntheticSourceDelegate = syntheticSourceDelegate; + this.syntheticSourceDelegate = Optional.ofNullable(syntheticSourceDelegate); this.eagerGlobalOrdinals = eagerGlobalOrdinals; this.indexPhrases = indexPhrases; } @@ -721,22 +686,24 @@ public TextFieldType(String name, boolean indexed, boolean stored, Map new StoredFieldSortedBinaryIndexFieldData( name(), @@ -1183,8 +1182,8 @@ protected BytesRef storedToBytesRef(Object stored) { } }; } - if (syntheticSourceDelegate != null) { - return syntheticSourceDelegate.fielddataBuilder(fieldDataContext); + if (syntheticSourceDelegate.isPresent()) { + return syntheticSourceDelegate.get().fielddataBuilder(fieldDataContext); } /* * We *shouldn't fall to this exception. The mapping should be @@ -1207,11 +1206,11 @@ protected BytesRef storedToBytesRef(Object stored) { ); } - public boolean isSyntheticSource() { - return isSyntheticSource; + public boolean isSyntheticSourceEnabled() { + return isSyntheticSourceEnabled; } - public KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate() { + public Optional syntheticSourceDelegate() { return syntheticSourceDelegate; } } @@ -1219,7 +1218,7 @@ public KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate() { public static class ConstantScoreTextFieldType extends TextFieldType { public ConstantScoreTextFieldType(String name, boolean indexed, boolean stored, TextSearchInfo tsi, Map meta) { - super(name, indexed, stored, tsi, false, null, meta, false, false); + super(name, indexed, stored, tsi, false, false, null, meta, false, false); } public ConstantScoreTextFieldType(String name) { @@ -1318,7 +1317,6 @@ public Query existsQuery(SearchExecutionContext context) { } - private final IndexVersion indexCreatedVersion; private final boolean index; private final boolean store; private final String indexOptions; @@ -1335,9 +1333,6 @@ public Query existsQuery(SearchExecutionContext context) { private final SubFieldInfo prefixFieldInfo; private final SubFieldInfo phraseFieldInfo; - private final boolean isSyntheticSourceEnabled; - private final boolean isWithinMultiField; - private TextFieldMapper( String simpleName, FieldType fieldType, @@ -1347,16 +1342,26 @@ private TextFieldMapper( BuilderParams builderParams, Builder builder ) { - super(simpleName, mappedFieldType, builderParams); + super( + simpleName, + builder.indexCreatedVersion, + builder.isSyntheticSourceEnabled, + builder.isWithinMultiField, + mappedFieldType, + builderParams + ); + assert mappedFieldType.getTextSearchInfo().isTokenized(); assert mappedFieldType.hasDocValues() == false; - if (fieldType.indexOptions() == IndexOptions.NONE && fieldType().fielddata()) { + + final boolean isIndexed = fieldType.indexOptions() != IndexOptions.NONE; + if (isIndexed == false && fieldType().fielddata()) { throw new IllegalArgumentException("Cannot enable fielddata on a [text] field that is not indexed: [" + fullPath() + "]"); } + this.fieldType = freezeAndDeduplicateFieldType(fieldType); this.prefixFieldInfo = prefixFieldInfo; this.phraseFieldInfo = phraseFieldInfo; - this.indexCreatedVersion = builder.indexCreatedVersion; this.indexAnalyzer = builder.analyzers.getIndexAnalyzer(); this.indexAnalyzers = builder.analyzers.indexAnalyzers; this.positionIncrementGap = builder.analyzers.positionIncrementGap.getValue(); @@ -1369,8 +1374,6 @@ private TextFieldMapper( this.indexPrefixes = builder.indexPrefixes.getValue(); this.freqFilter = builder.freqFilter.getValue(); this.fieldData = builder.fieldData.get(); - this.isSyntheticSourceEnabled = builder.isSyntheticSourceEnabled; - this.isWithinMultiField = builder.withinMultiField; } @Override @@ -1394,7 +1397,7 @@ public Map indexAnalyzers() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, isSyntheticSourceEnabled, isWithinMultiField).init(this); + return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, isWithinMultiField).init(this); } @Override @@ -1405,7 +1408,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio return; } - if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { + if (isIndexed() || fieldType.stored()) { Field field = new Field(fieldType().name(), value, fieldType); context.doc().add(field); if (fieldType.omitNorms()) { @@ -1418,6 +1421,21 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio context.doc().add(new Field(phraseFieldInfo.field, value, phraseFieldInfo.fieldType)); } } + + // if synthetic source needs to be supported, yet the field isn't stored, then we need to rely on something else + if (needsToSupportSyntheticSource() && fieldType.stored() == false) { + // if we can rely on the synthetic source delegate for synthetic source, then exit as there is nothing to do + if (fieldType().canUseSyntheticSourceDelegateForSyntheticSource(value)) { + return; + } + + // otherwise, store this field in Lucene so that synthetic source can load it + var utfBytes = context.parser().optimizedTextOrNull().bytes(); + var valuesBytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length()); + context.storeFieldForSyntheticSource(fullPath(), leafName(), valuesBytesRef, context.doc()); + // final String fieldName = fieldType().syntheticSourceFallbackFieldName(); + // context.doc().add(new StoredField(fieldName, value)); + } } @Override @@ -1578,8 +1596,13 @@ protected void doXContentBody(XContentBuilder builder, Params params) throws IOE b.indexPhrases.toXContent(builder, includeDefaults); } + private boolean isIndexed() { + return fieldType.indexOptions() != IndexOptions.NONE; + } + @Override protected SyntheticSourceSupport syntheticSourceSupport() { + // if we stored this field in Lucene, then use that for synthetic source if (store) { return new SyntheticSourceSupport.Native(() -> new StringStoredFieldFieldLoader(fullPath(), leafName()) { @Override @@ -1589,7 +1612,10 @@ protected void write(XContentBuilder b, Object value) throws IOException { }); } - var kwd = SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(this); + // otherwise, use the delegate + // note, the delegate itself might not be stored in Lucene due to various reasons (ex. it tripped ignore_above), in such cases, we + // should've added this field to ignored_source + var kwd = TextFieldMapper.SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(this); if (kwd != null) { return new SyntheticSourceSupport.Native(() -> kwd.syntheticFieldLoader(fullPath(), leafName())); } @@ -1599,13 +1625,17 @@ protected void write(XContentBuilder b, Object value) throws IOException { public static class SyntheticSourceHelper { public static KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate(FieldType fieldType, MultiFields multiFields) { + // if this field is already stored, then it doesn't need a delegate if (fieldType.stored()) { return null; } + + // otherwise, attempt to retrieve a keyword delegate to rely on for synthetic source var kwd = getKeywordFieldMapperForSyntheticSource(multiFields); if (kwd != null) { return kwd.fieldType(); } + return null; } @@ -1613,7 +1643,7 @@ public static KeywordFieldMapper getKeywordFieldMapperForSyntheticSource(Iterabl for (Mapper sub : multiFields) { if (sub.typeName().equals(KeywordFieldMapper.CONTENT_TYPE)) { KeywordFieldMapper kwd = (KeywordFieldMapper) sub; - if (kwd.hasNormalizer() == false && (kwd.fieldType().hasDocValues() || kwd.fieldType().isStored())) { + if (keywordFieldSupportsSyntheticSource(kwd)) { return kwd; } } @@ -1621,5 +1651,13 @@ public static KeywordFieldMapper getKeywordFieldMapperForSyntheticSource(Iterabl return null; } + + /** + * Returns whether the given keyword field supports synthetic source. + */ + private static boolean keywordFieldSupportsSyntheticSource(final KeywordFieldMapper keyword) { + // the field must be stored in some way, whether that be via store or doc values + return keyword.hasNormalizer() == false && (keyword.fieldType().hasDocValues()) || keyword.fieldType().isStored(); + } } } diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java index bc14a31978c18..b5e37b2405245 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java @@ -26,7 +26,6 @@ import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.plugins.internal.rewriter.QueryRewriteInterceptor; import org.elasticsearch.script.ScriptCompiler; @@ -260,11 +259,7 @@ MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMap if (fieldMapping != null || allowUnmappedFields) { return fieldMapping; } else if (mapUnmappedFieldAsString) { - TextFieldMapper.Builder builder = new TextFieldMapper.Builder( - name, - getIndexAnalyzers(), - getIndexSettings() != null && SourceFieldMapper.isSynthetic(getIndexSettings()) - ); + TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name, getIndexAnalyzers()); return builder.build(MapperBuilderContext.root(false, false)).fieldType(); } else { throw new QueryShardException(this, "No field mapping can be found for the field with name [{}]", name); diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index 8e60895cdbca7..87d25272cb96f 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -92,11 +92,9 @@ public > IFD getForField(String type, String field if (docValues) { fieldType = new KeywordFieldMapper.Builder(fieldName, IndexVersion.current()).build(context).fieldType(); } else { - fieldType = new TextFieldMapper.Builder( - fieldName, - createDefaultIndexAnalyzers(), - SourceFieldMapper.isSynthetic(indexService.getIndexSettings()) - ).fielddata(true).build(context).fieldType(); + fieldType = new TextFieldMapper.Builder(fieldName, createDefaultIndexAnalyzers()).fielddata(true) + .build(context) + .fieldType(); } } else if (type.equals("float")) { fieldType = new NumberFieldMapper.Builder( diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java index a7277b79e5c00..371f7172f5275 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java @@ -15,7 +15,6 @@ import org.apache.lucene.index.SortedSetDocValues; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import java.util.List; @@ -54,11 +53,10 @@ public void testFilterByFrequency() throws Exception { { indexService.clearCaches(false, true); - MappedFieldType ft = new TextFieldMapper.Builder( - "high_freq", - createDefaultIndexAnalyzers(), - SourceFieldMapper.isSynthetic(indexService.getIndexSettings()) - ).fielddata(true).fielddataFrequencyFilter(0, random.nextBoolean() ? 100 : 0.5d, 0).build(builderContext).fieldType(); + MappedFieldType ft = new TextFieldMapper.Builder("high_freq", createDefaultIndexAnalyzers()).fielddata(true) + .fielddataFrequencyFilter(0, random.nextBoolean() ? 100 : 0.5d, 0) + .build(builderContext) + .fieldType(); IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataOperation.SEARCH); for (LeafReaderContext context : contexts) { LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context); @@ -70,11 +68,7 @@ public void testFilterByFrequency() throws Exception { } { indexService.clearCaches(false, true); - MappedFieldType ft = new TextFieldMapper.Builder( - "high_freq", - createDefaultIndexAnalyzers(), - SourceFieldMapper.isSynthetic(indexService.getIndexSettings()) - ).fielddata(true) + MappedFieldType ft = new TextFieldMapper.Builder("high_freq", createDefaultIndexAnalyzers()).fielddata(true) .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, 201, 100) .build(builderContext) .fieldType(); @@ -89,11 +83,7 @@ public void testFilterByFrequency() throws Exception { { indexService.clearCaches(false, true);// test # docs with value - MappedFieldType ft = new TextFieldMapper.Builder( - "med_freq", - createDefaultIndexAnalyzers(), - SourceFieldMapper.isSynthetic(indexService.getIndexSettings()) - ).fielddata(true) + MappedFieldType ft = new TextFieldMapper.Builder("med_freq", createDefaultIndexAnalyzers()).fielddata(true) .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, Integer.MAX_VALUE, 101) .build(builderContext) .fieldType(); @@ -109,11 +99,7 @@ public void testFilterByFrequency() throws Exception { { indexService.clearCaches(false, true); - MappedFieldType ft = new TextFieldMapper.Builder( - "med_freq", - createDefaultIndexAnalyzers(), - SourceFieldMapper.isSynthetic(indexService.getIndexSettings()) - ).fielddata(true) + MappedFieldType ft = new TextFieldMapper.Builder("med_freq", createDefaultIndexAnalyzers()).fielddata(true) .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, Integer.MAX_VALUE, 101) .build(builderContext) .fieldType(); diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index d55f712cb91b6..31e18f34c4020 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -34,7 +34,6 @@ import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; @@ -164,16 +163,12 @@ public void testClearField() throws Exception { ); final MapperBuilderContext context = MapperBuilderContext.root(false, false); - final MappedFieldType mapper1 = new TextFieldMapper.Builder( - "field_1", - createDefaultIndexAnalyzers(), - SourceFieldMapper.isSynthetic(indexService.getIndexSettings()) - ).fielddata(true).build(context).fieldType(); - final MappedFieldType mapper2 = new TextFieldMapper.Builder( - "field_2", - createDefaultIndexAnalyzers(), - SourceFieldMapper.isSynthetic(indexService.getIndexSettings()) - ).fielddata(true).build(context).fieldType(); + final MappedFieldType mapper1 = new TextFieldMapper.Builder("field_1", createDefaultIndexAnalyzers()).fielddata(true) + .build(context) + .fieldType(); + final MappedFieldType mapper2 = new TextFieldMapper.Builder("field_2", createDefaultIndexAnalyzers()).fielddata(true) + .build(context) + .fieldType(); final IndexWriter writer = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer())); Document doc = new Document(); doc.add(new StringField("field_1", "thisisastring", Store.NO)); @@ -235,11 +230,9 @@ public void testFieldDataCacheListener() throws Exception { ); final MapperBuilderContext context = MapperBuilderContext.root(false, false); - final MappedFieldType mapper1 = new TextFieldMapper.Builder( - "s", - createDefaultIndexAnalyzers(), - SourceFieldMapper.isSynthetic(indexService.getIndexSettings()) - ).fielddata(true).build(context).fieldType(); + final MappedFieldType mapper1 = new TextFieldMapper.Builder("s", createDefaultIndexAnalyzers()).fielddata(true) + .build(context) + .fieldType(); final IndexWriter writer = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer())); Document doc = new Document(); doc.add(new StringField("s", "thisisastring", Store.NO)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java index 75c984d6f4305..36d1ef22425a2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java @@ -27,9 +27,9 @@ public class DocumentParserContextTests extends ESTestCase { private final MapperBuilderContext root = MapperBuilderContext.root(false, false); public void testDynamicMapperSizeMultipleMappers() { - context.addDynamicMapper(new TextFieldMapper.Builder("foo", createDefaultIndexAnalyzers(), false).build(root)); + context.addDynamicMapper(new TextFieldMapper.Builder("foo", createDefaultIndexAnalyzers()).build(root)); assertEquals(1, context.getNewFieldsSize()); - context.addDynamicMapper(new TextFieldMapper.Builder("bar", createDefaultIndexAnalyzers(), false).build(root)); + context.addDynamicMapper(new TextFieldMapper.Builder("bar", createDefaultIndexAnalyzers()).build(root)); assertEquals(2, context.getNewFieldsSize()); context.addDynamicRuntimeField(new TestRuntimeField("runtime1", "keyword")); assertEquals(3, context.getNewFieldsSize()); @@ -44,9 +44,9 @@ public void testDynamicMapperSizeSameFieldMultipleRuntimeFields() { } public void testDynamicMapperSizeSameFieldMultipleMappers() { - context.addDynamicMapper(new TextFieldMapper.Builder("foo", createDefaultIndexAnalyzers(), false).build(root)); + context.addDynamicMapper(new TextFieldMapper.Builder("foo", createDefaultIndexAnalyzers()).build(root)); assertEquals(1, context.getNewFieldsSize()); - context.addDynamicMapper(new TextFieldMapper.Builder("foo", createDefaultIndexAnalyzers(), false).build(root)); + context.addDynamicMapper(new TextFieldMapper.Builder("foo", createDefaultIndexAnalyzers()).build(root)); assertEquals(1, context.getNewFieldsSize()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java index 132d24f2389bf..a2e37b2fb066d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java @@ -161,15 +161,7 @@ public void testFieldAliasWithDifferentNestedScopes() { private static FieldMapper createFieldMapper(String parent, String name) { return new BooleanFieldMapper.Builder(name, ScriptCompiler.NONE, false, IndexVersion.current(), null).build( - new MapperBuilderContext( - parent, - false, - false, - false, - ObjectMapper.Defaults.DYNAMIC, - MapperService.MergeReason.MAPPING_UPDATE, - false - ) + MapperBuilderContext.builder().path(parent).isSourceSynthetic(false).build() ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index 05a6c24a2b743..25e1079f38aaf 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -247,7 +247,8 @@ public void testFetchSourceValue() throws IOException { ScriptCompiler.NONE, Integer.MAX_VALUE, IndexVersion.current(), - randomFrom(Mapper.SourceKeepMode.values()) + randomFrom(Mapper.SourceKeepMode.values()), + false ).normalizer("lowercase").build(MapperBuilderContext.root(false, false)).fieldType(); assertEquals(List.of("value"), fetchSourceValue(normalizerMapper, "VALUE")); assertEquals(List.of("42"), fetchSourceValue(normalizerMapper, 42L)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java deleted file mode 100644 index 4c5bfeb66b075..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.mapper; - -import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.analysis.IndexAnalyzers; -import org.elasticsearch.script.ScriptCompiler; -import org.elasticsearch.test.ESTestCase; - -import java.util.Map; - -import static org.elasticsearch.index.mapper.MapperService.MergeReason.MAPPING_UPDATE; - -public class MultiFieldsTests extends ESTestCase { - - public void testMultiFieldsBuilderHasSyntheticSourceCompatibleKeywordField() { - var isStored = randomBoolean(); - var hasNormalizer = randomBoolean(); - - var builder = new FieldMapper.MultiFields.Builder(); - assertFalse(builder.hasSyntheticSourceCompatibleKeywordField()); - - var keywordFieldMapperBuilder = getKeywordFieldMapperBuilder(isStored, hasNormalizer); - builder.add(keywordFieldMapperBuilder); - - var expected = hasNormalizer == false; - assertEquals(expected, builder.hasSyntheticSourceCompatibleKeywordField()); - } - - public void testMultiFieldsBuilderHasSyntheticSourceCompatibleKeywordFieldDuringMerge() { - var isStored = randomBoolean(); - var hasNormalizer = randomBoolean(); - - var builder = new TextFieldMapper.Builder("text_field", createDefaultIndexAnalyzers(), false); - assertFalse(builder.multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField()); - - var keywordFieldMapperBuilder = getKeywordFieldMapperBuilder(isStored, hasNormalizer); - - var newField = new TextFieldMapper.Builder("text_field", createDefaultIndexAnalyzers(), false).addMultiField( - keywordFieldMapperBuilder - ).build(MapperBuilderContext.root(false, false)); - - builder.merge( - newField, - new FieldMapper.Conflicts("TextFieldMapper"), - MapperMergeContext.root(false, false, MAPPING_UPDATE, Long.MAX_VALUE) - ); - - var expected = hasNormalizer == false; - assertEquals(expected, builder.multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField()); - } - - private KeywordFieldMapper.Builder getKeywordFieldMapperBuilder(boolean isStored, boolean hasNormalizer) { - var keywordFieldMapperBuilder = new KeywordFieldMapper.Builder( - "field", - IndexAnalyzers.of(Map.of(), Map.of("normalizer", Lucene.STANDARD_ANALYZER), Map.of()), - ScriptCompiler.NONE, - Integer.MAX_VALUE, - IndexVersion.current(), - Mapper.SourceKeepMode.NONE - ); - if (isStored) { - keywordFieldMapperBuilder.stored(true); - if (randomBoolean()) { - keywordFieldMapperBuilder.docValues(false); - } - } - if (hasNormalizer) { - keywordFieldMapperBuilder.normalizer("normalizer"); - } - return keywordFieldMapperBuilder; - } -} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java index 7402405f5b883..e06820bab50c2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java @@ -37,10 +37,10 @@ private RootObjectMapper createMapping( rootBuilder.add(new ObjectMapper.Builder("disabled", Optional.empty()).enabled(disabledFieldEnabled)); ObjectMapper.Builder fooBuilder = new ObjectMapper.Builder("foo", Optional.empty()).enabled(fooFieldEnabled); if (includeBarField) { - fooBuilder.add(new TextFieldMapper.Builder("bar", createDefaultIndexAnalyzers(), false)); + fooBuilder.add(new TextFieldMapper.Builder("bar", createDefaultIndexAnalyzers())); } if (includeBazField) { - fooBuilder.add(new TextFieldMapper.Builder("baz", createDefaultIndexAnalyzers(), false)); + fooBuilder.add(new TextFieldMapper.Builder("baz", createDefaultIndexAnalyzers())); } rootBuilder.add(fooBuilder); return rootBuilder.build(MapperBuilderContext.root(false, false)); @@ -365,15 +365,7 @@ private static RootObjectMapper createRootSubobjectFalseLeafWithDots() { private static ObjectMapper.Builder createObjectSubobjectsFalseLeafWithDots() { KeywordFieldMapper.Builder fieldBuilder = new KeywordFieldMapper.Builder("host.name", IndexVersion.current()); KeywordFieldMapper fieldMapper = fieldBuilder.build( - new MapperBuilderContext( - "foo.metrics", - false, - false, - false, - ObjectMapper.Defaults.DYNAMIC, - MapperService.MergeReason.MAPPING_UPDATE, - false - ) + MapperBuilderContext.builder().path("foo.metrics").isSourceSynthetic(false).isDataStream(false).build() ); assertEquals("host.name", fieldMapper.leafName()); assertEquals("foo.metrics.host.name", fieldMapper.fullPath()); @@ -385,15 +377,7 @@ private static ObjectMapper.Builder createObjectSubobjectsFalseLeafWithDots() { private ObjectMapper.Builder createObjectSubobjectsFalseLeafWithMultiField() { TextFieldMapper.Builder fieldBuilder = createTextKeywordMultiField("host.name"); TextFieldMapper textKeywordMultiField = fieldBuilder.build( - new MapperBuilderContext( - "foo.metrics", - false, - false, - false, - ObjectMapper.Defaults.DYNAMIC, - MapperService.MergeReason.MAPPING_UPDATE, - false - ) + MapperBuilderContext.builder().path("foo.metrics").isSourceSynthetic(false).isDataStream(false).build() ); assertEquals("host.name", textKeywordMultiField.leafName()); assertEquals("foo.metrics.host.name", textKeywordMultiField.fullPath()); @@ -410,7 +394,7 @@ private TextFieldMapper.Builder createTextKeywordMultiField(String name) { } private TextFieldMapper.Builder createTextKeywordMultiField(String name, String multiFieldName) { - TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name, createDefaultIndexAnalyzers(), false); + TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name, createDefaultIndexAnalyzers()); builder.multiFieldsBuilder.add(new KeywordFieldMapper.Builder(multiFieldName, IndexVersion.current())); return builder; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java index 6dfcf5d7399d2..0216149f9a99b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java @@ -735,13 +735,14 @@ public void testStoreArraySourceNoopInNonSyntheticSourceMode() throws IOExceptio public void testNestedObjectWithMultiFieldsgetTotalFieldsCount() { ObjectMapper.Builder mapperBuilder = new ObjectMapper.Builder("parent_size_1", Optional.empty()).add( new ObjectMapper.Builder("child_size_2", Optional.empty()).add( - new TextFieldMapper.Builder("grand_child_size_3", createDefaultIndexAnalyzers(), false).addMultiField( - new KeywordFieldMapper.Builder("multi_field_size_4", IndexVersion.current()) + new TextFieldMapper.Builder("grand_child_size_3", createDefaultIndexAnalyzers()).addMultiField( + new KeywordFieldMapper.Builder("multi_field_size_4", IndexVersion.current(), true) ) .addMultiField( - new TextFieldMapper.Builder("grand_child_size_5", createDefaultIndexAnalyzers(), false).addMultiField( - new KeywordFieldMapper.Builder("multi_field_of_multi_field_size_6", IndexVersion.current()) - ) + new TextFieldMapper.Builder("grand_child_size_5", IndexVersion.current(), createDefaultIndexAnalyzers(), true) + .addMultiField( + new KeywordFieldMapper.Builder("multi_field_of_multi_field_size_6", IndexVersion.current(), true) + ) ) ) ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index 4d246d3c557a6..e784fe6ce38e0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -48,7 +48,7 @@ public class TextFieldTypeTests extends FieldTypeTestCase { private static TextFieldType createFieldType() { - return new TextFieldType("field", randomBoolean()); + return new TextFieldType("field", randomBoolean(), false); } public void testIsAggregatableDependsOnFieldData() { diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 0c31ab703862f..8b7ca1dca8ff8 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -113,7 +113,7 @@ public class SearchExecutionContextTests extends ESTestCase { public void testFailIfFieldMappingNotFound() { SearchExecutionContext context = createSearchExecutionContext(IndexMetadata.INDEX_UUID_NA_VALUE, null); context.setAllowUnmappedFields(false); - MappedFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); + MappedFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean(), false); MappedFieldType result = context.failIfFieldMappingNotFound("name", fieldType); assertThat(result, sameInstance(fieldType)); QueryShardException e = expectThrows(QueryShardException.class, () -> context.failIfFieldMappingNotFound("name", null)); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java index e7d19c0f56dbc..7fb45a12dcffe 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java @@ -426,7 +426,7 @@ public void testRangeFilter() throws IOException { * Tests a filter that needs the cache to be fast. */ public void testPhraseFilter() throws IOException { - MappedFieldType ft = new TextFieldMapper.TextFieldType("test", randomBoolean()); + MappedFieldType ft = new TextFieldMapper.TextFieldType("test", randomBoolean(), false); AggregationBuilder builder = new FiltersAggregationBuilder( "test", new KeyedFilter("q1", new MatchPhraseQueryBuilder("test", "will find me").slop(0)) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorTests.java index 60f688a88eb49..4e816f1111fd5 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorTests.java @@ -41,7 +41,7 @@ public class SamplerAggregatorTests extends AggregatorTestCase { * Uses the sampler aggregation to find the minimum value of a field out of the top 3 scoring documents in a search. */ public void testSampler() throws IOException { - TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean(), false); MappedFieldType numericFieldType = new NumberFieldMapper.NumberFieldType("int", NumberFieldMapper.NumberType.LONG); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); @@ -75,7 +75,7 @@ public void testSampler() throws IOException { } public void testRidiculousSize() throws IOException { - TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean(), false); MappedFieldType numericFieldType = new NumberFieldMapper.NumberFieldType("int", NumberFieldMapper.NumberType.LONG); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java index 4d421b84de95e..504b757ffa42f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java @@ -106,7 +106,7 @@ protected List unsupportedMappedFieldTypes() { } public void testSignificance(SignificanceHeuristic heuristic) throws IOException { - TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean(), false); textFieldType.setFielddata(true); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); @@ -203,7 +203,7 @@ public void testSignificance() throws IOException { * @throws IOException on test setup failure */ public void testSamplingConsistency() throws IOException { - TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean(), false); textFieldType.setFielddata(true); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); @@ -305,7 +305,7 @@ public void testNumericSignificance() throws IOException { * Uses the significant terms aggregation on an index with unmapped field */ public void testUnmapped() throws IOException { - TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean(), false); textFieldType.setFielddata(true); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); @@ -370,7 +370,7 @@ public void testRangeField() throws IOException { } public void testFieldAlias() throws IOException { - TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean(), false); textFieldType.setFielddata(true); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); @@ -424,7 +424,7 @@ public void testFieldAlias() throws IOException { } public void testFieldBackground() throws IOException { - TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean(), false); textFieldType.setFielddata(true); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java index c076b2b62a0e2..615b6d1116189 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java @@ -70,7 +70,7 @@ protected List unsupportedMappedFieldTypes() { * Uses the significant text aggregation to find the keywords in text fields */ public void testSignificance() throws IOException { - TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean(), false); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); @@ -123,7 +123,7 @@ public void testSignificance() throws IOException { * Uses the significant text aggregation to find the keywords in text fields and include/exclude selected terms */ public void testIncludeExcludes() throws IOException { - TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean(), false); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); @@ -182,7 +182,7 @@ public void testIncludeExcludes() throws IOException { } public void testMissingField() throws IOException { - TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean(), false); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); indexWriterConfig.setMaxBufferedDocs(100); @@ -209,7 +209,7 @@ public void testMissingField() throws IOException { } public void testFieldAlias() throws IOException { - TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean(), false); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); @@ -267,7 +267,7 @@ public void testFieldAlias() throws IOException { } public void testInsideTermsAgg() throws IOException { - TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean(), false); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); @@ -322,7 +322,7 @@ private void indexDocuments(IndexWriter writer) throws IOException { * Test documents with arrays of text */ public void testSignificanceOnTextArrays() throws IOException { - TextFieldType textFieldType = new TextFieldType("text", randomBoolean()); + TextFieldType textFieldType = new TextFieldType("text", randomBoolean(), false); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index bc4b4765e9d78..d5633e9d36d44 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MappingLookup; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.IdsQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; @@ -324,11 +323,7 @@ public void testBuildSearchContextHighlight() throws IOException { ) { @Override public MappedFieldType getFieldType(String name) { - TextFieldMapper.Builder builder = new TextFieldMapper.Builder( - name, - createDefaultIndexAnalyzers(), - SourceFieldMapper.isSynthetic(idxSettings) - ); + TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name, createDefaultIndexAnalyzers()); return builder.build(MapperBuilderContext.root(false, false)).fieldType(); } }; diff --git a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java index 209dfdcc16969..0e69f164ac021 100644 --- a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MappingLookup; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -164,11 +163,7 @@ public void testBuildRescoreSearchContext() throws ElasticsearchParseException, ) { @Override public MappedFieldType getFieldType(String name) { - TextFieldMapper.Builder builder = new TextFieldMapper.Builder( - name, - createDefaultIndexAnalyzers(), - SourceFieldMapper.isSynthetic(idxSettings) - ); + TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name, createDefaultIndexAnalyzers()); return builder.build(MapperBuilderContext.root(false, false)).fieldType(); } }; @@ -231,11 +226,7 @@ public void testRewritingKeepsSettings() throws IOException { ) { @Override public MappedFieldType getFieldType(String name) { - TextFieldMapper.Builder builder = new TextFieldMapper.Builder( - name, - createDefaultIndexAnalyzers(), - SourceFieldMapper.isSynthetic(idxSettings) - ); + TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name, createDefaultIndexAnalyzers()); return builder.build(MapperBuilderContext.root(false, false)).fieldType(); } }; diff --git a/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java b/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java index 6b53daeb40c4e..bfae91cf39fcc 100644 --- a/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java +++ b/server/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java @@ -806,7 +806,7 @@ public void testParsingContextFromDocument() { assertTrue(context.contains("category1")); document = new LuceneDocument(); - TextFieldMapper.TextFieldType text = new TextFieldMapper.TextFieldType("category", randomBoolean()); + TextFieldMapper.TextFieldType text = new TextFieldMapper.TextFieldType("category", randomBoolean(), false); document.add(new Field(text.name(), "category1", TextFieldMapper.Defaults.FIELD_TYPE)); // Ignore stored field document.add(new StoredField(text.name(), "category1", TextFieldMapper.Defaults.FIELD_TYPE)); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java index b979544ac0d6a..419a8b03f22c6 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregatorTests.java @@ -114,7 +114,7 @@ public void testUnmappedWithMissingField() throws IOException { } public void testMissing() throws IOException { - final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); + final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean(), false); fieldType.setFielddata(true); final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").field(fieldType.name()) @@ -190,7 +190,7 @@ public void testQueryFiltering() throws IOException { @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/47469") public void testSingleValuedFieldWithFormatter() throws IOException { - TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); + TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean(), false); fieldType.setFielddata(true); StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").field("text") @@ -216,7 +216,7 @@ public void testSingleValuedFieldWithFormatter() throws IOException { public void testNestedAggregation() throws IOException { MappedFieldType numericFieldType = new NumberFieldMapper.NumberFieldType("value", NumberFieldMapper.NumberType.INTEGER); - TextFieldMapper.TextFieldType textFieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); + TextFieldMapper.TextFieldType textFieldType = new TextFieldMapper.TextFieldType("text", randomBoolean(), false); textFieldType.setFielddata(true); TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.NUMERIC) @@ -266,7 +266,7 @@ public void testNestedAggregation() throws IOException { } public void testValueScriptSingleValuedField() throws IOException { - final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); + final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean(), false); fieldType.setFielddata(true); final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").field(fieldType.name()) @@ -288,7 +288,7 @@ public void testValueScriptSingleValuedField() throws IOException { } public void testValueScriptMultiValuedField() throws IOException { - final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); + final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean(), false); fieldType.setFielddata(true); final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").field(fieldType.name()) @@ -315,7 +315,7 @@ public void testValueScriptMultiValuedField() throws IOException { } public void testFieldScriptSingleValuedField() throws IOException { - final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); + final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean(), false); fieldType.setFielddata(true); final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").script( @@ -338,7 +338,7 @@ public void testFieldScriptSingleValuedField() throws IOException { } public void testFieldScriptMultiValuedField() throws IOException { - final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); + final TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean(), false); fieldType.setFielddata(true); final StringStatsAggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").script( @@ -370,7 +370,7 @@ private void testAggregation( CheckedConsumer buildIndex, Consumer verify ) throws IOException { - TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean()); + TextFieldMapper.TextFieldType fieldType = new TextFieldMapper.TextFieldType("text", randomBoolean(), false); fieldType.setFielddata(true); AggregationBuilder aggregationBuilder = new StringStatsAggregationBuilder("_name").field("text"); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java index b1311736bf976..1e8c6a5698853 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java @@ -519,7 +519,7 @@ private MappedFieldType numberFieldType(NumberType numberType, String name) { } private MappedFieldType textFieldType(String name) { - return new TextFieldMapper.TextFieldType(name, randomBoolean()); + return new TextFieldMapper.TextFieldType(name, randomBoolean(), false); } private MappedFieldType geoPointFieldType(String name) { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java index 19a645c146242..91228aba9da8f 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java @@ -1587,7 +1587,8 @@ private TextFieldMapper.TextFieldType storedTextField(String name) { false, true, new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), - true, // TODO randomize - if the field is stored we should load from the stored field even if there is source + true, // TODO randomize - if the field is stored we should load from the stored field even if there is source] + false, null, Map.of(), false, @@ -1602,6 +1603,7 @@ private TextFieldMapper.TextFieldType textFieldWithDelegate(String name, Keyword false, new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), randomBoolean(), + false, delegate, Map.of(), false, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/EqualsSyntheticSourceDelegate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/EqualsSyntheticSourceDelegate.java index 812e741c64d29..d3b646b9473a3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/EqualsSyntheticSourceDelegate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/EqualsSyntheticSourceDelegate.java @@ -48,7 +48,7 @@ private Builder(String name, String value) { @Override protected org.apache.lucene.search.Query doToQuery(SearchExecutionContext context) { TextFieldMapper.TextFieldType ft = (TextFieldMapper.TextFieldType) context.getFieldType(fieldName); - return ft.syntheticSourceDelegate().termQuery(value, context); + return ft.syntheticSourceDelegate().orElse(null).termQuery(value, context); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java index ca1816feba426..123360d65d269 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java @@ -337,7 +337,7 @@ protected final org.apache.lucene.search.Query doToQuery(SearchExecutionContext if (ft == null) { return new MatchNoDocsQuery("missing field [" + field() + "]"); } - ft = ((TextFieldMapper.TextFieldType) ft).syntheticSourceDelegate(); + ft = ((TextFieldMapper.TextFieldType) ft).syntheticSourceDelegate().orElse(null); BooleanQuery.Builder builder = new BooleanQuery.Builder(); builder.add(next().toQuery(context), BooleanClause.Occur.FILTER); @@ -437,7 +437,7 @@ protected final org.apache.lucene.search.Query doToQuery(SearchExecutionContext if (ft == null) { return new MatchNoDocsQuery("missing field [" + field() + "]"); } - ft = ((TextFieldMapper.TextFieldType) ft).syntheticSourceDelegate(); + ft = ((TextFieldMapper.TextFieldType) ft).syntheticSourceDelegate().orElse(null); org.apache.lucene.search.Query svNext = simple(ft, context); org.apache.lucene.search.Query ignored = new TermQuery(new org.apache.lucene.index.Term(IgnoredFieldMapper.NAME, ft.name())); diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java index 3e0bc05430835..3dfda01d1e063 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java @@ -24,6 +24,7 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MappingParserContext; +import org.elasticsearch.index.mapper.TextFamilyFieldMapper; import org.elasticsearch.index.mapper.TextParams; import org.elasticsearch.index.mapper.TextSearchInfo; @@ -36,7 +37,7 @@ /** * A {@link FieldMapper} that assigns every document the same value. */ -public class PatternedTextFieldMapper extends FieldMapper { +public class PatternedTextFieldMapper extends TextFamilyFieldMapper { public static final FeatureFlag PATTERNED_TEXT_MAPPER = new FeatureFlag("patterned_text"); @@ -60,12 +61,27 @@ public static class Builder extends FieldMapper.Builder { private final IndexSettings indexSettings; private final Parameter> meta = Parameter.metaParam(); private final TextParams.Analyzers analyzers; + private final boolean isWithinMultiField; + + private boolean isSyntheticSourceEnabled; public Builder(String name, MappingParserContext context) { - this(name, context.indexVersionCreated(), context.getIndexSettings(), context.getIndexAnalyzers()); + this( + name, + context.indexVersionCreated(), + context.getIndexSettings(), + context.getIndexAnalyzers(), + context.isWithinMultiField() + ); } - public Builder(String name, IndexVersion indexCreatedVersion, IndexSettings indexSettings, IndexAnalyzers indexAnalyzers) { + public Builder( + String name, + IndexVersion indexCreatedVersion, + IndexSettings indexSettings, + IndexAnalyzers indexAnalyzers, + boolean isWithinMultiField + ) { super(name); this.indexCreatedVersion = indexCreatedVersion; this.indexSettings = indexSettings; @@ -75,6 +91,7 @@ public Builder(String name, IndexVersion indexCreatedVersion, IndexSettings inde m -> ((PatternedTextFieldMapper) m).positionIncrementGap, indexCreatedVersion ); + this.isWithinMultiField = isWithinMultiField; } @Override @@ -92,20 +109,25 @@ private PatternedTextFieldType buildFieldType(MapperBuilderContext context) { tsi, indexAnalyzer, context.isSourceSynthetic(), + isWithinMultiField, meta.getValue() ); } @Override public PatternedTextFieldMapper build(MapperBuilderContext context) { + this.isSyntheticSourceEnabled = context.isSourceSynthetic(); + PatternedTextFieldType patternedTextFieldType = buildFieldType(context); BuilderParams builderParams = builderParams(this, context); var templateIdMapper = KeywordFieldMapper.Builder.buildWithDocValuesSkipper( patternedTextFieldType.templateIdFieldName(), indexSettings.getMode(), indexCreatedVersion, - true + true, + isWithinMultiField ).indexed(false).build(context); + return new PatternedTextFieldMapper(leafName(), patternedTextFieldType, builderParams, this, templateIdMapper); } } @@ -127,9 +149,18 @@ private PatternedTextFieldMapper( Builder builder, KeywordFieldMapper templateIdMapper ) { - super(simpleName, mappedFieldPatternedTextFieldType, builderParams); + super( + simpleName, + builder.indexCreatedVersion, + builder.isSyntheticSourceEnabled, + builder.isWithinMultiField, + mappedFieldPatternedTextFieldType, + builderParams + ); + assert mappedFieldPatternedTextFieldType.getTextSearchInfo().isTokenized(); assert mappedFieldPatternedTextFieldType.hasDocValues() == false; + this.fieldType = Defaults.FIELD_TYPE; this.indexCreatedVersion = builder.indexCreatedVersion; this.indexAnalyzers = builder.analyzers.indexAnalyzers; @@ -146,7 +177,7 @@ public Map indexAnalyzers() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), indexCreatedVersion, indexSettings, indexAnalyzers).init(this); + return new Builder(leafName(), indexCreatedVersion, indexSettings, indexAnalyzers, isWithinMultiField).init(this); } @Override diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java index e23e1428fbe24..017c924f158b6 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java @@ -63,12 +63,19 @@ public class PatternedTextFieldType extends StringFieldType { private final Analyzer indexAnalyzer; private final TextFieldMapper.TextFieldType textFieldType; - PatternedTextFieldType(String name, TextSearchInfo tsi, Analyzer indexAnalyzer, boolean isSyntheticSource, Map meta) { + PatternedTextFieldType( + String name, + TextSearchInfo tsi, + Analyzer indexAnalyzer, + boolean isSyntheticSource, + boolean isWithinMultiField, + Map meta + ) { // Though this type is based on doc_values, hasDocValues is set to false as the patterned_text type is not aggregatable. // This does not stop its child .template type from being aggregatable. - super(name, true, false, false, tsi, meta); + super(name, true, false, false, tsi, meta, isSyntheticSource, isWithinMultiField); this.indexAnalyzer = Objects.requireNonNull(indexAnalyzer); - this.textFieldType = new TextFieldMapper.TextFieldType(name, isSyntheticSource); + this.textFieldType = new TextFieldMapper.TextFieldType(name, isSyntheticSource, isWithinMultiField); } PatternedTextFieldType(String name) { @@ -77,6 +84,7 @@ public class PatternedTextFieldType extends StringFieldType { new TextSearchInfo(PatternedTextFieldMapper.Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), Lucene.STANDARD_ANALYZER, false, + false, Collections.emptyMap() ); } @@ -247,7 +255,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext if (fieldDataContext.fielddataOperation() != FielddataOperation.SCRIPT) { throw new IllegalArgumentException(CONTENT_TYPE + " fields do not support sorting and aggregations"); } - if (textFieldType.isSyntheticSource()) { + if (textFieldType.isSyntheticSourceEnabled()) { return new PatternedTextIndexFieldData.Builder(this); } return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java index 29f298894477a..5a3d102cf3433 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregatorTests.java @@ -80,7 +80,7 @@ public void testCategorizationWithoutSubAggs() throws Exception { }, new AggTestConfig( new CategorizeTextAggregationBuilder("my_agg", TEXT_FIELD_NAME), - new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean()), + new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean(), false), longField(NUMERIC_FIELD_NAME) ) ); @@ -118,7 +118,7 @@ public void testCategorizationWithSubAggs() throws Exception { }, new AggTestConfig( aggBuilder, - new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean()), + new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean(), false), longField(NUMERIC_FIELD_NAME) ) ); @@ -173,7 +173,7 @@ public void testCategorizationWithMultiBucketSubAggs() throws Exception { }, new AggTestConfig( aggBuilder, - new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean()), + new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean(), false), longField(NUMERIC_FIELD_NAME) ) ); @@ -261,7 +261,7 @@ public void testCategorizationAsSubAgg() throws Exception { }, new AggTestConfig( aggBuilder, - new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean()), + new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean(), false), longField(NUMERIC_FIELD_NAME) ) ); @@ -316,7 +316,7 @@ public void testCategorizationWithSubAggsManyDocs() throws Exception { }, new AggTestConfig( aggBuilder, - new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean()), + new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean(), false), longField(NUMERIC_FIELD_NAME) ) ); @@ -400,7 +400,7 @@ public void testCategorizationAsSubAggWithExtendedBounds() throws Exception { }, new AggTestConfig( aggBuilder, - new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean()), + new TextFieldMapper.TextFieldType(TEXT_FIELD_NAME, randomBoolean(), false), longField(NUMERIC_FIELD_NAME) ) );