diff --git a/docs/changelog/124594.yaml b/docs/changelog/124594.yaml new file mode 100644 index 0000000000000..08417c1304c38 --- /dev/null +++ b/docs/changelog/124594.yaml @@ -0,0 +1,5 @@ +pr: 124594 +summary: Store arrays offsets for numeric fields natively with synthetic source +area: Mapping +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 68c41de6f4787..82d706b4cf996 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -126,6 +126,7 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT = def(8_525_0_00, Version.LUCENE_9_12_1); public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT_BACKPORT = def(8_526_0_00, Version.LUCENE_9_12_1); public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD = def(8_527_0_00, Version.LUCENE_9_12_1); + public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_NUMBER = def(8_528_0_00, Version.LUCENE_9_12_1); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java index 0793dd748c67e..9e7a23cdeaee8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java @@ -352,7 +352,8 @@ public boolean newDynamicLongField(DocumentParserContext context, String name) t ScriptCompiler.NONE, context.indexSettings().getSettings(), context.indexSettings().getIndexVersionCreated(), - context.indexSettings().getMode() + context.indexSettings().getMode(), + context.indexSettings().sourceKeepMode() ), context ); @@ -370,7 +371,8 @@ public boolean newDynamicDoubleField(DocumentParserContext context, String name) ScriptCompiler.NONE, context.indexSettings().getSettings(), context.indexSettings().getIndexVersionCreated(), - context.indexSettings().getMode() + context.indexSettings().getMode(), + context.indexSettings().sourceKeepMode() ), context ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldArrayContext.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldArrayContext.java index f344d51fb38b8..e942fbbcd3939 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldArrayContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldArrayContext.java @@ -14,7 +14,6 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.IndexVersions; import java.io.IOException; import java.util.ArrayList; @@ -89,16 +88,18 @@ static String getOffsetsFieldName( boolean hasDocValues, boolean isStored, FieldMapper.Builder fieldMapperBuilder, - IndexVersion indexCreatedVersion + IndexVersion indexCreatedVersion, + IndexVersion minSupportedVersion ) { var sourceKeepMode = fieldMapperBuilder.sourceKeepMode.orElse(indexSourceKeepMode); if (context.isSourceSynthetic() && sourceKeepMode == Mapper.SourceKeepMode.ARRAYS && hasDocValues && isStored == false + && context.isInNestedContext() == false && fieldMapperBuilder.copyTo.copyToFields().isEmpty() && fieldMapperBuilder.multiFieldsBuilder.hasMultiFields() == false - && indexCreatedVersion.onOrAfter(IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD)) { + && indexCreatedVersion.onOrAfter(minSupportedVersion)) { // Skip stored, we will be synthesizing from stored fields, no point to keep track of the offsets // Skip copy_to and multi fields, supporting that requires more work. However, copy_to usage is rare in metrics and // logging use cases diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index e1645a87abc7f..2bf3dc4d218f2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -200,7 +200,8 @@ public IpFieldMapper build(MapperBuilderContext context) { hasDocValues.getValue(), stored.getValue(), this, - indexCreatedVersion + indexCreatedVersion, + IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD ); return new IpFieldMapper( leafName(), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index aff1428e8cd9b..b62ce5d2f6ff8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -41,6 +41,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; @@ -389,7 +390,8 @@ public KeywordFieldMapper build(MapperBuilderContext context) { hasDocValues.getValue(), stored.getValue(), this, - indexCreatedVersion + indexCreatedVersion, + IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD ); return new KeywordFieldMapper( leafName(), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index f798a5a6f83ff..b94c6678d8025 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; @@ -69,6 +70,7 @@ import java.io.IOException; import java.math.BigDecimal; import java.time.ZoneId; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -79,6 +81,8 @@ import java.util.function.BiFunction; import java.util.function.Function; +import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName; + /** A {@link FieldMapper} for numeric types: byte, short, int, long, float and double. */ public class NumberFieldMapper extends FieldMapper { @@ -128,6 +132,7 @@ public static final class Builder extends FieldMapper.DimensionBuilder { private final IndexVersion indexCreatedVersion; private final IndexMode indexMode; + private final SourceKeepMode indexSourceKeepMode; public Builder( String name, @@ -135,13 +140,23 @@ public Builder( ScriptCompiler compiler, Settings settings, IndexVersion indexCreatedVersion, - IndexMode mode + IndexMode mode, + SourceKeepMode indexSourceKeepMode ) { - this(name, type, compiler, IGNORE_MALFORMED_SETTING.get(settings), COERCE_SETTING.get(settings), indexCreatedVersion, mode); + this( + name, + type, + compiler, + IGNORE_MALFORMED_SETTING.get(settings), + COERCE_SETTING.get(settings), + indexCreatedVersion, + mode, + indexSourceKeepMode + ); } public static Builder docValuesOnly(String name, NumberType type, IndexVersion indexCreatedVersion) { - Builder builder = new Builder(name, type, ScriptCompiler.NONE, false, false, indexCreatedVersion, null); + Builder builder = new Builder(name, type, ScriptCompiler.NONE, false, false, indexCreatedVersion, null, null); builder.indexed.setValue(false); builder.dimension.setValue(false); return builder; @@ -154,7 +169,8 @@ public Builder( boolean ignoreMalformedByDefault, boolean coerceByDefault, IndexVersion indexCreatedVersion, - IndexMode mode + IndexMode mode, + SourceKeepMode indexSourceKeepMode ) { super(name); this.type = type; @@ -210,6 +226,8 @@ public Builder( this.script.precludesParameters(ignoreMalformed, coerce, nullValue); addScriptValidation(script, indexed, hasDocValues); + + this.indexSourceKeepMode = indexSourceKeepMode; } Builder nullValue(Number number) { @@ -273,7 +291,16 @@ public NumberFieldMapper build(MapperBuilderContext context) { MappedFieldType ft = new NumberFieldType(context.buildFullName(leafName()), this, context.isSourceSynthetic()); hasScript = script.get() != null; onScriptError = onScriptErrorParam.getValue(); - return new NumberFieldMapper(leafName(), ft, builderParams(this, context), context.isSourceSynthetic(), this); + String offsetsFieldName = getOffsetsFieldName( + context, + indexSourceKeepMode, + hasDocValues.getValue(), + stored.getValue(), + this, + indexCreatedVersion, + IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_NUMBER + ); + return new NumberFieldMapper(leafName(), ft, builderParams(this, context), context.isSourceSynthetic(), this, offsetsFieldName); } } @@ -412,6 +439,11 @@ public void addFields(LuceneDocument document, String name, Number value, boolea } } + @Override + public long toSortableLong(Number value) { + return HalfFloatPoint.halfFloatToSortableShort(value.floatValue()); + } + @Override public IndexFieldData.Builder getFieldDataBuilder(MappedFieldType ft, ValuesSourceType valuesSourceType) { return new SortedDoublesIndexFieldData.Builder( @@ -446,13 +478,8 @@ private static void validateFiniteValue(float value) { } @Override - SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) { - return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) { - @Override - protected void writeValue(XContentBuilder b, long value) throws IOException { - b.value(HalfFloatPoint.sortableShortToHalfFloat((short) value)); - } - }; + public void writeValue(XContentBuilder b, long value) throws IOException { + b.value(HalfFloatPoint.sortableShortToHalfFloat((short) value)); } @Override @@ -601,6 +628,11 @@ public void addFields(LuceneDocument document, String name, Number value, boolea } } + @Override + public long toSortableLong(Number value) { + return NumericUtils.floatToSortableInt(value.floatValue()); + } + @Override public IndexFieldData.Builder getFieldDataBuilder(MappedFieldType ft, ValuesSourceType valuesSourceType) { return new SortedDoublesIndexFieldData.Builder( @@ -635,13 +667,8 @@ private static void validateFiniteValue(float value) { } @Override - SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) { - return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) { - @Override - protected void writeValue(XContentBuilder b, long value) throws IOException { - b.value(NumericUtils.sortableIntToFloat((int) value)); - } - }; + public void writeValue(XContentBuilder b, long value) throws IOException { + b.value(NumericUtils.sortableIntToFloat((int) value)); } @Override @@ -756,6 +783,11 @@ public void addFields(LuceneDocument document, String name, Number value, boolea } } + @Override + public long toSortableLong(Number value) { + return NumericUtils.doubleToSortableLong(value.doubleValue()); + } + @Override public IndexFieldData.Builder getFieldDataBuilder(MappedFieldType ft, ValuesSourceType valuesSourceType) { return new SortedDoublesIndexFieldData.Builder( @@ -790,13 +822,8 @@ private static void validateParsed(double value) { } @Override - SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) { - return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) { - @Override - protected void writeValue(XContentBuilder b, long value) throws IOException { - b.value(NumericUtils.sortableLongToDouble(value)); - } - }; + public void writeValue(XContentBuilder b, long value) throws IOException { + b.value(NumericUtils.sortableLongToDouble(value)); } @Override @@ -839,12 +866,12 @@ public Number parsePoint(byte[] value) { } @Override - public Short parse(XContentParser parser, boolean coerce) throws IOException { + public Byte parse(XContentParser parser, boolean coerce) throws IOException { int value = parser.intValue(coerce); if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) { throw new IllegalArgumentException("Value [" + value + "] is out of range for a byte"); } - return (short) value; + return (byte) value; } @Override @@ -880,6 +907,11 @@ public void addFields(LuceneDocument document, String name, Number value, boolea INTEGER.addFields(document, name, value, indexed, docValued, stored); } + @Override + public long toSortableLong(Number value) { + return INTEGER.toSortableLong(value); + } + @Override Number valueForSearch(Number value) { return value.byteValue(); @@ -913,8 +945,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder( } @Override - SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) { - return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed); + public void writeValue(XContentBuilder b, long value) throws IOException { + b.value(value); } @Override @@ -998,6 +1030,11 @@ public void addFields(LuceneDocument document, String name, Number value, boolea INTEGER.addFields(document, name, value, indexed, docValued, stored); } + @Override + public long toSortableLong(Number value) { + return INTEGER.toSortableLong(value); + } + @Override Number valueForSearch(Number value) { return value.shortValue(); @@ -1031,8 +1068,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder( } @Override - SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) { - return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed); + public void writeValue(XContentBuilder b, long value) throws IOException { + b.value(value); } @Override @@ -1195,6 +1232,11 @@ public void addFields(LuceneDocument document, String name, Number value, boolea } } + @Override + public long toSortableLong(Number value) { + return value.intValue(); + } + @Override public IndexFieldData.Builder getFieldDataBuilder(MappedFieldType ft, ValuesSourceType valuesSourceType) { return new SortedNumericIndexFieldData.Builder( @@ -1223,8 +1265,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder( } @Override - SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) { - return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed); + public void writeValue(XContentBuilder b, long value) throws IOException { + b.value(value); } @Override @@ -1347,6 +1389,11 @@ public void addFields(LuceneDocument document, String name, Number value, boolea } } + @Override + public long toSortableLong(Number value) { + return value.longValue(); + } + @Override public IndexFieldData.Builder getFieldDataBuilder(MappedFieldType ft, ValuesSourceType valuesSourceType) { return new SortedNumericIndexFieldData.Builder( @@ -1375,8 +1422,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder( } @Override - SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) { - return syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed); + public void writeValue(XContentBuilder b, long value) throws IOException { + b.value(value); } @Override @@ -1434,7 +1481,8 @@ private boolean isOutOfRange(Object value) { c.scriptCompiler(), c.getSettings(), c.indexVersionCreated(), - c.getIndexSettings().getMode() + c.getIndexSettings().getMode(), + c.getIndexSettings().sourceKeepMode() ), MINIMUM_COMPATIBILITY_VERSION ); @@ -1495,6 +1543,13 @@ public abstract void addFields( boolean stored ); + /** + * For a given {@code Number}, returns the sortable long representation that will be stored in the doc values. + * @param value number to convert + * @return sortable long representation + */ + public abstract long toSortableLong(Number value); + public FieldValues compile(String fieldName, Script script, ScriptCompiler compiler) { // only implemented for long and double fields throw new IllegalArgumentException("Unknown parameter [script] for mapper [" + fieldName + "]"); @@ -1667,17 +1722,13 @@ public double reduceToStoredPrecision(double value) { return ((Number) value).doubleValue(); } - abstract SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed); + abstract void writeValue(XContentBuilder builder, long longValue) throws IOException; - private static SourceLoader.SyntheticFieldLoader syntheticLongFieldLoader( - String fieldName, - String fieldSimpleName, - boolean ignoreMalformed - ) { + SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) { return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) { @Override - protected void writeValue(XContentBuilder b, long value) throws IOException { - b.value(value); + public void writeValue(XContentBuilder b, long value) throws IOException { + NumberType.this.writeValue(b, value); } }; } @@ -2044,15 +2095,18 @@ public MetricType getMetricType() { private boolean allowMultipleValues; private final IndexVersion indexCreatedVersion; private final boolean isSyntheticSource; + private final String offsetsFieldName; private final IndexMode indexMode; + private final SourceKeepMode indexSourceKeepMode; private NumberFieldMapper( String simpleName, MappedFieldType mappedFieldType, BuilderParams builderParams, boolean isSyntheticSource, - Builder builder + Builder builder, + String offsetsFieldName ) { super(simpleName, mappedFieldType, builderParams); this.type = builder.type; @@ -2073,6 +2127,8 @@ private NumberFieldMapper( this.indexCreatedVersion = builder.indexCreatedVersion; this.isSyntheticSource = isSyntheticSource; this.indexMode = builder.indexMode; + this.offsetsFieldName = offsetsFieldName; + this.indexSourceKeepMode = builder.indexSourceKeepMode; } boolean coerce() { @@ -2089,6 +2145,11 @@ public NumberFieldType fieldType() { return (NumberFieldType) super.fieldType(); } + @Override + public String getOffsetFieldName() { + return offsetsFieldName; + } + public NumberType type() { return type; } @@ -2117,7 +2178,20 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio } if (value != null) { indexValue(context, value); + } else { + value = fieldType().nullValue; } + if (offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField()) { + if (value != null) { + // We cannot simply cast value to Comparable<> because we need to also capture the potential loss of precision that occurs + // when the value is stored into the doc values. + long sortableLongValue = type.toSortableLong(value); + context.getOffSetContext().recordOffset(offsetsFieldName, sortableLongValue); + } else { + context.getOffSetContext().recordNull(offsetsFieldName); + } + } + } /** @@ -2178,11 +2252,16 @@ protected void indexScriptValues( @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), type, scriptCompiler, ignoreMalformedByDefault, coerceByDefault, indexCreatedVersion, indexMode) - .dimension(dimension) - .metric(metricType) - .allowMultipleValues(allowMultipleValues) - .init(this); + return new Builder( + leafName(), + type, + scriptCompiler, + ignoreMalformedByDefault, + coerceByDefault, + indexCreatedVersion, + indexMode, + indexSourceKeepMode + ).dimension(dimension).metric(metricType).allowMultipleValues(allowMultipleValues).init(this); } @Override @@ -2194,10 +2273,23 @@ public void doValidate(MappingLookup lookup) { } } + private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() { + if (offsetsFieldName != null) { + var layers = new ArrayList(); + layers.add(new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(fullPath(), offsetsFieldName, type::writeValue)); + if (ignoreMalformed.value()) { + layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath())); + } + return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers); + } else { + return type.syntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()); + } + } + @Override protected SyntheticSourceSupport syntheticSourceSupport() { if (hasDocValues) { - return new SyntheticSourceSupport.Native(() -> type.syntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value())); + return new SyntheticSourceSupport.Native(this::docValuesSyntheticFieldLoader); } return super.syntheticSourceSupport(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer.java b/server/src/main/java/org/elasticsearch/index/mapper/SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer.java new file mode 100644 index 0000000000000..9469a2628e63b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer.java @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.elasticsearch.common.io.stream.ByteArrayStreamInput; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; + +class SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer implements CompositeSyntheticFieldLoader.DocValuesLayer { + @FunctionalInterface + interface NumericValueWriter { + void writeLongValue(XContentBuilder b, long value) throws IOException; + } + + private final String fullPath; + private final String offsetsFieldName; + private final NumericValueWriter valueWriter; + private NumericDocValuesWithOffsetsLoader docValuesLoader; + + SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(String fullPath, String offsetsFieldName, NumericValueWriter valueWriter) { + this.fullPath = fullPath; + this.offsetsFieldName = offsetsFieldName; + this.valueWriter = valueWriter; + } + + @Override + public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException { + SortedNumericDocValues valueDocValues = DocValues.getSortedNumeric(leafReader, fullPath); + SortedDocValues offsetDocValues = DocValues.getSorted(leafReader, offsetsFieldName); + + return docValuesLoader = new NumericDocValuesWithOffsetsLoader(valueDocValues, offsetDocValues, valueWriter); + } + + @Override + public boolean hasValue() { + return docValuesLoader != null && docValuesLoader.hasValue(); + } + + @Override + public long valueCount() { + if (docValuesLoader != null) { + return docValuesLoader.count(); + } else { + return 0; + } + } + + @Override + public void write(XContentBuilder b) throws IOException { + if (docValuesLoader != null) { + docValuesLoader.write(b); + } + } + + @Override + public String fieldName() { + return fullPath; + } + + private static final class NumericDocValuesWithOffsetsLoader implements DocValuesLoader { + private final SortedDocValues offsetDocValues; + private final SortedNumericDocValues valueDocValues; + private final NumericValueWriter writer; + private final ByteArrayStreamInput scratch = new ByteArrayStreamInput(); + + private boolean hasValue; + private boolean hasOffset; + private int[] offsetToOrd; + + NumericDocValuesWithOffsetsLoader( + SortedNumericDocValues valueDocValues, + SortedDocValues offsetDocValues, + NumericValueWriter writer + ) { + this.valueDocValues = valueDocValues; + this.offsetDocValues = offsetDocValues; + this.writer = writer; + } + + @Override + public boolean advanceToDoc(int docId) throws IOException { + hasValue = valueDocValues.advanceExact(docId); + hasOffset = offsetDocValues.advanceExact(docId); + if (hasValue || hasOffset) { + if (hasOffset) { + int offsetOrd = offsetDocValues.ordValue(); + var encodedValue = offsetDocValues.lookupOrd(offsetOrd); + scratch.reset(encodedValue.bytes, encodedValue.offset, encodedValue.length); + offsetToOrd = FieldArrayContext.parseOffsetArray(scratch); + } else { + offsetToOrd = null; + } + return true; + } else { + offsetToOrd = null; + return false; + } + } + + public boolean hasValue() { + return hasOffset || (hasValue && valueDocValues.docValueCount() > 0); + } + + public int count() { + if (hasValue) { + if (offsetToOrd != null) { + // Even though there may only be one value, the fact that offsets were recorded means that + // the value was in an array, so we need to trick CompositeSyntheticFieldLoader into + // always serializing this layer as an array + return offsetToOrd.length + 1; + } else { + return valueDocValues.docValueCount(); + } + } else { + if (hasOffset) { + // same as above, even though there are no values, the presence of recorded offsets means + // there was an array containing zero or more null values in the original source + return 2; + } else { + return 0; + } + } + } + + public void write(XContentBuilder b) throws IOException { + if (hasValue == false && hasOffset == false) { + return; + } + + if (offsetToOrd != null && hasValue) { + int count = valueDocValues.docValueCount(); + long[] values = new long[count]; + int duplicates = 0; + for (int i = 0; i < count; i++) { + long value = valueDocValues.nextValue(); + if (i > 0 && value == values[i - duplicates - 1]) { + duplicates++; + continue; + } + + values[i - duplicates] = value; + } + + for (int offset : offsetToOrd) { + if (offset == -1) { + b.nullValue(); + } else { + writer.writeLongValue(b, values[offset]); + } + } + } else if (offsetToOrd != null) { + // in cased all values are NULLs + for (int offset : offsetToOrd) { + assert offset == -1; + b.nullValue(); + } + } else { + for (int i = 0; i < valueDocValues.docValueCount(); i++) { + writer.writeLongValue(b, valueDocValues.nextValue()); + } + } + } + + } +} diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index f809a53d753fb..39e652e11c547 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -106,6 +106,7 @@ public > IFD getForField(String type, String field false, true, IndexVersion.current(), + null, null ).docValues(docValues).build(context).fieldType(); } else if (type.equals("double")) { @@ -116,6 +117,7 @@ public > IFD getForField(String type, String field false, true, IndexVersion.current(), + null, null ).docValues(docValues).build(context).fieldType(); } else if (type.equals("long")) { @@ -126,6 +128,7 @@ public > IFD getForField(String type, String field false, true, IndexVersion.current(), + null, null ).docValues(docValues).build(context).fieldType(); } else if (type.equals("int")) { @@ -136,6 +139,7 @@ public > IFD getForField(String type, String field false, true, IndexVersion.current(), + null, null ).docValues(docValues).build(context).fieldType(); } else if (type.equals("short")) { @@ -146,6 +150,7 @@ public > IFD getForField(String type, String field false, true, IndexVersion.current(), + null, null ).docValues(docValues).build(context).fieldType(); } else if (type.equals("byte")) { @@ -156,6 +161,7 @@ public > IFD getForField(String type, String field false, true, IndexVersion.current(), + null, null ).docValues(docValues).build(context).fieldType(); } else if (type.equals("geo_point")) { diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 07e836403e7ef..d55f712cb91b6 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -85,13 +85,14 @@ public void testGetForFieldDefaults() { assertTrue(fd instanceof SortedSetOrdinalsIndexFieldData); for (MappedFieldType mapper : Arrays.asList( - new NumberFieldMapper.Builder("int", BYTE, ScriptCompiler.NONE, false, true, IndexVersion.current(), null).build(context) + new NumberFieldMapper.Builder("int", BYTE, ScriptCompiler.NONE, false, true, IndexVersion.current(), null, null).build(context) .fieldType(), - new NumberFieldMapper.Builder("int", SHORT, ScriptCompiler.NONE, false, true, IndexVersion.current(), null).build(context) + new NumberFieldMapper.Builder("int", SHORT, ScriptCompiler.NONE, false, true, IndexVersion.current(), null, null).build(context) .fieldType(), - new NumberFieldMapper.Builder("int", INTEGER, ScriptCompiler.NONE, false, true, IndexVersion.current(), null).build(context) - .fieldType(), - new NumberFieldMapper.Builder("long", LONG, ScriptCompiler.NONE, false, true, IndexVersion.current(), null).build(context) + new NumberFieldMapper.Builder("int", INTEGER, ScriptCompiler.NONE, false, true, IndexVersion.current(), null, null).build( + context + ).fieldType(), + new NumberFieldMapper.Builder("long", LONG, ScriptCompiler.NONE, false, true, IndexVersion.current(), null, null).build(context) .fieldType() )) { ifdService.clear(); @@ -106,6 +107,7 @@ public void testGetForFieldDefaults() { false, true, IndexVersion.current(), + null, null ).build(context).fieldType(); ifdService.clear(); @@ -119,6 +121,7 @@ public void testGetForFieldDefaults() { false, true, IndexVersion.current(), + null, null ).build(context).fieldType(); ifdService.clear(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ByteOffsetDocValuesLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ByteOffsetDocValuesLoaderTests.java new file mode 100644 index 0000000000000..f0ea6410e6960 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/ByteOffsetDocValuesLoaderTests.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +public class ByteOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase { + @Override + protected String getFieldTypeName() { + return "byte"; + } + + @Override + protected Object randomValue() { + return randomByte(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ByteSyntheticSourceNativeArrayIntegrationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ByteSyntheticSourceNativeArrayIntegrationTests.java new file mode 100644 index 0000000000000..747b64a60328f --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/ByteSyntheticSourceNativeArrayIntegrationTests.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import com.carrotsearch.randomizedtesting.generators.RandomStrings; + +public class ByteSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase { + + @Override + protected String getFieldTypeName() { + return "byte"; + } + + @Override + protected Byte getRandomValue() { + return randomByte(); + } + + @Override + protected String getMalformedValue() { + return RandomStrings.randomAsciiOfLength(random(), 8); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index fcadc7b238a43..5a03034663c26 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -571,7 +571,7 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) } @Override - protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed) { + protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) { // Serializing and deserializing BigDecimal values may lead to parsing errors, a test artifact. return syntheticSourceSupportInternal(ignoreMalformed, false); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java index 615cf30475daa..14e2b1b42fdb9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java @@ -155,4 +155,8 @@ public void execute() { protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) { return new NumberSyntheticSourceSupport(Number::doubleValue, ignoreMalformed); } + + protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) { + return new NumberSyntheticSourceSupportForKeepTests(Number::doubleValue, ignoreMalformed, sourceKeepMode); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleOffsetDocValuesLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleOffsetDocValuesLoaderTests.java new file mode 100644 index 0000000000000..a67b6e5336915 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleOffsetDocValuesLoaderTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +public class DoubleOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase { + + @Override + protected String getFieldTypeName() { + return "double"; + } + + @Override + protected Object randomValue() { + return randomDouble(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleSyntheticSourceNativeArrayIntegrationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleSyntheticSourceNativeArrayIntegrationTests.java new file mode 100644 index 0000000000000..2f70224a204bc --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleSyntheticSourceNativeArrayIntegrationTests.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import com.carrotsearch.randomizedtesting.generators.RandomStrings; + +public class DoubleSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase { + + @Override + protected String getFieldTypeName() { + return "double"; + } + + @Override + protected Double getRandomValue() { + return randomDouble(); + } + + @Override + protected String getMalformedValue() { + return RandomStrings.randomAsciiOfLength(random(), 8); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java index 4792fbe43679a..902dfb99e7108 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java @@ -55,6 +55,12 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) return new NumberSyntheticSourceSupport(Number::floatValue, ignoreMalformed); } + @Override + protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) { + return new NumberSyntheticSourceSupportForKeepTests(Number::floatValue, ignoreMalformed, sourceKeepMode); + + } + @Override protected Function loadBlockExpected() { return v -> { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FloatOffsetDocValuesLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FloatOffsetDocValuesLoaderTests.java new file mode 100644 index 0000000000000..e92d01028305c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/FloatOffsetDocValuesLoaderTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +public class FloatOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase { + + @Override + protected String getFieldTypeName() { + return "float"; + } + + @Override + protected Object randomValue() { + return randomFloat(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FloatSyntheticSourceNativeArrayIntegrationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FloatSyntheticSourceNativeArrayIntegrationTests.java new file mode 100644 index 0000000000000..f7969ee235785 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/FloatSyntheticSourceNativeArrayIntegrationTests.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import com.carrotsearch.randomizedtesting.generators.RandomStrings; + +public class FloatSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase { + + @Override + protected String getFieldTypeName() { + return "float"; + } + + @Override + protected Float getRandomValue() { + return randomFloat(); + } + + @Override + protected String getMalformedValue() { + return RandomStrings.randomAsciiOfLength(random(), 8); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java index 71098fc2b895b..0361df4385a37 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java @@ -55,6 +55,16 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) ); } + @Override + protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) { + return new NumberSyntheticSourceSupportForKeepTests( + n -> HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(n.floatValue())), + ignoreMalformed, + sourceKeepMode + ); + + } + @Override protected Function loadBlockExpected() { return v -> { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatOffsetDocValuesLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatOffsetDocValuesLoaderTests.java new file mode 100644 index 0000000000000..b2715bc6ff316 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatOffsetDocValuesLoaderTests.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.sandbox.document.HalfFloatPoint; + +public class HalfFloatOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase { + + @Override + public String getFieldTypeName() { + return "half_float"; + } + + @Override + public Object randomValue() { + return HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(randomFloat())); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatSyntheticSourceNativeArrayIntegrationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatSyntheticSourceNativeArrayIntegrationTests.java new file mode 100644 index 0000000000000..96689ba29c0b4 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatSyntheticSourceNativeArrayIntegrationTests.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import com.carrotsearch.randomizedtesting.generators.RandomStrings; + +import org.apache.lucene.sandbox.document.HalfFloatPoint; + +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; + +public class HalfFloatSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase { + + public void testSynthesizeArray() throws Exception { + var inputArrayValues = new Float[][] { new Float[] { 0.78151345F, 0.6886488F, 0.6882413F } }; + var expectedArrayValues = new Float[inputArrayValues.length][inputArrayValues[0].length]; + for (int i = 0; i < inputArrayValues.length; i++) { + for (int j = 0; j < inputArrayValues[i].length; j++) { + expectedArrayValues[i][j] = HalfFloatPoint.sortableShortToHalfFloat( + HalfFloatPoint.halfFloatToSortableShort(inputArrayValues[i][j]) + ); + } + } + + var mapping = jsonBuilder().startObject() + .startObject("properties") + .startObject("field") + .field("type", getFieldTypeName()) + .endObject() + .endObject() + .endObject(); + + verifySyntheticArray(inputArrayValues, expectedArrayValues, mapping, "_id"); + } + + @Override + protected String getFieldTypeName() { + return "half_float"; + } + + @Override + protected Float getRandomValue() { + return HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(randomFloat())); + } + + @Override + protected String getMalformedValue() { + return RandomStrings.randomAsciiOfLength(random(), 8); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IPSyntheticSourceNativeArrayIntegrationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IPSyntheticSourceNativeArrayIntegrationTests.java index 2ad08ebb10aae..0707a63f66bd0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IPSyntheticSourceNativeArrayIntegrationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IPSyntheticSourceNativeArrayIntegrationTests.java @@ -9,6 +9,8 @@ package org.elasticsearch.index.mapper; +import com.carrotsearch.randomizedtesting.generators.RandomStrings; + import org.elasticsearch.common.network.NetworkAddress; import java.util.ArrayList; @@ -28,6 +30,11 @@ protected String getRandomValue() { return NetworkAddress.format(randomIp(true)); } + @Override + protected String getMalformedValue() { + return RandomStrings.randomAsciiOfLength(random(), 8); + } + public void testSynthesizeArray() throws Exception { var arrayValues = new Object[][] { new Object[] { "192.168.1.4", "192.168.1.3", null, "192.168.1.2", null, "192.168.1.1" }, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 6bcc94924d551..11e2305838705 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.test.FieldMaskingReader; import org.elasticsearch.xcontent.XContentBuilder; import org.hamcrest.Matchers; +import org.junit.Before; import java.io.IOException; import java.math.BigInteger; @@ -25,9 +26,9 @@ import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.TreeSet; public class IgnoredSourceFieldMapperTests extends MapperServiceTestCase { - private DocumentMapper getDocumentMapperWithFieldLimit() throws IOException { return createMapperService( Settings.builder() @@ -743,7 +744,7 @@ public void testIndexStoredArraySourceSingleLeafElement() throws IOException { b.startObject("int_value").field("type", "integer").endObject(); })).documentMapper(); var syntheticSource = syntheticSource(documentMapper, b -> b.array("int_value", new int[] { 10 })); - assertEquals("{\"int_value\":10}", syntheticSource); + assertEquals("{\"int_value\":[10]}", syntheticSource); ParsedDocument doc = documentMapper.parse(source(syntheticSource)); assertNull(doc.rootDoc().getField("_ignored_source")); } @@ -757,6 +758,8 @@ public void testIndexStoredArraySourceSingleLeafElementAndNull() throws IOExcept } public void testIndexStoredArraySourceSingleLeafElementInObjectArray() throws IOException { + roundtripMaskedFields.add("path.int_value.offsets"); + DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(mapping(b -> { b.startObject("path").field("synthetic_source_keep", "none").startObject("properties"); { @@ -843,6 +846,8 @@ public void testIndexStoredArraySourceRootObjectArray() throws IOException { } public void testIndexStoredArraySourceRootObjectArrayWithBypass() throws IOException { + roundtripMaskedFields.add("path.int_value.offsets"); + DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(mapping(b -> { b.startObject("path"); { @@ -895,6 +900,8 @@ public void testIndexStoredArraySourceNestedValueArray() throws IOException { } public void testIndexStoredArraySourceNestedValueArrayDisabled() throws IOException { + roundtripMaskedFields.add("path.obj.foo.offsets"); + DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(mapping(b -> { b.startObject("path"); { @@ -1125,6 +1132,8 @@ public void testNestedArray() throws IOException { } public void testConflictingFieldNameAfterArray() throws IOException { + roundtripMaskedFields.add("path.to.id.offsets"); + DocumentMapper documentMapper = createSytheticSourceMapperService(mapping(b -> { b.startObject("path").startObject("properties"); { @@ -2448,6 +2457,15 @@ public void testSingleDeepIgnoredField() throws IOException { assertEquals("{\"top\":{\"level1\":{\"level2\":{\"n\":25}}}}", syntheticSource); } + private Set roundtripMaskedFields; + + @Before + public void resetRoundtripMaskedFields() { + roundtripMaskedFields = new TreeSet<>( + Set.of(SourceFieldMapper.RECOVERY_SOURCE_NAME, IgnoredSourceFieldMapper.NAME, SourceFieldMapper.RECOVERY_SOURCE_SIZE_NAME) + ); + } + protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) throws IOException { // We exclude ignored source field since in some cases it contains an exact copy of a part of document source. @@ -2455,14 +2473,8 @@ protected void validateRoundTripReader(String syntheticSource, DirectoryReader r // and since the copy is exact, contents of ignored source are different. assertReaderEquals( "round trip " + syntheticSource, - new FieldMaskingReader( - Set.of(SourceFieldMapper.RECOVERY_SOURCE_NAME, IgnoredSourceFieldMapper.NAME, SourceFieldMapper.RECOVERY_SOURCE_SIZE_NAME), - reader - ), - new FieldMaskingReader( - Set.of(SourceFieldMapper.RECOVERY_SOURCE_NAME, IgnoredSourceFieldMapper.NAME, SourceFieldMapper.RECOVERY_SOURCE_SIZE_NAME), - roundTripReader - ) + new FieldMaskingReader(roundtripMaskedFields, reader), + new FieldMaskingReader(roundtripMaskedFields, roundTripReader) ); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IntegerOffsetDocValuesLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IntegerOffsetDocValuesLoaderTests.java new file mode 100644 index 0000000000000..f88b73926d48b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/IntegerOffsetDocValuesLoaderTests.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +public class IntegerOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase { + @Override + protected String getFieldTypeName() { + return "integer"; + } + + @Override + protected Object randomValue() { + return randomInt(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IntegerSyntheticSourceNativeArrayIntegrationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IntegerSyntheticSourceNativeArrayIntegrationTests.java new file mode 100644 index 0000000000000..eaee32e08cb9c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/IntegerSyntheticSourceNativeArrayIntegrationTests.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import com.carrotsearch.randomizedtesting.generators.RandomStrings; + +public class IntegerSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase { + + @Override + protected String getFieldTypeName() { + return "integer"; + } + + @Override + protected Integer getRandomValue() { + return randomInt(); + } + + @Override + protected String getMalformedValue() { + return RandomStrings.randomAsciiOfLength(random(), 8); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpOffsetDocValuesLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpOffsetDocValuesLoaderTests.java index dadfd22199aec..1ef1e7e24b255 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpOffsetDocValuesLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpOffsetDocValuesLoaderTests.java @@ -35,7 +35,7 @@ protected String getFieldTypeName() { } @Override - protected String randomValue() { + protected Object randomValue() { return NetworkAddress.format(randomIp(true)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordOffsetDocValuesLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordOffsetDocValuesLoaderTests.java index 55e935e11996c..d8f9317fc06f3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordOffsetDocValuesLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordOffsetDocValuesLoaderTests.java @@ -30,7 +30,7 @@ protected String getFieldTypeName() { } @Override - protected String randomValue() { + protected Object randomValue() { return randomAlphanumericOfLength(2); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordSyntheticSourceNativeArrayIntegrationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordSyntheticSourceNativeArrayIntegrationTests.java index 6f59f617ba259..41e0c644ee20e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordSyntheticSourceNativeArrayIntegrationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordSyntheticSourceNativeArrayIntegrationTests.java @@ -28,6 +28,11 @@ protected String getRandomValue() { return RandomStrings.randomAsciiOfLength(random(), 8); } + @Override + public Object getMalformedValue() { + return null; + } + public void testSynthesizeArray() throws Exception { var arrayValues = new Object[][] { new Object[] { "z", "y", null, "x", null, "v" }, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongOffsetDocValuesLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongOffsetDocValuesLoaderTests.java new file mode 100644 index 0000000000000..74b44e80f4028 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongOffsetDocValuesLoaderTests.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +public class LongOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase { + + public void testOffsetArray() throws Exception { + verifyOffsets("{\"field\":[26,24,25,3,2,1]}"); + verifyOffsets("{\"field\":[26,null,25,3,null,1]}"); + verifyOffsets("{\"field\":[5,5,6,-3,-9,-9,5,2,5,6,-3,-9]}"); + } + + public void testOffsetNestedArray() throws Exception { + verifyOffsets("{\"field\":[\"26\",[\"24\"],[\"3\"],null,\"1\"]}", "{\"field\":[26,24,3,null,1]}"); + verifyOffsets("{\"field\":[\"26\",[\"24\", [\"11\"]],[\"3\", [\"12\"]],null,\"1\"]}", "{\"field\":[26,24,11,3,12,null,1]}"); + } + + @Override + protected String getFieldTypeName() { + return "long"; + } + + @Override + protected Object randomValue() { + return randomLong(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongSyntheticSourceNativeArrayIntegrationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongSyntheticSourceNativeArrayIntegrationTests.java new file mode 100644 index 0000000000000..c73b6e32abd07 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongSyntheticSourceNativeArrayIntegrationTests.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import com.carrotsearch.randomizedtesting.generators.RandomStrings; + +import java.util.ArrayList; +import java.util.List; + +public class LongSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase { + + @Override + protected String getFieldTypeName() { + return "long"; + } + + @Override + public Long getRandomValue() { + return randomLong(); + } + + @Override + public String getMalformedValue() { + return RandomStrings.randomAsciiOfLength(random(), 8); + } + + public void testSynthesizeArray() throws Exception { + var arrayValues = new Object[][] { + new Object[] { 26, null, 25, null, 24, null, 22 }, + new Object[] { null, 2, null, -1 }, + new Object[] { null }, + new Object[] { null, null, null }, + new Object[] { 3, 2, 1 }, + new Object[] { 1 }, + new Object[] { 3, 3, 3, 3, 3 } }; + verifySyntheticArray(arrayValues); + } + + public void testSynthesizeObjectArray() throws Exception { + List> documents = new ArrayList<>(); + { + List document = new ArrayList<>(); + document.add(new Object[] { 26, 25, 24 }); + document.add(new Object[] { 13, 12, 13 }); + document.add(new Object[] { 3, 2, 1 }); + documents.add(document); + } + { + List document = new ArrayList<>(); + document.add(new Object[] { -12, 14, 6 }); + document.add(new Object[] { 1 }); + document.add(new Object[] { -200, 4 }); + documents.add(document); + } + verifySyntheticObjectArray(documents); + } + + public void testSynthesizeArrayInObjectField() throws Exception { + List documents = new ArrayList<>(); + documents.add(new Object[] { 26, 25, 24 }); + documents.add(new Object[] { 13, 12, 13 }); + documents.add(new Object[] { 3, 2, 1 }); + documents.add(new Object[] { -20, 5, 7 }); + documents.add(new Object[] { 8, 8, 8 }); + documents.add(new Object[] { 7, 6, 5 }); + verifySyntheticArrayInObject(documents); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java index 4b44f2444f27e..b02ceb31c96db 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java @@ -15,28 +15,26 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReader; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.IdsQueryBuilder; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.xcontent.XContentBuilder; -import org.hamcrest.Matchers; import java.io.IOException; import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; import java.util.Set; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.nullValue; public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCase { @@ -49,7 +47,7 @@ public void testSynthesizeEmptyArray() throws Exception { public void testSynthesizeArrayRandom() throws Exception { var arrayValues = new Object[randomInt(64)]; for (int j = 0; j < arrayValues.length; j++) { - arrayValues[j] = NetworkAddress.format(randomIp(true)); + arrayValues[j] = getRandomValue(); } verifySyntheticArray(new Object[][] { arrayValues }); } @@ -67,9 +65,187 @@ public void testSynthesizeArrayInObjectFieldRandom() throws Exception { verifySyntheticArrayInObject(documents); } + public void testSynthesizeArrayRandomIgnoresMalformed() throws Exception { + assumeTrue("supports ignore_malformed", getMalformedValue() != null); + int numDocs = randomIntBetween(8, 256); + List expectedDocuments = new ArrayList<>(numDocs); + List inputDocuments = new ArrayList<>(numDocs); + for (int i = 0; i < numDocs; i++) { + Object[] values = new Object[randomInt(64)]; + Object[] malformed = new Object[randomInt(64)]; + for (int j = 0; j < values.length; j++) { + values[j] = getRandomValue(); + } + for (int j = 0; j < malformed.length; j++) { + malformed[j] = getMalformedValue(); + } + + var expectedDocument = jsonBuilder().startObject(); + var inputDocument = jsonBuilder().startObject(); + + boolean expectedIsArray = values.length != 0 || malformed.length != 1; + if (expectedIsArray) { + expectedDocument.startArray("field"); + } else { + expectedDocument.field("field"); + } + inputDocument.startArray("field"); + + int valuesIdx = 0; + int malformedIdx = 0; + for (int j = 0; j < values.length + malformed.length; j++) { + if (j < values.length) { + expectedDocument.value(values[j]); + } else { + expectedDocument.value(malformed[j - values.length]); + } + + if (valuesIdx == values.length) { + inputDocument.value(malformed[malformedIdx++]); + } else if (malformedIdx == malformed.length) { + inputDocument.value(values[valuesIdx++]); + } else { + if (randomBoolean()) { + inputDocument.value(values[valuesIdx++]); + } else { + inputDocument.value(malformed[malformedIdx++]); + } + } + } + + if (expectedIsArray) { + expectedDocument.endArray(); + } + expectedDocument.endObject(); + inputDocument.endArray().endObject(); + + expectedDocuments.add(expectedDocument); + inputDocuments.add(inputDocument); + } + + var mapping = jsonBuilder().startObject() + .startObject("properties") + .startObject("field") + .field("type", getFieldTypeName()) + .field("ignore_malformed", true) + .endObject() + .endObject() + .endObject(); + var indexService = createIndex( + "test-index", + Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(), + mapping + ); + for (int i = 0; i < inputDocuments.size(); i++) { + var document = inputDocuments.get(i); + var indexRequest = new IndexRequest("test-index"); + indexRequest.id("my-id-" + i); + + indexRequest.source(document); + client().index(indexRequest).actionGet(); + } + + var refreshRequest = new RefreshRequest("test-index"); + client().execute(RefreshAction.INSTANCE, refreshRequest).actionGet(); + + for (int i = 0; i < expectedDocuments.size(); i++) { + var document = expectedDocuments.get(i); + String expectedSource = Strings.toString(document); + var searchRequest = new SearchRequest("test-index"); + searchRequest.source().query(new IdsQueryBuilder().addIds("my-id-" + i)); + var searchResponse = client().search(searchRequest).actionGet(); + try { + var hit = searchResponse.getHits().getHits()[0]; + assertThat(hit.getId(), equalTo("my-id-" + i)); + assertThat(hit.getSourceAsString(), equalTo(expectedSource)); + } finally { + searchResponse.decRef(); + } + } + } + + public void testSynthesizeRandomArrayInNestedContext() throws Exception { + var arrayValues = new Object[randomIntBetween(1, 8)][randomIntBetween(2, 64)]; + for (int i = 0; i < arrayValues.length; i++) { + for (int j = 0; j < arrayValues[i].length; j++) { + arrayValues[i][j] = randomInt(10) == 0 ? null : getRandomValue(); + } + } + + var mapping = jsonBuilder().startObject() + .startObject("properties") + .startObject("parent") + .field("type", "nested") + .startObject("properties") + .startObject("field") + .field("type", getFieldTypeName()) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + var indexService = createIndex( + "test-index", + Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(), + mapping + ); + + var indexRequest = new IndexRequest("test-index"); + indexRequest.id("my-id-1"); + var source = jsonBuilder().startObject().startArray("parent"); + for (Object[] arrayValue : arrayValues) { + source.startObject().array("field", arrayValue).endObject(); + } + source.endArray().endObject(); + indexRequest.source(source); + indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + client().index(indexRequest).actionGet(); + + var expectedSource = jsonBuilder().startObject(); + if (arrayValues.length > 1) { + expectedSource.startArray("parent"); + } else { + expectedSource.field("parent"); + } + for (Object[] arrayValue : arrayValues) { + expectedSource.startObject(); + expectedSource.array("field", arrayValue); + expectedSource.endObject(); + } + if (arrayValues.length > 1) { + expectedSource.endArray(); + } + expectedSource.endObject(); + var expected = Strings.toString(expectedSource); + + var searchRequest = new SearchRequest("test-index"); + searchRequest.source().query(new IdsQueryBuilder().addIds("my-id-1")); + var searchResponse = client().search(searchRequest).actionGet(); + try { + var hit = searchResponse.getHits().getHits()[0]; + assertThat(hit.getId(), equalTo("my-id-1")); + assertThat(hit.getSourceAsString(), equalTo(expected)); + } finally { + searchResponse.decRef(); + } + + assertThat(indexService.mapperService().mappingLookup().getMapper("parent.field").getOffsetFieldName(), nullValue()); + + try (var searcher = indexService.getShard(0).acquireSearcher(getTestName())) { + var reader = searcher.getDirectoryReader(); + var document = reader.storedFields().document(0); + Set storedFieldNames = new LinkedHashSet<>(document.getFields().stream().map(IndexableField::name).toList()); + assertThat(storedFieldNames, contains("_ignored_source")); + assertThat(FieldInfos.getMergedFieldInfos(reader).fieldInfo("parent.field.offsets"), nullValue()); + } + } + protected abstract String getFieldTypeName(); - protected abstract String getRandomValue(); + protected abstract Object getRandomValue(); + + protected abstract Object getMalformedValue(); protected void verifySyntheticArray(Object[][] arrays) throws IOException { var mapping = jsonBuilder().startObject() @@ -83,46 +259,53 @@ protected void verifySyntheticArray(Object[][] arrays) throws IOException { } protected void verifySyntheticArray(Object[][] arrays, XContentBuilder mapping, String... expectedStoredFields) throws IOException { + verifySyntheticArray(arrays, arrays, mapping, expectedStoredFields); + } + + private XContentBuilder arrayToSource(Object[] array) throws IOException { + var source = jsonBuilder().startObject(); + if (array != null) { + source.startArray("field"); + for (Object arrayValue : array) { + source.value(arrayValue); + } + source.endArray(); + } else { + source.field("field").nullValue(); + } + return source.endObject(); + } + + protected void verifySyntheticArray( + Object[][] inputArrays, + Object[][] expectedArrays, + XContentBuilder mapping, + String... expectedStoredFields + ) throws IOException { + assertThat(inputArrays.length, equalTo(expectedArrays.length)); + var indexService = createIndex( "test-index", Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(), mapping ); - for (int i = 0; i < arrays.length; i++) { - var array = arrays[i]; - + for (int i = 0; i < inputArrays.length; i++) { var indexRequest = new IndexRequest("test-index"); indexRequest.id("my-id-" + i); - var source = jsonBuilder().startObject(); - if (array != null) { - source.startArray("field"); - for (Object arrayValue : array) { - source.value(arrayValue); - } - source.endArray(); - } else { - source.field("field").nullValue(); - } - indexRequest.source(source.endObject()); + var inputSource = arrayToSource(inputArrays[i]); + indexRequest.source(inputSource); indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); client().index(indexRequest).actionGet(); + var expectedSource = arrayToSource(expectedArrays[i]); + var searchRequest = new SearchRequest("test-index"); searchRequest.source().query(new IdsQueryBuilder().addIds("my-id-" + i)); var searchResponse = client().search(searchRequest).actionGet(); try { var hit = searchResponse.getHits().getHits()[0]; assertThat(hit.getId(), equalTo("my-id-" + i)); - var sourceAsMap = hit.getSourceAsMap(); - assertThat(sourceAsMap, hasKey("field")); - var actualArray = (List) sourceAsMap.get("field"); - if (array == null) { - assertThat(actualArray, nullValue()); - } else if (array.length == 0) { - assertThat(actualArray, empty()); - } else { - assertThat(actualArray, Matchers.contains(array)); - } + assertThat(hit.getSourceAsString(), equalTo(Strings.toString(expectedSource))); } finally { searchResponse.decRef(); } @@ -130,7 +313,7 @@ protected void verifySyntheticArray(Object[][] arrays, XContentBuilder mapping, try (var searcher = indexService.getShard(0).acquireSearcher(getTestName())) { var reader = searcher.getDirectoryReader(); - for (int i = 0; i < arrays.length; i++) { + for (int i = 0; i < expectedArrays.length; i++) { var document = reader.storedFields().document(i); // Verify that there is no ignored source: Set storedFieldNames = new LinkedHashSet<>(document.getFields().stream().map(IndexableField::name).toList()); @@ -169,8 +352,9 @@ protected void verifySyntheticObjectArray(List> documents) throws source.array("field", arrayValue); source.endObject(); } - source.endArray(); - indexRequest.source(source.endObject()); + source.endArray().endObject(); + var expectedSource = Strings.toString(source); + indexRequest.source(source); indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); client().index(indexRequest).actionGet(); @@ -180,13 +364,7 @@ protected void verifySyntheticObjectArray(List> documents) throws try { var hit = searchResponse.getHits().getHits()[0]; assertThat(hit.getId(), equalTo("my-id-" + i)); - var sourceAsMap = hit.getSourceAsMap(); - var objectArray = (List) sourceAsMap.get("object"); - for (int j = 0; j < document.size(); j++) { - var expected = document.get(j); - List actual = (List) ((Map) objectArray.get(j)).get("field"); - assertThat(actual, Matchers.contains(expected)); - } + assertThat(hit.getSourceAsString(), equalTo(expectedSource)); } finally { searchResponse.decRef(); } @@ -223,7 +401,7 @@ protected void verifySyntheticArrayInObject(List documents) throws IOE .startObject("object") .startObject("properties") .startObject("field") - .field("type", "keyword") + .field("type", getFieldTypeName()) .endObject() .endObject() .endObject() @@ -238,8 +416,9 @@ protected void verifySyntheticArrayInObject(List documents) throws IOE var source = jsonBuilder().startObject(); source.startObject("object"); source.array("field", arrayValue); - source.endObject(); - indexRequest.source(source.endObject()); + source.endObject().endObject(); + var expectedSource = Strings.toString(source); + indexRequest.source(source); indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); client().index(indexRequest).actionGet(); @@ -249,17 +428,7 @@ protected void verifySyntheticArrayInObject(List documents) throws IOE try { var hit = searchResponse.getHits().getHits()[0]; assertThat(hit.getId(), equalTo("my-id-" + i)); - var sourceAsMap = hit.getSourceAsMap(); - var objectArray = (Map) sourceAsMap.get("object"); - - List actual = (List) objectArray.get("field"); - if (arrayValue == null) { - assertThat(actual, nullValue()); - } else if (arrayValue.length == 0) { - assertThat(actual, empty()); - } else { - assertThat(actual, Matchers.contains(arrayValue)); - } + assertThat(hit.getSourceAsString(), equalTo(expectedSource)); } finally { searchResponse.decRef(); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index 164e0232bf409..d4420f4baa0f4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -943,6 +943,7 @@ public void testFetchSourceValue() throws IOException { false, true, IndexVersion.current(), + null, null ).build(MapperBuilderContext.root(false, false)).fieldType(); assertEquals(List.of(3), fetchSourceValue(mapper, 3.14)); @@ -956,6 +957,7 @@ public void testFetchSourceValue() throws IOException { false, true, IndexVersion.current(), + null, null ).nullValue(2.71f).build(MapperBuilderContext.root(false, false)).fieldType(); assertEquals(List.of(2.71f), fetchSourceValue(nullValueMapper, "")); @@ -970,6 +972,7 @@ public void testFetchHalfFloatFromSource() throws IOException { false, true, IndexVersion.current(), + null, null ).build(MapperBuilderContext.root(false, false)).fieldType(); /* diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java index 1f8a2a754428b..7402405f5b883 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java @@ -336,6 +336,7 @@ public void testConflictingDynamicUpdate() { false, true, IndexVersion.current(), + null, null ) ).build(MapperBuilderContext.root(false, false)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java index 64718c4ac816b..c18f85b8e6c0d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java @@ -17,6 +17,7 @@ import org.elasticsearch.xcontent.XContentType; import java.io.IOException; +import java.util.HashSet; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.nullValue; @@ -159,25 +160,32 @@ public void testOffsetArrayWithNulls() throws Exception { } public void testOffsetArrayRandom() throws Exception { - StringBuilder values = new StringBuilder(); + String values; int numValues = randomIntBetween(0, 256); - for (int i = 0; i < numValues; i++) { - if (randomInt(10) == 1) { - values.append("null"); - } else { - String randomValue = randomValue(); - values.append('"').append(randomValue).append('"'); - } - if (i != (numValues - 1)) { - values.append(','); + + var previousValues = new HashSet<>(); + try (XContentBuilder b = XContentBuilder.builder(XContentType.JSON.xContent());) { + b.startArray(); + for (int i = 0; i < numValues; i++) { + if (randomInt(10) == 1) { + b.nullValue(); + } else if (randomInt(10) == 1 && previousValues.size() > 0) { + b.value(randomFrom(previousValues)); + } else { + Object value = randomValue(); + previousValues.add(value); + b.value(value); + } } + b.endArray(); + values = Strings.toString(b); } - verifyOffsets("{\"field\":[" + values + "]}"); + verifyOffsets("{\"field\":" + values + "}"); } protected abstract String getFieldTypeName(); - protected abstract String randomValue(); + protected abstract Object randomValue(); protected void verifyOffsets(String source) throws IOException { verifyOffsets(source, source); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ShortOffsetDocValuesLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ShortOffsetDocValuesLoaderTests.java new file mode 100644 index 0000000000000..dfde1f7ef58a2 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/ShortOffsetDocValuesLoaderTests.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +public class ShortOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase { + @Override + protected String getFieldTypeName() { + return "short"; + } + + @Override + protected Object randomValue() { + return randomShort(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ShortSyntheticSourceNativeArrayIntegrationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ShortSyntheticSourceNativeArrayIntegrationTests.java new file mode 100644 index 0000000000000..d42c8d4e95172 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/ShortSyntheticSourceNativeArrayIntegrationTests.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import com.carrotsearch.randomizedtesting.generators.RandomStrings; + +public class ShortSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase { + + @Override + protected String getFieldTypeName() { + return "short"; + } + + @Override + protected Short getRandomValue() { + return randomShort(); + } + + @Override + protected String getMalformedValue() { + return RandomStrings.randomAsciiOfLength(random(), 8); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 72abb8a179dfe..e856bd5329385 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -1689,12 +1689,14 @@ public final void testSyntheticSourceInNestedObject() throws IOException { }), equalTo("{}")); } - protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed) { + protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode keepMode) { return syntheticSourceSupport(ignoreMalformed); } public void testSyntheticSourceKeepNone() throws IOException { - SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed()).example(1); + SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed(), Mapper.SourceKeepMode.NONE).example( + 1 + ); DocumentMapper mapper = createSytheticSourceMapperService(mapping(b -> { b.startObject("field"); b.field("synthetic_source_keep", "none"); @@ -1705,7 +1707,9 @@ public void testSyntheticSourceKeepNone() throws IOException { } public void testSyntheticSourceKeepAll() throws IOException { - SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed()).example(1); + SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed(), Mapper.SourceKeepMode.ALL).example( + 1 + ); DocumentMapper mapperAll = createSytheticSourceMapperService(mapping(b -> { b.startObject("field"); b.field("synthetic_source_keep", "all"); @@ -1722,7 +1726,8 @@ public void testSyntheticSourceKeepAll() throws IOException { } public void testSyntheticSourceKeepArrays() throws IOException { - SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed()).example(1); + SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed(), Mapper.SourceKeepMode.ARRAYS) + .example(1); DocumentMapper mapperAll = createSytheticSourceMapperService(mapping(b -> { b.startObject("field"); b.field("synthetic_source_keep", randomSyntheticSourceKeep()); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index f1e467c36046f..da646358a090a 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -396,7 +396,36 @@ protected Matcher blockItemMatcher(Object expected) { protected abstract Number randomNumber(); - protected final class NumberSyntheticSourceSupport implements SyntheticSourceSupport { + protected final class NumberSyntheticSourceSupportForKeepTests extends NumberSyntheticSourceSupport { + private final boolean preserveSource; + + protected NumberSyntheticSourceSupportForKeepTests( + Function round, + boolean ignoreMalformed, + Mapper.SourceKeepMode sourceKeepMode + ) { + super(round, ignoreMalformed); + this.preserveSource = sourceKeepMode == Mapper.SourceKeepMode.ALL; + } + + @Override + public boolean preservesExactSource() { + return preserveSource; + } + + @Override + public SyntheticSourceExample example(int maxVals) { + var example = super.example(maxVals); + return new SyntheticSourceExample( + example.expectedForSyntheticSource(), + example.expectedForSyntheticSource(), + example.expectedForBlockLoader(), + example.mapping() + ); + } + } + + protected class NumberSyntheticSourceSupport implements SyntheticSourceSupport { private final Long nullValue = usually() ? null : randomNumber().longValue(); private final boolean coerce = rarely(); private final boolean docValues = randomBoolean(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java index a297f5d13254b..64a7868607cf1 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java @@ -123,4 +123,9 @@ protected void registerParameters(ParameterChecker checker) throws IOException { protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) { return new NumberSyntheticSourceSupport(Number::longValue, ignoreMalformed); } + + @Override + protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) { + return new NumberSyntheticSourceSupportForKeepTests(Number::longValue, ignoreMalformed, sourceKeepMode); + } } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapper.java index 70adabf414f66..0464d7112f0f2 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapper.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapper.java @@ -168,8 +168,15 @@ public static final class Builder extends FieldMapper.Builder { private final IndexVersion indexCreatedVersion; private final IndexMode indexMode; - - public Builder(String name, Boolean ignoreMalformedByDefault, IndexVersion indexCreatedVersion, IndexMode mode) { + private final SourceKeepMode indexSourceKeepMode; + + public Builder( + String name, + Boolean ignoreMalformedByDefault, + IndexVersion indexCreatedVersion, + IndexMode mode, + SourceKeepMode indexSourceKeepMode + ) { super(name); this.ignoreMalformed = Parameter.boolParam( Names.IGNORE_MALFORMED, @@ -181,6 +188,7 @@ public Builder(String name, Boolean ignoreMalformedByDefault, IndexVersion index this.timeSeriesMetric = TimeSeriesParams.metricParam(m -> toType(m).metricType, MetricType.GAUGE); this.indexCreatedVersion = Objects.requireNonNull(indexCreatedVersion); this.indexMode = mode; + this.indexSourceKeepMode = indexSourceKeepMode; } @Override @@ -238,7 +246,8 @@ public AggregateMetricDoubleFieldMapper build(MapperBuilderContext context) { false, false, indexCreatedVersion, - indexMode + indexMode, + indexSourceKeepMode ).allowMultipleValues(false); } else { builder = new NumberFieldMapper.Builder( @@ -248,7 +257,8 @@ public AggregateMetricDoubleFieldMapper build(MapperBuilderContext context) { false, true, indexCreatedVersion, - indexMode + indexMode, + indexSourceKeepMode ).allowMultipleValues(false); } NumberFieldMapper fieldMapper = builder.build(context); @@ -274,7 +284,13 @@ public AggregateMetricDoubleFieldMapper build(MapperBuilderContext context) { } public static final FieldMapper.TypeParser PARSER = new TypeParser( - (n, c) -> new Builder(n, IGNORE_MALFORMED_SETTING.get(c.getSettings()), c.indexVersionCreated(), c.getIndexSettings().getMode()), + (n, c) -> new Builder( + n, + IGNORE_MALFORMED_SETTING.get(c.getSettings()), + c.indexVersionCreated(), + c.getIndexSettings().getMode(), + c.getIndexSettings().sourceKeepMode() + ), notInMultiFields(CONTENT_TYPE) ); @@ -673,6 +689,7 @@ public MetricType getMetricType() { private final TimeSeriesParams.MetricType metricType; private final IndexMode indexMode; + private final SourceKeepMode indexSourceKeepMode; private AggregateMetricDoubleFieldMapper( String simpleName, @@ -690,6 +707,7 @@ private AggregateMetricDoubleFieldMapper( this.metricType = builder.timeSeriesMetric.getValue(); this.indexCreatedVersion = builder.indexCreatedVersion; this.indexMode = builder.indexMode; + this.indexSourceKeepMode = builder.indexSourceKeepMode; } @Override @@ -842,7 +860,8 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), ignoreMalformedByDefault, indexCreatedVersion, indexMode).metric(metricType).init(this); + return new Builder(leafName(), ignoreMalformedByDefault, indexCreatedVersion, indexMode, indexSourceKeepMode).metric(metricType) + .init(this); } @Override diff --git a/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapperTests.java b/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapperTests.java index 176311565ec88..d6b762ab4b79e 100644 --- a/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapperTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.plugins.Plugin; @@ -118,7 +119,8 @@ public void testSyntheticSourceManyNullValue() throws IOException { } public void testSyntheticSourceIndexLevelKeepArrays() throws IOException { - SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed()).example(1); + SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed(), Mapper.SourceKeepMode.ARRAYS) + .example(1); XContentBuilder mappings = mapping(b -> { b.startObject("field"); example.mapping().accept(b); diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index f554a84048fde..ed65c7efc3d82 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberTypeOutOfRangeSpec; @@ -382,6 +383,11 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) return new NumberSyntheticSourceSupport(ignoreMalformed); } + @Override + protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) { + return syntheticSourceSupport(ignoreMalformed); + } + @Override protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java index 34f1f2f97a328..8b4b999b2a867 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java @@ -720,6 +720,7 @@ private Map createFieldTypes(RollupJobConfig job) { false, false, IndexVersion.current(), + null, null ).build(MapperBuilderContext.root(false, false)).fieldType(); fieldTypes.put(ft.name(), ft); @@ -744,6 +745,7 @@ private Map createFieldTypes(RollupJobConfig job) { false, false, IndexVersion.current(), + null, null ).build(MapperBuilderContext.root(false, false)).fieldType(); fieldTypes.put(ft.name(), ft);