diff --git a/docs/changelog/124927.yaml b/docs/changelog/124927.yaml new file mode 100644 index 0000000000000..7b3c5b4663ff0 --- /dev/null +++ b/docs/changelog/124927.yaml @@ -0,0 +1,5 @@ +pr: 124927 +summary: Use `FallbackSyntheticSourceBlockLoader` for `shape` and `geo_shape` +area: Mapping +type: enhancement +issues: [] diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index d68f417f91b42..8c38c5abca13f 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -342,7 +342,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { var nullValueAdjusted = nullValue != null ? adjustSourceValue(nullValue, scalingFactor) : null; - return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport(nullValue) { + return new FallbackSyntheticSourceBlockLoader.SingleValueReader(nullValue) { @Override public void convertValue(Object value, List accumulator) { if (coerce && value.equals("")) { diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldBlockLoaderTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldBlockLoaderTests.java index c44efd2e52da2..17c64bb3e0eaf 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldBlockLoaderTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldBlockLoaderTests.java @@ -18,8 +18,8 @@ import java.util.Map; public class ScaledFloatFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { - public ScaledFloatFieldBlockLoaderTests() { - super(FieldType.SCALED_FLOAT); + public ScaledFloatFieldBlockLoaderTests(Params params) { + super(FieldType.SCALED_FLOAT, params); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 6e00cc765bd8b..894c053e3c0c0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.geo.GeometryFormatterFactory; @@ -67,12 +68,16 @@ public abstract void parse(XContentParser parser, CheckedConsumer consumer) { try (XContentParser parser = wrapObject(sourceMap)) { - parse(parser, v -> consumer.accept(normalizeFromSource(v)), NoopMalformedValueHandler.INSTANCE); + parseFromSource(parser, consumer); } catch (IOException e) { throw new UncheckedIOException(e); } } + private void parseFromSource(XContentParser parser, Consumer consumer) throws IOException { + parse(parser, v -> consumer.accept(normalizeFromSource(v)), NoopMalformedValueHandler.INSTANCE); + } + /** * Normalize a geometry when reading from source. When reading from source we can skip * some expensive steps as the geometry has already been indexed. @@ -187,6 +192,80 @@ protected BlockLoader blockLoaderFromSource(BlockLoaderContext blContext) { } protected abstract Object nullValueAsSource(T nullValue); + + protected BlockLoader blockLoaderFromFallbackSyntheticSource(BlockLoaderContext blContext) { + return new FallbackSyntheticSourceBlockLoader(new GeometriesFallbackSyntheticSourceReader(), name()) { + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); + } + }; + } + + private class GeometriesFallbackSyntheticSourceReader implements FallbackSyntheticSourceBlockLoader.Reader { + private final Function, List> formatter; + + private GeometriesFallbackSyntheticSourceReader() { + this.formatter = getFormatter(GeometryFormatterFactory.WKB); + } + + @Override + public void convertValue(Object value, List accumulator) { + final List values = new ArrayList<>(); + + geometryParser.fetchFromSource(value, v -> { + if (v != null) { + values.add(v); + } else if (nullValue != null) { + values.add(nullValue); + } + }); + var formatted = formatter.apply(values); + + for (var formattedValue : formatted) { + if (formattedValue instanceof byte[] wkb) { + accumulator.add(new BytesRef(wkb)); + } else { + throw new IllegalArgumentException( + "Unsupported source type for spatial geometry: " + formattedValue.getClass().getSimpleName() + ); + } + } + } + + @Override + public void parse(XContentParser parser, List accumulator) throws IOException { + final List values = new ArrayList<>(); + + geometryParser.parseFromSource(parser, v -> { + if (v != null) { + values.add(v); + } else if (nullValue != null) { + values.add(nullValue); + } + }); + var formatted = formatter.apply(values); + + for (var formattedValue : formatted) { + if (formattedValue instanceof byte[] wkb) { + accumulator.add(new BytesRef(wkb)); + } else { + throw new IllegalArgumentException( + "Unsupported source type for spatial geometry: " + formattedValue.getClass().getSimpleName() + ); + } + } + } + + @Override + public void writeToBlock(List values, BlockLoader.Builder blockBuilder) { + var bytesRefBuilder = (BlockLoader.BytesRefBuilder) blockBuilder; + + for (var value : values) { + bytesRefBuilder.appendBytesRef(value); + } + } + } } private final Explicit ignoreMalformed; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 17d8589402385..6917a3c39b1a9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -337,7 +337,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { - return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport(nullValue) { + return new FallbackSyntheticSourceBlockLoader.SingleValueReader(nullValue) { @Override public void convertValue(Object value, List accumulator) { try { @@ -366,10 +366,10 @@ protected void parseNonNullValue(XContentParser parser, List accumulato @Override public void writeToBlock(List values, BlockLoader.Builder blockBuilder) { - var longBuilder = (BlockLoader.BooleanBuilder) blockBuilder; + var booleanBuilder = (BlockLoader.BooleanBuilder) blockBuilder; for (var value : values) { - longBuilder.appendBoolean(value); + booleanBuilder.appendBoolean(value); } } }; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 461871845191f..518d8d3cc1d51 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -871,7 +871,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { Function dateParser = this::parse; - return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport(nullValue) { + return new FallbackSyntheticSourceBlockLoader.SingleValueReader(nullValue) { @Override public void convertValue(Object value, List accumulator) { try { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java index 8474b754e951d..f606de4691e53 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java @@ -235,13 +235,6 @@ private void parseFieldFromParent(IgnoredSourceFieldMapper.NameValue nameValue, } private void parseWithReader(XContentParser parser, List blockValues) throws IOException { - if (parser.currentToken() == XContentParser.Token.START_ARRAY) { - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - reader.parse(parser, blockValues); - } - return; - } - reader.parse(parser, blockValues); } @@ -274,10 +267,15 @@ public interface Reader { void writeToBlock(List values, Builder blockBuilder); } - public abstract static class ReaderWithNullValueSupport implements Reader { + /** + * Reader for field types that don't parse arrays (arrays are always treated as multiple values) + * as opposed to field types that treat arrays as special cases (for example point). + * @param + */ + public abstract static class SingleValueReader implements Reader { private final Object nullValue; - public ReaderWithNullValueSupport(Object nullValue) { + public SingleValueReader(Object nullValue) { this.nullValue = nullValue; } @@ -289,6 +287,18 @@ public void parse(XContentParser parser, List accumulator) throws IOException } return; } + if (parser.currentToken() == XContentParser.Token.START_ARRAY) { + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { + if (nullValue != null) { + convertValue(nullValue, accumulator); + } + } else { + parseNonNullValue(parser, accumulator); + } + } + return; + } parseNonNullValue(parser, accumulator); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index b0ad02dc644b8..aff1428e8cd9b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -669,7 +669,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { var nullValueBytes = nullValue != null ? new BytesRef(nullValue) : null; - return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport(nullValueBytes) { + return new FallbackSyntheticSourceBlockLoader.SingleValueReader(nullValueBytes) { @Override public void convertValue(Object value, List accumulator) { String stringValue = ((BytesRef) value).utf8ToString(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 27339430efd5a..f798a5a6f83ff 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1738,8 +1738,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - abstract static class NumberFallbackSyntheticSourceReader extends FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport< - Number> { + abstract static class NumberFallbackSyntheticSourceReader extends FallbackSyntheticSourceBlockLoader.SingleValueReader { private final NumberType type; private final Number nullValue; private final boolean coerce; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/BooleanFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/BooleanFieldBlockLoaderTests.java index 390a779788ebd..e85cabd8cda59 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/BooleanFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/BooleanFieldBlockLoaderTests.java @@ -17,13 +17,13 @@ import java.util.Objects; public class BooleanFieldBlockLoaderTests extends BlockLoaderTestCase { - public BooleanFieldBlockLoaderTests() { - super(FieldType.BOOLEAN); + public BooleanFieldBlockLoaderTests(Params params) { + super(FieldType.BOOLEAN.toString(), params); } @Override @SuppressWarnings("unchecked") - protected Object expected(Map fieldMapping, Object value, boolean syntheticSource) { + protected Object expected(Map fieldMapping, Object value) { var rawNullValue = fieldMapping.get("null_value"); Boolean nullValue; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ByteFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ByteFieldBlockLoaderTests.java index 0f8283c4fd426..bdc91ced07ac0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ByteFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ByteFieldBlockLoaderTests.java @@ -15,8 +15,8 @@ import java.util.Map; public class ByteFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { - public ByteFieldBlockLoaderTests() { - super(FieldType.BYTE); + public ByteFieldBlockLoaderTests(Params params) { + super(FieldType.BYTE, params); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DateFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DateFieldBlockLoaderTests.java index b17e0c1707b2c..eeab5e01dea15 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DateFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DateFieldBlockLoaderTests.java @@ -23,13 +23,13 @@ import java.util.Objects; public class DateFieldBlockLoaderTests extends BlockLoaderTestCase { - public DateFieldBlockLoaderTests() { - super(FieldType.DATE); + public DateFieldBlockLoaderTests(Params params) { + super(FieldType.DATE.toString(), params); } @Override @SuppressWarnings("unchecked") - protected Object expected(Map fieldMapping, Object value, boolean syntheticSource) { + protected Object expected(Map fieldMapping, Object value) { var format = (String) fieldMapping.get("format"); var nullValue = fieldMapping.get("null_value") != null ? format(fieldMapping.get("null_value"), format) : null; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DoubleFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DoubleFieldBlockLoaderTests.java index 5a504487680c5..c4d427aa97b0b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DoubleFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DoubleFieldBlockLoaderTests.java @@ -15,8 +15,8 @@ import java.util.Map; public class DoubleFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { - public DoubleFieldBlockLoaderTests() { - super(FieldType.DOUBLE); + public DoubleFieldBlockLoaderTests(Params params) { + super(FieldType.DOUBLE, params); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/FloatFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/FloatFieldBlockLoaderTests.java index 9b82293d33918..e6fb36aaf711b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/FloatFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/FloatFieldBlockLoaderTests.java @@ -15,8 +15,8 @@ import java.util.Map; public class FloatFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { - public FloatFieldBlockLoaderTests() { - super(FieldType.FLOAT); + public FloatFieldBlockLoaderTests(Params params) { + super(FieldType.FLOAT, params); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/HalfFloatFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/HalfFloatFieldBlockLoaderTests.java index 23e6c86747f15..4de768b9b3b4d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/HalfFloatFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/HalfFloatFieldBlockLoaderTests.java @@ -16,8 +16,8 @@ import java.util.Map; public class HalfFloatFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { - public HalfFloatFieldBlockLoaderTests() { - super(FieldType.HALF_FLOAT); + public HalfFloatFieldBlockLoaderTests(Params params) { + super(FieldType.HALF_FLOAT, params); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/IntegerFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/IntegerFieldBlockLoaderTests.java index 6a9c9ca4ba7d5..3de1279e61011 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/IntegerFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/IntegerFieldBlockLoaderTests.java @@ -15,8 +15,8 @@ import java.util.Map; public class IntegerFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { - public IntegerFieldBlockLoaderTests() { - super(FieldType.INTEGER); + public IntegerFieldBlockLoaderTests(Params params) { + super(FieldType.INTEGER, params); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java index b7f7bd9de5ef7..1d16ebb6aca5c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java @@ -20,13 +20,13 @@ import java.util.stream.Stream; public class KeywordFieldBlockLoaderTests extends BlockLoaderTestCase { - public KeywordFieldBlockLoaderTests() { - super(FieldType.KEYWORD); + public KeywordFieldBlockLoaderTests(Params params) { + super(FieldType.KEYWORD.toString(), params); } @SuppressWarnings("unchecked") @Override - protected Object expected(Map fieldMapping, Object value, boolean syntheticSource) { + protected Object expected(Map fieldMapping, Object value) { var nullValue = (String) fieldMapping.get("null_value"); var ignoreAbove = fieldMapping.get("ignore_above") == null @@ -44,7 +44,8 @@ protected Object expected(Map fieldMapping, Object value, boolea Function, Stream> convertValues = s -> s.map(v -> convert(v, nullValue, ignoreAbove)) .filter(Objects::nonNull); - if ((boolean) fieldMapping.getOrDefault("doc_values", false)) { + boolean hasDocValues = hasDocValues(fieldMapping, true); + if (hasDocValues) { // Sorted and no duplicates var resultList = convertValues.andThen(Stream::distinct) .andThen(Stream::sorted) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/LongFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/LongFieldBlockLoaderTests.java index b6425029175e2..43a973f96f585 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/LongFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/LongFieldBlockLoaderTests.java @@ -15,8 +15,8 @@ import java.util.Map; public class LongFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { - public LongFieldBlockLoaderTests() { - super(FieldType.LONG); + public LongFieldBlockLoaderTests(Params params) { + super(FieldType.LONG, params); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ShortFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ShortFieldBlockLoaderTests.java index 1f3286c3398c4..6f5008164ebb3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ShortFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ShortFieldBlockLoaderTests.java @@ -15,8 +15,8 @@ import java.util.Map; public class ShortFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { - public ShortFieldBlockLoaderTests() { - super(FieldType.SHORT); + public ShortFieldBlockLoaderTests(Params params) { + super(FieldType.SHORT, params); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java index a8b1e68061041..9d49a9664a5a1 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java @@ -9,6 +9,8 @@ package org.elasticsearch.index.mapper; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.store.Directory; @@ -18,7 +20,6 @@ import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; import org.elasticsearch.logsdb.datageneration.DocumentGenerator; -import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.Mapping; import org.elasticsearch.logsdb.datageneration.MappingGenerator; import org.elasticsearch.logsdb.datageneration.Template; @@ -33,6 +34,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -40,13 +42,36 @@ import java.util.stream.Stream; public abstract class BlockLoaderTestCase extends MapperServiceTestCase { - private final FieldType fieldType; + private static final MappedFieldType.FieldExtractPreference[] PREFERENCES = new MappedFieldType.FieldExtractPreference[] { + MappedFieldType.FieldExtractPreference.NONE }; + + @ParametersFactory(argumentFormatting = "preference=%s") + public static List args() { + List args = new ArrayList<>(); + for (boolean syntheticSource : new boolean[] { false, true }) { + for (MappedFieldType.FieldExtractPreference preference : PREFERENCES) { + args.add(new Object[] { new Params(syntheticSource, preference) }); + } + } + return args; + } + + public record Params(boolean syntheticSource, MappedFieldType.FieldExtractPreference preference) {} + + private final String fieldType; + protected final Params params; + private final String fieldName; private final MappingGenerator mappingGenerator; private final DocumentGenerator documentGenerator; - protected BlockLoaderTestCase(FieldType fieldType) { + protected BlockLoaderTestCase(String fieldType, Params params) { + this(fieldType, List.of(), params); + } + + protected BlockLoaderTestCase(String fieldType, Collection customHandlers, Params params) { this.fieldType = fieldType; + this.params = params; this.fieldName = randomAlphaOfLengthBetween(5, 10); var specification = DataGeneratorSpecification.builder() @@ -65,6 +90,7 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle( return new DataSourceResponse.ObjectMappingParametersGenerator(HashMap::new); // just defaults } })) + .withDataSourceHandlers(customHandlers) .build(); this.mappingGenerator = new MappingGenerator(specification); @@ -83,10 +109,9 @@ public void testFieldHasValueWithEmptyFieldInfos() { public void testBlockLoader() throws IOException { var template = new Template(Map.of(fieldName, new Template.Leaf(fieldName, fieldType))); - var syntheticSource = randomBoolean(); var mapping = mappingGenerator.generate(template); - runTest(template, mapping, syntheticSource, fieldName); + runTest(template, mapping, fieldName); } @SuppressWarnings("unchecked") @@ -110,34 +135,34 @@ public void testBlockLoaderForFieldInObject() throws IOException { currentLevel.put(fieldName, new Template.Leaf(fieldName, fieldType)); var template = new Template(top); - var syntheticSource = randomBoolean(); - var mapping = mappingGenerator.generate(template); - if (syntheticSource && randomBoolean()) { + if (params.syntheticSource && randomBoolean()) { // force fallback synthetic source in the hierarchy var docMapping = (Map) mapping.raw().get("_doc"); var topLevelMapping = (Map) ((Map) docMapping.get("properties")).get("top"); topLevelMapping.put("synthetic_source_keep", "all"); } - runTest(template, mapping, syntheticSource, fullFieldName.toString()); + runTest(template, mapping, fullFieldName.toString()); } - private void runTest(Template template, Mapping mapping, boolean syntheticSource, String fieldName) throws IOException { + private void runTest(Template template, Mapping mapping, String fieldName) throws IOException { var mappingXContent = XContentBuilder.builder(XContentType.JSON.xContent()).map(mapping.raw()); - var mapperService = syntheticSource ? createSytheticSourceMapperService(mappingXContent) : createMapperService(mappingXContent); + var mapperService = params.syntheticSource + ? createSytheticSourceMapperService(mappingXContent) + : createMapperService(mappingXContent); var document = documentGenerator.generate(template, mapping); var documentXContent = XContentBuilder.builder(XContentType.JSON.xContent()).map(document); Object blockLoaderResult = setupAndInvokeBlockLoader(mapperService, documentXContent, fieldName); - Object expected = expected(mapping.lookup().get(fieldName), getFieldValue(document, fieldName), syntheticSource); + Object expected = expected(mapping.lookup().get(fieldName), getFieldValue(document, fieldName)); assertEquals(expected, blockLoaderResult); } - protected abstract Object expected(Map fieldMapping, Object value, boolean syntheticSource); + protected abstract Object expected(Map fieldMapping, Object value); private Object getFieldValue(Map document, String fieldName) { var rawValues = new ArrayList<>(); @@ -258,8 +283,7 @@ public IndexSettings indexSettings() { @Override public MappedFieldType.FieldExtractPreference fieldExtractPreference() { - // TODO randomize when adding support for fields that care about this - return MappedFieldType.FieldExtractPreference.NONE; + return params.preference; } @Override @@ -283,4 +307,8 @@ public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { } }); } + + protected static boolean hasDocValues(Map fieldMapping, boolean defaultValue) { + return (boolean) fieldMapping.getOrDefault("doc_values", defaultValue); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldBlockLoaderTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldBlockLoaderTestCase.java index 43c2663ac71e8..095bb5722c7af 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldBlockLoaderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldBlockLoaderTestCase.java @@ -16,20 +16,21 @@ import java.util.Objects; public abstract class NumberFieldBlockLoaderTestCase extends BlockLoaderTestCase { - public NumberFieldBlockLoaderTestCase(FieldType fieldType) { - super(fieldType); + public NumberFieldBlockLoaderTestCase(FieldType fieldType, Params params) { + super(fieldType.toString(), params); } @Override @SuppressWarnings("unchecked") - protected Object expected(Map fieldMapping, Object value, boolean syntheticSource) { + protected Object expected(Map fieldMapping, Object value) { var nullValue = fieldMapping.get("null_value") != null ? convert((Number) fieldMapping.get("null_value"), fieldMapping) : null; if (value instanceof List == false) { return convert(value, nullValue, fieldMapping); } - if ((boolean) fieldMapping.getOrDefault("doc_values", false)) { + boolean hasDocValues = hasDocValues(fieldMapping, true); + if (hasDocValues) { // Sorted var resultList = ((List) value).stream() .map(v -> convert(v, nullValue, fieldMapping)) diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java index cfdec40bf9190..67c6c7092b3b2 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java @@ -61,12 +61,14 @@ private void generateFields(Map document, Map arrayLength = objectArrayGenerator.lengthGenerator().get(); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java index 851812268e9ba..5f3c6a959896a 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java @@ -25,7 +25,7 @@ import org.elasticsearch.logsdb.datageneration.fields.leaf.UnsignedLongFieldDataGenerator; /** - * Lists all leaf field types that are supported for data generation. + * Lists all leaf field types that are supported for data generation by default. */ public enum FieldType { KEYWORD("keyword"), @@ -66,6 +66,25 @@ public FieldDataGenerator generator(String fieldName, DataSource dataSource) { }; } + public static FieldType tryParse(String name) { + return switch (name) { + case "keyword" -> FieldType.KEYWORD; + case "long" -> FieldType.LONG; + case "unsigned_long" -> FieldType.UNSIGNED_LONG; + case "integer" -> FieldType.INTEGER; + case "short" -> FieldType.SHORT; + case "byte" -> FieldType.BYTE; + case "double" -> FieldType.DOUBLE; + case "float" -> FieldType.FLOAT; + case "half_float" -> FieldType.HALF_FLOAT; + case "scaled_float" -> FieldType.SCALED_FLOAT; + case "counted_keyword" -> FieldType.COUNTED_KEYWORD; + case "boolean" -> FieldType.BOOLEAN; + case "date" -> FieldType.DATE; + default -> null; + }; + } + @Override public String toString() { return name; diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/MappingGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/MappingGenerator.java index 9a7add456fd67..d5ec1c7f9f36a 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/MappingGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/MappingGenerator.java @@ -112,11 +112,11 @@ private void generateMapping( ) .mappingGenerator(); - mappingParameters.put("type", leaf.type().toString()); + mappingParameters.put("type", leaf.type()); mappingParameters.putAll(mappingParametersGenerator.get()); // For simplicity we only copy to keyword fields, synthetic source logic to handle copy_to is generic. - if (leaf.type() == FieldType.KEYWORD) { + if (leaf.type().equals(FieldType.KEYWORD.toString())) { context.addCopyToCandidate(fieldName); } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Template.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Template.java index a46e07446052a..71c8a54e5e0f6 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Template.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/Template.java @@ -21,7 +21,7 @@ public record Template(Map template) { public sealed interface Entry permits Leaf, Object {} - public record Leaf(String name, FieldType type) implements Entry {} + public record Leaf(String name, String type) implements Entry {} public record Object(String name, boolean nested, Map children) implements Entry {} } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/TemplateGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/TemplateGenerator.java index f0a3a866a2673..db6abe655caa8 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/TemplateGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/TemplateGenerator.java @@ -59,7 +59,7 @@ private void generateChildFields(Map mapping, int depth, generateChildFields(children, depth + 1, nestedFieldsCount); } else { var fieldTypeInfo = fieldTypeGenerator.get(); - mapping.put(fieldName, new Template.Leaf(fieldName, fieldTypeInfo.fieldType())); + mapping.put(fieldName, new Template.Leaf(fieldName, fieldTypeInfo.fieldType().toString())); } } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSource.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSource.java index 2fe62d2c967df..cd0b9184ce42c 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSource.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSource.java @@ -32,6 +32,7 @@ public DataSource(Collection additionalHandlers) { this.handlers.addAll(additionalHandlers); + this.handlers.add(new DefaultFieldDataGeneratorHandler()); this.handlers.add(new DefaultPrimitiveTypesHandler()); this.handlers.add(new DefaultWrappersHandler()); this.handlers.add(new DefaultObjectGenerationHandler()); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java index 2a17f9311faa9..e4a3c9579595a 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java @@ -10,6 +10,10 @@ package org.elasticsearch.logsdb.datageneration.datasource; public interface DataSourceHandler { + default DataSourceResponse.FieldDataGenerator handle(DataSourceRequest.FieldDataGenerator request) { + return null; + } + default DataSourceResponse.LongGenerator handle(DataSourceRequest.LongGenerator request) { return null; } @@ -54,6 +58,14 @@ default DataSourceResponse.InstantGenerator handle(DataSourceRequest.InstantGene return null; } + default DataSourceResponse.GeoShapeGenerator handle(DataSourceRequest.GeoShapeGenerator request) { + return null; + } + + default DataSourceResponse.ShapeGenerator handle(DataSourceRequest.ShapeGenerator request) { + return null; + } + default DataSourceResponse.NullWrapper handle(DataSourceRequest.NullWrapper request) { return null; } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java index 2a1ca7297d7db..10d12acf2a5d5 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java @@ -11,7 +11,6 @@ import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; -import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping; import java.util.Set; @@ -21,6 +20,14 @@ public interface DataSourceRequest { TResponse accept(DataSourceHandler handler); + record FieldDataGenerator(String fieldName, String fieldType, DataSource dataSource) + implements + DataSourceRequest { + public DataSourceResponse.FieldDataGenerator accept(DataSourceHandler handler) { + return handler.handle(this); + } + } + record LongGenerator() implements DataSourceRequest { public DataSourceResponse.LongGenerator accept(DataSourceHandler handler) { return handler.handle(this); @@ -87,6 +94,18 @@ public DataSourceResponse.InstantGenerator accept(DataSourceHandler handler) { } } + record GeoShapeGenerator() implements DataSourceRequest { + public DataSourceResponse.GeoShapeGenerator accept(DataSourceHandler handler) { + return handler.handle(this); + } + } + + record ShapeGenerator() implements DataSourceRequest { + public DataSourceResponse.ShapeGenerator accept(DataSourceHandler handler) { + return handler.handle(this); + } + } + record NullWrapper() implements DataSourceRequest { public DataSourceResponse.NullWrapper accept(DataSourceHandler handler) { return handler.handle(this); @@ -141,7 +160,7 @@ public DataSourceResponse.ObjectArrayGenerator accept(DataSourceHandler handler) record LeafMappingParametersGenerator( String fieldName, - FieldType fieldType, + String fieldType, Set eligibleCopyToFields, DynamicMapping dynamicMapping ) implements DataSourceRequest { diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java index e7a64471e024c..780b9a8fe6cc5 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java @@ -9,7 +9,7 @@ package org.elasticsearch.logsdb.datageneration.datasource; -import org.elasticsearch.logsdb.datageneration.FieldType; +import org.elasticsearch.geometry.Geometry; import java.time.Instant; import java.util.Map; @@ -18,6 +18,8 @@ import java.util.function.Supplier; public interface DataSourceResponse { + record FieldDataGenerator(org.elasticsearch.logsdb.datageneration.FieldDataGenerator generator) implements DataSourceResponse {} + record LongGenerator(Supplier generator) implements DataSourceResponse {} record UnsignedLongGenerator(Supplier generator) implements DataSourceResponse {} @@ -40,6 +42,10 @@ record BooleanGenerator(Supplier generator) implements DataSourceRespon record InstantGenerator(Supplier generator) implements DataSourceResponse {} + record GeoShapeGenerator(Supplier generator) implements DataSourceResponse {} + + record ShapeGenerator(Supplier generator) implements DataSourceResponse {} + record NullWrapper(Function, Supplier> wrapper) implements DataSourceResponse {} record ArrayWrapper(Function, Supplier> wrapper) implements DataSourceResponse {} @@ -63,7 +69,7 @@ interface ChildFieldGenerator extends DataSourceResponse { } record FieldTypeGenerator(Supplier generator) implements DataSourceResponse { - public record FieldTypeInfo(FieldType fieldType) {} + public record FieldTypeInfo(String fieldType) {} } record ObjectArrayGenerator(Supplier> lengthGenerator) implements DataSourceResponse {} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultFieldDataGeneratorHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultFieldDataGeneratorHandler.java new file mode 100644 index 0000000000000..7287c0edd6b69 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultFieldDataGeneratorHandler.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.logsdb.datageneration.datasource; + +import org.elasticsearch.logsdb.datageneration.FieldType; + +public class DefaultFieldDataGeneratorHandler implements DataSourceHandler { + public DataSourceResponse.FieldDataGenerator handle(DataSourceRequest.FieldDataGenerator request) { + var fieldType = FieldType.tryParse(request.fieldType()); + if (fieldType == null) { + // This is a custom field type + return null; + } + + return new DataSourceResponse.FieldDataGenerator(fieldType.generator(request.fieldName(), request.dataSource())); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java index 93faf795ff565..369f26d36b54d 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java @@ -27,6 +27,12 @@ public class DefaultMappingParametersHandler implements DataSourceHandler { @Override public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { + var fieldType = FieldType.tryParse(request.fieldType()); + if (fieldType == null) { + // This is a custom field type + return null; + } + var map = new HashMap(); map.put("store", ESTestCase.randomBoolean()); map.put("index", ESTestCase.randomBoolean()); @@ -35,9 +41,9 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques map.put(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, ESTestCase.randomFrom("none", "arrays", "all")); } - return new DataSourceResponse.LeafMappingParametersGenerator(switch (request.fieldType()) { + return new DataSourceResponse.LeafMappingParametersGenerator(switch (fieldType) { case KEYWORD -> keywordMapping(request, map); - case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, HALF_FLOAT, UNSIGNED_LONG -> numberMapping(map, request.fieldType()); + case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, HALF_FLOAT, UNSIGNED_LONG -> numberMapping(map, fieldType); case SCALED_FLOAT -> scaledFloatMapping(map); case COUNTED_KEYWORD -> plain(Map.of("index", ESTestCase.randomBoolean())); case BOOLEAN -> booleanMapping(map); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultObjectGenerationHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultObjectGenerationHandler.java index 56ec676e53d55..29ffd8d965712 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultObjectGenerationHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultObjectGenerationHandler.java @@ -62,7 +62,7 @@ public String generateFieldName() { public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) { return new DataSourceResponse.FieldTypeGenerator( - () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(ESTestCase.randomFrom(FieldType.values())) + () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(ESTestCase.randomFrom(FieldType.values()).toString()) ); } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/Wrappers.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/Wrappers.java index 74be106620d46..348e8fdad098a 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/Wrappers.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/Wrappers.java @@ -19,7 +19,7 @@ public class Wrappers { * Applies default wrappers for raw values - adds nulls and wraps values in arrays. * @return */ - static Supplier defaults(Supplier rawValues, DataSource dataSource) { + public static Supplier defaults(Supplier rawValues, DataSource dataSource) { var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); @@ -30,7 +30,11 @@ static Supplier defaults(Supplier rawValues, DataSource dataSour * Applies default wrappers for raw values and also adds malformed values. * @return */ - static Supplier defaultsWithMalformed(Supplier rawValues, Supplier malformedValues, DataSource dataSource) { + public static Supplier defaultsWithMalformed( + Supplier rawValues, + Supplier malformedValues, + DataSource dataSource + ) { var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); var malformed = dataSource.get(new DataSourceRequest.MalformedWrapper(malformedValues)); var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/Matcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/Matcher.java index dd87e23351c0d..7fa82dea1a638 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/Matcher.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/Matcher.java @@ -30,7 +30,11 @@ public static MappingsStep>> matchSource() { } public interface MappingsStep { - SettingsStep mappings(XContentBuilder actualMappings, XContentBuilder expectedMappings); + SettingsStep mappings( + Map> mappingLookup, + XContentBuilder actualMappings, + XContentBuilder expectedMappings + ); } public interface SettingsStep { @@ -104,6 +108,7 @@ private static class SourceMatcherBuilder SettingsStep>>, CompareStep>>, ExpectedStep>> { + private Map> mappingLookup; private XContentBuilder expectedMappings; private XContentBuilder actualMappings; private Settings.Builder expectedSettings; @@ -121,9 +126,11 @@ public ExpectedStep>> settings(Settings.Builder actualS private SourceMatcherBuilder() {} public SettingsStep>> mappings( + final Map> mappingLookup, final XContentBuilder actualMappings, final XContentBuilder expectedMappings ) { + this.mappingLookup = mappingLookup; this.actualMappings = actualMappings; this.expectedMappings = expectedMappings; @@ -132,8 +139,16 @@ public SettingsStep>> mappings( @Override public MatchResult isEqualTo(List> actual) { - return new SourceMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings, actual, expected, ignoringSort) - .match(); + return new SourceMatcher( + mappingLookup, + actualMappings, + actualSettings, + expectedMappings, + expectedSettings, + actual, + expected, + ignoringSort + ).match(); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java index eeb33dd60e60d..cb795f1aaa4b0 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java @@ -476,6 +476,46 @@ Object convert(Object value, Object nullValue, DateTimeFormatter dateTimeFormatt } } + class ShapeMatcher implements FieldSpecificMatcher { + private final XContentBuilder actualMappings; + private final Settings.Builder actualSettings; + private final XContentBuilder expectedMappings; + private final Settings.Builder expectedSettings; + + ShapeMatcher( + XContentBuilder actualMappings, + Settings.Builder actualSettings, + XContentBuilder expectedMappings, + Settings.Builder expectedSettings + ) { + this.actualMappings = actualMappings; + this.actualSettings = actualSettings; + this.expectedMappings = expectedMappings; + this.expectedSettings = expectedSettings; + } + + @Override + public MatchResult match( + List actual, + List expected, + Map actualMapping, + Map expectedMapping + ) { + // Since fallback synthetic source is used, should always match exactly. + return actual.equals(expected) + ? MatchResult.match() + : MatchResult.noMatch( + formatErrorMessage( + actualMappings, + actualSettings, + expectedMappings, + expectedSettings, + "Values of type [geo_shape] don't match, values " + prettyPrintCollections(actual, expected) + ) + ); + } + } + /** * Generic matcher that supports common matching logic like null values. */ diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/MappingTransforms.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/MappingTransforms.java index 312be273dcd3e..2d961f1f1408e 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/MappingTransforms.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/MappingTransforms.java @@ -26,7 +26,7 @@ record FieldMapping(Map mappingParameters, List>> { + private final Map> mappingLookup; + private final Map actualNormalizedMapping; private final Map expectedNormalizedMapping; @@ -32,6 +34,7 @@ public class SourceMatcher extends GenericEqualsMatcher private final DynamicFieldMatcher dynamicFieldMatcher; public SourceMatcher( + final Map> mappingLookup, final XContentBuilder actualMappings, final Settings.Builder actualSettings, final XContentBuilder expectedMappings, @@ -42,6 +45,8 @@ public SourceMatcher( ) { super(actualMappings, actualSettings, expectedMappings, expectedSettings, actual, expected, ignoringSort); + this.mappingLookup = mappingLookup; + var actualMappingAsMap = XContentHelper.convertToMap(BytesReference.bytes(actualMappings), false, actualMappings.contentType()) .v2(); this.actualNormalizedMapping = MappingTransforms.normalizeMapping(actualMappingAsMap); @@ -95,6 +100,8 @@ public SourceMatcher( new FieldSpecificMatcher.CountedKeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings) ); put("boolean", new FieldSpecificMatcher.BooleanMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); + put("geo_shape", new FieldSpecificMatcher.ShapeMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); + put("shape", new FieldSpecificMatcher.ShapeMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); } }; this.dynamicFieldMatcher = new DynamicFieldMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings); @@ -114,8 +121,8 @@ public MatchResult match() { ); } - var sortedAndFlattenedActual = actual.stream().map(SourceTransforms::normalize).toList(); - var sortedAndFlattenedExpected = expected.stream().map(SourceTransforms::normalize).toList(); + var sortedAndFlattenedActual = actual.stream().map(s -> SourceTransforms.normalize(s, mappingLookup)).toList(); + var sortedAndFlattenedExpected = expected.stream().map(s -> SourceTransforms.normalize(s, mappingLookup)).toList(); for (int i = 0; i < sortedAndFlattenedActual.size(); i++) { var actual = sortedAndFlattenedActual.get(i); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceTransforms.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceTransforms.java index c86fe2f90b0d1..e3eb80799c416 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceTransforms.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceTransforms.java @@ -31,10 +31,10 @@ class SourceTransforms { * * @return flattened map */ - public static Map> normalize(Map map) { + public static Map> normalize(Map documentMap, Map> mappingLookup) { var flattened = new TreeMap>(); - descend(null, map, flattened); + descend(null, documentMap, flattened, mappingLookup); return flattened; } @@ -55,28 +55,44 @@ public static List normalizeValues(List values, Function tran // Synthetic source modifications: // * null values are not present // * duplicates are removed - return new ArrayList<>( - values.stream().filter(v -> v != null && Objects.equals(v, "null") == false).map(transform).collect(Collectors.toSet()) - ); + return values.stream() + .filter(v -> v != null && Objects.equals(v, "null") == false) + .map(transform) + .distinct() + .collect(Collectors.toList()); } - private static void descend(String pathFromRoot, Map currentLevel, Map> flattened) { + private static void descend( + String pathFromRoot, + Map currentLevel, + Map> flattened, + Map> mappingLookup + ) { for (var entry : currentLevel.entrySet()) { var pathToCurrentField = pathFromRoot == null ? entry.getKey() : pathFromRoot + "." + entry.getKey(); if (entry.getValue() instanceof List list) { for (var fieldValue : list) { - handleField(pathToCurrentField, fieldValue, flattened); + handleField(pathToCurrentField, fieldValue, flattened, mappingLookup); } } else { - handleField(pathToCurrentField, entry.getValue(), flattened); + handleField(pathToCurrentField, entry.getValue(), flattened, mappingLookup); } } } @SuppressWarnings("unchecked") - private static void handleField(String pathToCurrentField, Object currentField, Map> flattened) { - if (currentField instanceof Map map) { - descend(pathToCurrentField, (Map) map, flattened); + private static void handleField( + String pathToCurrentField, + Object currentField, + Map> flattened, + Map> mappingLookup + ) { + var mapping = mappingLookup.get(pathToCurrentField); + // Values of some fields are complex objects so we need to double-check that this is actually and object or a nested field + // we can descend into. + if (currentField instanceof Map map + && (mapping == null || mapping.get("type").equals("object") || mapping.get("type").equals("nested"))) { + descend(pathToCurrentField, (Map) map, flattened, mappingLookup); } else { flattened.computeIfAbsent(pathToCurrentField, k -> new ArrayList<>()).add(currentField); } diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationSnapshotTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationSnapshotTests.java index bac61daf554d5..d9f392dabe847 100644 --- a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationSnapshotTests.java +++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationSnapshotTests.java @@ -220,11 +220,11 @@ public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeG return new DataSourceResponse.FieldTypeGenerator(() -> { if (fieldType == FieldType.KEYWORD) { fieldType = FieldType.LONG; - return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.KEYWORD); + return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.KEYWORD.toString()); } fieldType = FieldType.KEYWORD; - return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.LONG); + return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.LONG.toString()); }); } @@ -235,11 +235,11 @@ public DataSourceResponse.DynamicMappingGenerator handle(DataSourceRequest.Dynam @Override public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { - if (request.fieldType() == FieldType.KEYWORD) { + if (request.fieldType().equals(FieldType.KEYWORD.toString())) { return new DataSourceResponse.LeafMappingParametersGenerator(() -> Map.of("store", "true")); } - if (request.fieldType() == FieldType.LONG) { + if (request.fieldType().equals(FieldType.LONG.toString())) { return new DataSourceResponse.LeafMappingParametersGenerator(() -> Map.of("index", "false")); } diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java index f5ba8bd02fa88..ee282425b7855 100644 --- a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java +++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/DataGenerationTests.java @@ -91,7 +91,7 @@ public DataSourceResponse.ChildFieldGenerator handle(DataSourceRequest.ChildFiel public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) { return new DataSourceResponse.FieldTypeGenerator( () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo( - FieldType.values()[generatedFields++ % FieldType.values().length] + FieldType.values()[generatedFields++ % FieldType.values().length].toString() ) ); @@ -166,7 +166,7 @@ public DataSourceResponse.ObjectArrayGenerator handle(DataSourceRequest.ObjectAr @Override public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) { return new DataSourceResponse.FieldTypeGenerator( - () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.LONG) + () -> new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(FieldType.LONG.toString()) ); } }; diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/FieldTypeTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/FieldTypeTests.java new file mode 100644 index 0000000000000..60276333f365f --- /dev/null +++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/FieldTypeTests.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.logsdb.datageneration; + +import org.elasticsearch.test.ESTestCase; + +public class FieldTypeTests extends ESTestCase { + public void testRoundTrip() { + for (var v : FieldType.values()) { + assertEquals(v.toString(), FieldType.tryParse(v.toString()).toString()); + } + } +} diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/SourceMatcherTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/SourceMatcherTests.java index aba5cbc9878e1..342394fe14b86 100644 --- a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/SourceMatcherTests.java +++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/SourceMatcherTests.java @@ -27,6 +27,7 @@ public void testDynamicMatch() throws IOException { ); var sut = new SourceMatcher( + Map.of(), XContentBuilder.builder(XContentType.JSON.xContent()).startObject().endObject(), Settings.builder(), XContentBuilder.builder(XContentType.JSON.xContent()).startObject().endObject(), @@ -49,6 +50,7 @@ public void testDynamicMismatch() throws IOException { ); var sut = new SourceMatcher( + Map.of(), XContentBuilder.builder(XContentType.JSON.xContent()).startObject().endObject(), Settings.builder(), XContentBuilder.builder(XContentType.JSON.xContent()).startObject().endObject(), @@ -77,7 +79,7 @@ public void testMappedMatch() throws IOException { mapping.endObject(); mapping.endObject(); - var sut = new SourceMatcher(mapping, Settings.builder(), mapping, Settings.builder(), values, values, false); + var sut = new SourceMatcher(Map.of(), mapping, Settings.builder(), mapping, Settings.builder(), values, values, false); assertTrue(sut.match().isMatch()); } @@ -102,7 +104,7 @@ public void testMappedMismatch() throws IOException { mapping.endObject(); mapping.endObject(); - var sut = new SourceMatcher(mapping, Settings.builder(), mapping, Settings.builder(), actual, expected, false); + var sut = new SourceMatcher(Map.of(), mapping, Settings.builder(), mapping, Settings.builder(), actual, expected, false); assertFalse(sut.match().isMatch()); } @@ -119,7 +121,7 @@ public void testCountedKeywordMatch() throws IOException { mapping.endObject(); mapping.endObject(); - var sut = new SourceMatcher(mapping, Settings.builder(), mapping, Settings.builder(), actual, expected, false); + var sut = new SourceMatcher(Map.of(), mapping, Settings.builder(), mapping, Settings.builder(), actual, expected, false); assertTrue(sut.match().isMatch()); } @@ -136,7 +138,7 @@ public void testCountedKeywordMismatch() throws IOException { mapping.endObject(); mapping.endObject(); - var sut = new SourceMatcher(mapping, Settings.builder(), mapping, Settings.builder(), actual, expected, false); + var sut = new SourceMatcher(Map.of(), mapping, Settings.builder(), mapping, Settings.builder(), actual, expected, false); assertFalse(sut.match().isMatch()); } } diff --git a/x-pack/plugin/logsdb/build.gradle b/x-pack/plugin/logsdb/build.gradle index f75e0485c9c4d..d7ee083028937 100644 --- a/x-pack/plugin/logsdb/build.gradle +++ b/x-pack/plugin/logsdb/build.gradle @@ -34,6 +34,7 @@ dependencies { compileOnly project(path: xpackModule('core')) testImplementation project(':modules:data-streams') testImplementation(testArtifact(project(xpackModule('core')))) + javaRestTestImplementation(testArtifact(project(xpackModule('spatial')))) internalClusterTestImplementation(testArtifact(project(xpackModule('core')))) } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java index fb890f3ac7ae7..f66188f1ecb1f 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java @@ -16,14 +16,22 @@ import org.elasticsearch.logsdb.datageneration.MappingGenerator; import org.elasticsearch.logsdb.datageneration.Template; import org.elasticsearch.logsdb.datageneration.TemplateGenerator; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; import org.elasticsearch.logsdb.datageneration.fields.PredefinedField; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.spatial.datageneration.GeoShapeDataSourceHandler; +import org.elasticsearch.xpack.spatial.datageneration.ShapeDataSourceHandler; import java.io.IOException; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.stream.Collectors; public class DataGenerationHelper { private final boolean keepArraySource; @@ -67,7 +75,36 @@ public DataGenerationHelper(Consumer builder (ignored) -> ESTestCase.randomLongBetween(1000, 2000) ) ) - ); + ) + .withDataSourceHandlers(List.of(new GeoShapeDataSourceHandler(), new ShapeDataSourceHandler())) + .withDataSourceHandlers(List.of(new DataSourceHandler() { + @Override + public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) { + return new DataSourceResponse.FieldTypeGenerator(new Supplier<>() { + // geo_shape and shape tends to produce really big values so let's limit how many we generate + private int shapesGenerated = 0; + + @Override + public DataSourceResponse.FieldTypeGenerator.FieldTypeInfo get() { + // Base set of field types + var options = Arrays.stream(FieldType.values()).map(FieldType::toString).collect(Collectors.toSet()); + // Custom types coming from specific functionality modules + + if (shapesGenerated < 5) { + options.add("geo_shape"); + options.add("shape"); + } + + var randomChoice = ESTestCase.randomFrom(options); + if (randomChoice.equals("geo_shape") || randomChoice.equals("shape")) { + shapesGenerated += 1; + } + + return new DataSourceResponse.FieldTypeGenerator.FieldTypeInfo(ESTestCase.randomFrom(options)); + } + }); + } + })); // Customize builder if necessary builderConfigurator.accept(specificationBuilder); @@ -80,11 +117,15 @@ public DataGenerationHelper(Consumer builder this.mapping = new MappingGenerator(specification).generate(template); } - void logsDbMapping(XContentBuilder builder) throws IOException { + Mapping mapping() { + return this.mapping; + } + + void writeLogsDbMapping(XContentBuilder builder) throws IOException { builder.map(mapping.raw()); } - void standardMapping(XContentBuilder builder) throws IOException { + void writeStandardMapping(XContentBuilder builder) throws IOException { builder.map(mapping.raw()); } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT.java index d9abdc2cde446..490f653c3ec16 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT.java @@ -38,11 +38,11 @@ public void contenderSettings(Settings.Builder builder) { @Override public void baselineMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.logsDbMapping(builder); + dataGenerationHelper.writeLogsDbMapping(builder); } @Override public void contenderMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.standardMapping(builder); + dataGenerationHelper.writeStandardMapping(builder); } } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusReindexedIntoStoredSourceChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusReindexedIntoStoredSourceChallengeRestIT.java index 776a6faf7fa07..ceaeb7cf362d6 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusReindexedIntoStoredSourceChallengeRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusReindexedIntoStoredSourceChallengeRestIT.java @@ -38,11 +38,11 @@ public void contenderSettings(Settings.Builder builder) { @Override public void baselineMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.logsDbMapping(builder); + dataGenerationHelper.writeLogsDbMapping(builder); } @Override public void contenderMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.logsDbMapping(builder); + dataGenerationHelper.writeLogsDbMapping(builder); } } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusReindexedLogsDbChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusReindexedLogsDbChallengeRestIT.java index 8b00c647b5dd0..1a90dfbbd8ac6 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusReindexedLogsDbChallengeRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/LogsDbVersusReindexedLogsDbChallengeRestIT.java @@ -38,11 +38,11 @@ public void contenderSettings(Settings.Builder builder) { @Override public void baselineMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.logsDbMapping(builder); + dataGenerationHelper.writeLogsDbMapping(builder); } @Override public void contenderMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.logsDbMapping(builder); + dataGenerationHelper.writeLogsDbMapping(builder); } } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java index 2a8c85efc863b..29fe49cffee16 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java @@ -65,12 +65,12 @@ protected StandardVersusLogsIndexModeChallengeRestIT(DataGenerationHelper dataGe @Override public void baselineMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.standardMapping(builder); + dataGenerationHelper.writeStandardMapping(builder); } @Override public void contenderMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.logsDbMapping(builder); + dataGenerationHelper.writeLogsDbMapping(builder); } @Override @@ -130,7 +130,7 @@ public void testMatchAllQuery() throws IOException { .size(numberOfDocuments); final MatchResult matchResult = Matcher.matchSource() - .mappings(getContenderMappings(), getBaselineMappings()) + .mappings(dataGenerationHelper.mapping().lookup(), getContenderMappings(), getBaselineMappings()) .settings(getContenderSettings(), getBaselineSettings()) .expected(getQueryHits(queryBaseline(searchSourceBuilder))) .ignoringSort(true) @@ -148,7 +148,7 @@ public void testTermsQuery() throws IOException { .size(numberOfDocuments); final MatchResult matchResult = Matcher.matchSource() - .mappings(getContenderMappings(), getBaselineMappings()) + .mappings(dataGenerationHelper.mapping().lookup(), getContenderMappings(), getBaselineMappings()) .settings(getContenderSettings(), getBaselineSettings()) .expected(getQueryHits(queryBaseline(searchSourceBuilder))) .ignoringSort(true) @@ -218,7 +218,7 @@ public void testEsqlSource() throws IOException { final String query = "FROM $index METADATA _source, _id | KEEP _source, _id | LIMIT " + numberOfDocuments; final MatchResult matchResult = Matcher.matchSource() - .mappings(getContenderMappings(), getBaselineMappings()) + .mappings(dataGenerationHelper.mapping().lookup(), getContenderMappings(), getBaselineMappings()) .settings(getContenderSettings(), getBaselineSettings()) .expected(getEsqlSourceResults(esqlBaseline(query))) .ignoringSort(true) diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusStandardReindexedIntoLogsDbChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusStandardReindexedIntoLogsDbChallengeRestIT.java index 5adf44f10be45..d84f12e73e5b2 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusStandardReindexedIntoLogsDbChallengeRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusStandardReindexedIntoLogsDbChallengeRestIT.java @@ -38,11 +38,11 @@ public void contenderSettings(Settings.Builder builder) { @Override public void baselineMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.standardMapping(builder); + dataGenerationHelper.writeStandardMapping(builder); } @Override public void contenderMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.logsDbMapping(builder); + dataGenerationHelper.writeLogsDbMapping(builder); } } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StoredSourceLogsDbVersusReindexedLogsDbChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StoredSourceLogsDbVersusReindexedLogsDbChallengeRestIT.java index a0672daafb243..c67707b6c3f2f 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StoredSourceLogsDbVersusReindexedLogsDbChallengeRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StoredSourceLogsDbVersusReindexedLogsDbChallengeRestIT.java @@ -39,11 +39,11 @@ public void contenderSettings(Settings.Builder builder) { @Override public void baselineMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.logsDbMapping(builder); + dataGenerationHelper.writeLogsDbMapping(builder); } @Override public void contenderMappings(XContentBuilder builder) throws IOException { - dataGenerationHelper.logsDbMapping(builder); + dataGenerationHelper.writeLogsDbMapping(builder); } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 3226ebb277e15..76e5063885994 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -364,7 +364,7 @@ private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBloc var nullValueEncoded = nullValueFormatted != null ? (Number) unsignedToSortableSignedLong(parseUnsignedLong(nullValueFormatted)) : null; - return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport(nullValueFormatted) { + return new FallbackSyntheticSourceBlockLoader.SingleValueReader(nullValueFormatted) { @Override public void convertValue(Object value, List accumulator) { if (value.equals("")) { diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldBlockLoaderTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldBlockLoaderTests.java index 562b1da7f07e3..98fa464605fdc 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldBlockLoaderTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldBlockLoaderTests.java @@ -18,8 +18,8 @@ public class UnsignedLongFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { private static final long MASK_2_63 = 0x8000000000000000L; - public UnsignedLongFieldBlockLoaderTests() { - super(FieldType.UNSIGNED_LONG); + public UnsignedLongFieldBlockLoaderTests(Params params) { + super(FieldType.UNSIGNED_LONG, params); } @Override diff --git a/x-pack/plugin/spatial/build.gradle b/x-pack/plugin/spatial/build.gradle index 1e7b21b6294a9..fb71a8bf21fee 100644 --- a/x-pack/plugin/spatial/build.gradle +++ b/x-pack/plugin/spatial/build.gradle @@ -2,6 +2,7 @@ import org.elasticsearch.gradle.internal.info.BuildParams apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-cluster-test' +apply plugin: 'elasticsearch.internal-test-artifact' esplugin { name = 'spatial' diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index f7c5f1b8072f3..558bd2b5a3faa 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -193,6 +193,7 @@ public GeoShapeWithDocValuesFieldMapper build(MapperBuilderContext context) { parser, scriptValues(), geoFormatterFactory, + context.isSourceSynthetic(), meta.get() ); hasScript = script.get() != null; @@ -212,6 +213,7 @@ public static final class GeoShapeWithDocValuesFieldType extends AbstractShapeGe private final GeoFormatterFactory geoFormatterFactory; private final FieldValues scriptValues; + private final boolean isSyntheticSource; public GeoShapeWithDocValuesFieldType( String name, @@ -222,11 +224,13 @@ public GeoShapeWithDocValuesFieldType( GeoShapeParser parser, FieldValues scriptValues, GeoFormatterFactory geoFormatterFactory, + boolean isSyntheticSource, Map meta ) { super(name, indexed, isStored, hasDocValues, parser, orientation, meta); this.scriptValues = scriptValues; this.geoFormatterFactory = geoFormatterFactory; + this.isSyntheticSource = isSyntheticSource; } @Override @@ -302,9 +306,14 @@ protected Function, List> getFormatter(String format) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return blContext.fieldExtractPreference() == FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS - ? new GeoBoundsBlockLoader(name()) - : blockLoaderFromSource(blContext); + if (blContext.fieldExtractPreference() == FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS) { + return new GeoBoundsBlockLoader(name()); + } + if (isSyntheticSource) { + return blockLoaderFromFallbackSyntheticSource(blContext); + } + + return blockLoaderFromSource(blContext); } static class GeoBoundsBlockLoader extends AbstractShapeGeometryFieldMapper.AbstractShapeGeometryFieldType.BoundsBlockLoader { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java index c5f5f64d3e3d8..472ba687dab87 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java @@ -122,6 +122,7 @@ public ShapeFieldMapper build(MapperBuilderContext context) { hasDocValues.get(), orientation.get().value(), parser, + context.isSourceSynthetic(), meta.get() ); return new ShapeFieldMapper(leafName(), ft, builderParams(this, context), parser, this); @@ -138,6 +139,7 @@ public ShapeFieldMapper build(MapperBuilderContext context) { ); public static final class ShapeFieldType extends AbstractShapeGeometryFieldType implements ShapeQueryable { + private final boolean isSyntheticSource; public ShapeFieldType( String name, @@ -145,9 +147,11 @@ public ShapeFieldType( boolean hasDocValues, Orientation orientation, Parser parser, + boolean isSyntheticSource, Map meta ) { super(name, indexed, false, hasDocValues, parser, orientation, meta); + this.isSyntheticSource = isSyntheticSource; } @Override @@ -189,9 +193,15 @@ protected Function, List> getFormatter(String format) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return blContext.fieldExtractPreference() == FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS - ? new CartesianBoundsBlockLoader(name()) - : blockLoaderFromSource(blContext); + if (blContext.fieldExtractPreference() == FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS) { + return new CartesianBoundsBlockLoader(name()); + } + + if (isSyntheticSource) { + return blockLoaderFromFallbackSyntheticSource(blContext); + } + + return blockLoaderFromSource(blContext); } static class CartesianBoundsBlockLoader extends BoundsBlockLoader { diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeDataSourceHandler.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeDataSourceHandler.java new file mode 100644 index 0000000000000..183bb4b22d551 --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeDataSourceHandler.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.datageneration; + +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.ShapeType; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.spatial.util.GeoTestUtils; + +import java.util.HashMap; + +public class GeoShapeDataSourceHandler implements DataSourceHandler { + @Override + public DataSourceResponse.GeoShapeGenerator handle(DataSourceRequest.GeoShapeGenerator request) { + return new DataSourceResponse.GeoShapeGenerator(this::generateValidGeoShape); + } + + @Override + public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { + if (request.fieldType().equals("geo_shape") == false) { + return null; + } + + return new DataSourceResponse.LeafMappingParametersGenerator(() -> { + var map = new HashMap(); + map.put("store", ESTestCase.randomBoolean()); + map.put("index", ESTestCase.randomBoolean()); + map.put("doc_values", ESTestCase.randomBoolean()); + + if (ESTestCase.randomBoolean()) { + map.put("ignore_malformed", ESTestCase.randomBoolean()); + } + + return map; + }); + } + + @Override + public DataSourceResponse.FieldDataGenerator handle(DataSourceRequest.FieldDataGenerator request) { + if (request.fieldType().equals("geo_shape") == false) { + return null; + } + + return new DataSourceResponse.FieldDataGenerator(new GeoShapeFieldDataGenerator(request.dataSource())); + } + + private Geometry generateValidGeoShape() { + while (true) { + var geometry = GeometryTestUtils.randomGeometryWithoutCircle(0, false); + if (geometry.type() == ShapeType.ENVELOPE) { + // Not supported in GeoJson, skip it + continue; + } + try { + GeoTestUtils.binaryGeoShapeDocValuesField("f", geometry); + return geometry; + } catch (IllegalArgumentException ignored) { + // Some generated shapes are not suitable for the storage format, ignore them + } + } + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeFieldDataGenerator.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeFieldDataGenerator.java new file mode 100644 index 0000000000000..a8650b50def45 --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/GeoShapeFieldDataGenerator.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.datageneration; + +import org.elasticsearch.common.geo.GeoJson; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.datasource.DataSource; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.fields.leaf.Wrappers; + +import java.util.Map; +import java.util.function.Supplier; + +public class GeoShapeFieldDataGenerator implements FieldDataGenerator { + private final Supplier formattedGeoShapes; + private final Supplier formattedGeoShapesWithMalformed; + + public GeoShapeFieldDataGenerator(DataSource dataSource) { + var geoShapes = dataSource.get(new DataSourceRequest.GeoShapeGenerator()).generator(); + var serializeToGeoJson = dataSource.get(new DataSourceRequest.TransformWrapper(0.5, g -> GeoJson.toMap((Geometry) g))); + + var formattedGeoShapes = serializeToGeoJson.wrapper().andThen(values -> (Supplier) () -> { + var value = values.get(); + if (value instanceof Geometry g) { + // did not transform + return WellKnownText.toWKT(g); + } + return value; + }).apply(geoShapes::get); + this.formattedGeoShapes = Wrappers.defaults(formattedGeoShapes, dataSource); + + var longs = dataSource.get(new DataSourceRequest.LongGenerator()).generator(); + this.formattedGeoShapesWithMalformed = Wrappers.defaultsWithMalformed(formattedGeoShapes, longs::get, dataSource); + } + + @Override + public Object generateValue(Map fieldMapping) { + if (fieldMapping == null) { + // dynamically mapped and dynamic mapping does not play well with this type (it gets mapped as an object) + // return null to skip indexing this field + return null; + } + + if (fieldMapping != null && (Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) { + return formattedGeoShapesWithMalformed.get(); + } + + return formattedGeoShapes.get(); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/ShapeDataSourceHandler.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/ShapeDataSourceHandler.java new file mode 100644 index 0000000000000..eb6bb1b9b34e0 --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/ShapeDataSourceHandler.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.datageneration; + +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.ShapeType; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.spatial.util.GeoTestUtils; + +import java.util.HashMap; + +public class ShapeDataSourceHandler implements DataSourceHandler { + @Override + public DataSourceResponse.ShapeGenerator handle(DataSourceRequest.ShapeGenerator request) { + return new DataSourceResponse.ShapeGenerator(this::generateValidShape); + } + + @Override + public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { + if (request.fieldType().equals("shape") == false) { + return null; + } + + return new DataSourceResponse.LeafMappingParametersGenerator(() -> { + var map = new HashMap(); + map.put("index", ESTestCase.randomBoolean()); + map.put("doc_values", ESTestCase.randomBoolean()); + + if (ESTestCase.randomBoolean()) { + map.put("ignore_malformed", ESTestCase.randomBoolean()); + } + + return map; + }); + } + + @Override + public DataSourceResponse.FieldDataGenerator handle(DataSourceRequest.FieldDataGenerator request) { + if (request.fieldType().equals("shape") == false) { + return null; + } + + return new DataSourceResponse.FieldDataGenerator(new ShapeFieldDataGenerator(request.dataSource())); + } + + private Geometry generateValidShape() { + while (true) { + var geometry = GeometryTestUtils.randomGeometryWithoutCircle(0, false); + if (geometry.type() == ShapeType.ENVELOPE) { + // Not supported in GeoJson, skip it + continue; + } + try { + GeoTestUtils.binaryCartesianShapeDocValuesField("f", geometry); + return geometry; + } catch (IllegalArgumentException ignored) { + // Some generated shapes are not suitable for the storage format, ignore them + } + } + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/ShapeFieldDataGenerator.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/ShapeFieldDataGenerator.java new file mode 100644 index 0000000000000..78f1e8ee1745e --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/datageneration/ShapeFieldDataGenerator.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.datageneration; + +import org.elasticsearch.common.geo.GeoJson; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.datasource.DataSource; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import org.elasticsearch.logsdb.datageneration.fields.leaf.Wrappers; + +import java.util.Map; +import java.util.function.Supplier; + +public class ShapeFieldDataGenerator implements FieldDataGenerator { + private final Supplier formattedShapes; + private final Supplier formattedShapesWithMalformed; + + public ShapeFieldDataGenerator(DataSource dataSource) { + var shapes = dataSource.get(new DataSourceRequest.ShapeGenerator()).generator(); + var serializeToGeoJson = dataSource.get(new DataSourceRequest.TransformWrapper(0.5, g -> GeoJson.toMap((Geometry) g))); + + var formattedShapes = serializeToGeoJson.wrapper().andThen(values -> (Supplier) () -> { + var value = values.get(); + if (value instanceof Geometry g) { + // did not transform + return WellKnownText.toWKT(g); + } + return value; + }).apply(shapes::get); + this.formattedShapes = Wrappers.defaults(formattedShapes, dataSource); + + var longs = dataSource.get(new DataSourceRequest.LongGenerator()).generator(); + this.formattedShapesWithMalformed = Wrappers.defaultsWithMalformed(formattedShapes, longs::get, dataSource); + } + + @Override + public Object generateValue(Map fieldMapping) { + if (fieldMapping == null) { + // dynamically mapped and dynamic mapping does not play well with this type (it gets mapped as an object) + // return null to skip indexing this field + return null; + } + + if (fieldMapping != null && (Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) { + return formattedShapesWithMalformed.get(); + } + + return formattedShapes.get(); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeFieldBlockLoaderTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeFieldBlockLoaderTests.java new file mode 100644 index 0000000000000..2c9a70a66514f --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeFieldBlockLoaderTests.java @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.index.mapper; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.geo.GeoJson; +import org.elasticsearch.common.geo.GeometryNormalizer; +import org.elasticsearch.common.geo.Orientation; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.utils.GeographyValidator; +import org.elasticsearch.geometry.utils.WellKnownBinary; +import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.index.mapper.BlockLoaderTestCase; +import org.elasticsearch.plugins.ExtensiblePlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xcontent.support.MapXContentParser; +import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; +import org.elasticsearch.xpack.spatial.datageneration.GeoShapeDataSourceHandler; + +import java.nio.ByteOrder; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class GeoShapeFieldBlockLoaderTests extends BlockLoaderTestCase { + public GeoShapeFieldBlockLoaderTests(Params params) { + super("geo_shape", List.of(new GeoShapeDataSourceHandler()), params); + } + + @Override + @SuppressWarnings("unchecked") + protected Object expected(Map fieldMapping, Object value) { + if (value instanceof List == false) { + return convert(value); + } + + // TODO FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS is currently not covered, it needs special logic + // As a result we always load from source (stored or fallback synthetic) and they should work the same. + var resultList = ((List) value).stream().map(this::convert).filter(Objects::nonNull).toList(); + return maybeFoldList(resultList); + } + + private Object convert(Object value) { + if (value instanceof String s) { + return toWKB(fromWKT(s)); + } + + if (value instanceof Map m) { + return toWKB(fromGeoJson(m)); + } + + // Malformed values are excluded + return null; + } + + private Geometry fromWKT(String s) { + try { + var geometry = WellKnownText.fromWKT(GeographyValidator.instance(true), false, s); + return normalize(geometry); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @SuppressWarnings("unchecked") + private Geometry fromGeoJson(Map map) { + try { + var parser = new MapXContentParser( + xContentRegistry(), + LoggingDeprecationHandler.INSTANCE, + (Map) map, + XContentType.JSON + ); + parser.nextToken(); + + var geometry = GeoJson.fromXContent(GeographyValidator.instance(true), false, true, parser); + return normalize(geometry); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private Geometry normalize(Geometry geometry) { + if (GeometryNormalizer.needsNormalize(Orientation.RIGHT, geometry)) { + return GeometryNormalizer.apply(Orientation.RIGHT, geometry); + } + + return geometry; + } + + private BytesRef toWKB(Geometry geometry) { + return new BytesRef(WellKnownBinary.toWKB(geometry, ByteOrder.LITTLE_ENDIAN)); + } + + @Override + protected Collection getPlugins() { + var plugin = new LocalStateSpatialPlugin(); + plugin.loadExtensions(new ExtensiblePlugin.ExtensionLoader() { + @Override + public List loadExtensions(Class extensionPointType) { + return List.of(); + } + }); + + return Collections.singletonList(plugin); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldBlockLoaderTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldBlockLoaderTests.java new file mode 100644 index 0000000000000..3acd64c501a4a --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldBlockLoaderTests.java @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.index.mapper; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.geo.GeoJson; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.utils.StandardValidator; +import org.elasticsearch.geometry.utils.WellKnownBinary; +import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.index.mapper.BlockLoaderTestCase; +import org.elasticsearch.plugins.ExtensiblePlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xcontent.support.MapXContentParser; +import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; +import org.elasticsearch.xpack.spatial.datageneration.ShapeDataSourceHandler; + +import java.nio.ByteOrder; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class ShapeFieldBlockLoaderTests extends BlockLoaderTestCase { + public ShapeFieldBlockLoaderTests(Params params) { + super("shape", List.of(new ShapeDataSourceHandler()), params); + } + + @Override + @SuppressWarnings("unchecked") + protected Object expected(Map fieldMapping, Object value) { + if (value instanceof List == false) { + return convert(value); + } + + // TODO FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS is currently not covered, it needs special logic + // As a result we always load from source (stored or fallback synthetic) and they should work the same. + var resultList = ((List) value).stream().map(this::convert).filter(Objects::nonNull).toList(); + return maybeFoldList(resultList); + } + + private Object convert(Object value) { + if (value instanceof String s) { + return toWKB(fromWKT(s)); + } + + if (value instanceof Map m) { + return toWKB(fromGeoJson(m)); + } + + // Malformed values are excluded + return null; + } + + private Geometry fromWKT(String s) { + try { + return WellKnownText.fromWKT(StandardValidator.instance(true), false, s); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @SuppressWarnings("unchecked") + private Geometry fromGeoJson(Map map) { + try { + var parser = new MapXContentParser( + xContentRegistry(), + LoggingDeprecationHandler.INSTANCE, + (Map) map, + XContentType.JSON + ); + parser.nextToken(); + + return GeoJson.fromXContent(StandardValidator.instance(true), false, true, parser); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private BytesRef toWKB(Geometry geometry) { + return new BytesRef(WellKnownBinary.toWKB(geometry, ByteOrder.LITTLE_ENDIAN)); + } + + @Override + protected Collection getPlugins() { + var plugin = new LocalStateSpatialPlugin(); + plugin.loadExtensions(new ExtensiblePlugin.ExtensionLoader() { + @Override + public List loadExtensions(Class extensionPointType) { + return List.of(); + } + }); + + return Collections.singletonList(plugin); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java index 66f5597be543e..d377604ba283b 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java @@ -206,6 +206,7 @@ public void testGeoShapeQueryAcrossDateline() throws IOException { null, null, null, + false, Collections.emptyMap() ); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java index 834d04e49014c..3e339ea35b93d 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java @@ -259,6 +259,7 @@ protected void testCase( null, null, null, + false, Collections.emptyMap() ); testCase( diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianShapeBoundsAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianShapeBoundsAggregatorTests.java index 8f479c7ed22c3..5515db1b9120e 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianShapeBoundsAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianShapeBoundsAggregatorTests.java @@ -71,6 +71,7 @@ public void testEmpty() throws Exception { true, Orientation.RIGHT, null, + false, Collections.emptyMap() ); try (IndexReader reader = w.getReader()) { @@ -100,6 +101,7 @@ public void testUnmappedFieldWithDocs() throws Exception { true, Orientation.RIGHT, null, + false, Collections.emptyMap() ); try (IndexReader reader = w.getReader()) { @@ -125,6 +127,7 @@ public void testMissing() throws Exception { true, Orientation.RIGHT, null, + false, Collections.emptyMap() ); @@ -158,6 +161,7 @@ public void testInvalidMissing() throws Exception { true, Orientation.RIGHT, null, + false, Collections.emptyMap() ); @@ -248,6 +252,7 @@ private void readAndAssertExtent(RandomIndexWriter w, TestPointCollection points true, Orientation.RIGHT, null, + false, Collections.emptyMap() ); try (IndexReader reader = w.getReader()) { diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianShapeCentroidAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianShapeCentroidAggregatorTests.java index 7d4ee29ffee0f..e1d3afe9e3e58 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianShapeCentroidAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianShapeCentroidAggregatorTests.java @@ -58,6 +58,7 @@ public void testEmpty() throws Exception { true, Orientation.RIGHT, null, + false, Collections.emptyMap() ); try (IndexReader reader = w.getReader()) { @@ -82,12 +83,21 @@ public void testUnmapped() throws Exception { true, Orientation.RIGHT, null, + false, Collections.emptyMap() ); InternalCartesianCentroid result = searchAndReduce(reader, new AggTestConfig(aggBuilder, fieldType)); assertNull(result.centroid()); - fieldType = new ShapeFieldMapper.ShapeFieldType("field", true, true, Orientation.RIGHT, null, Collections.emptyMap()); + fieldType = new ShapeFieldMapper.ShapeFieldType( + "field", + true, + true, + Orientation.RIGHT, + null, + false, + Collections.emptyMap() + ); result = searchAndReduce(reader, new AggTestConfig(aggBuilder, fieldType)); assertNull(result.centroid()); assertFalse(AggregationInspectionHelper.hasValue(result)); @@ -112,6 +122,7 @@ public void testUnmappedWithMissing() throws Exception { true, Orientation.RIGHT, null, + false, Collections.emptyMap() ); InternalCartesianCentroid result = searchAndReduce(reader, new AggTestConfig(aggBuilder, fieldType)); @@ -185,6 +196,7 @@ private void assertCentroid(RandomIndexWriter w, CartesianPoint expectedCentroid true, Orientation.RIGHT, null, + false, Collections.emptyMap() ); CartesianCentroidAggregationBuilder aggBuilder = new CartesianCentroidAggregationBuilder("my_agg").field("field"); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeBoundsAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeBoundsAggregatorTests.java index 50eb9fda9e818..027408c273bd3 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeBoundsAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeBoundsAggregatorTests.java @@ -62,6 +62,7 @@ public void testEmpty() throws Exception { null, null, null, + false, Collections.emptyMap() ); try (IndexReader reader = w.getReader()) { @@ -96,6 +97,7 @@ public void testUnmappedFieldWithDocs() throws Exception { null, null, null, + false, Collections.emptyMap() ); try (IndexReader reader = w.getReader()) { @@ -126,6 +128,7 @@ public void testMissing() throws Exception { null, null, null, + false, Collections.emptyMap() ); @@ -165,6 +168,7 @@ public void testInvalidMissing() throws Exception { null, null, null, + false, Collections.emptyMap() ); @@ -230,6 +234,7 @@ public void testRandomShapes() throws Exception { null, null, null, + false, Collections.emptyMap() ); try (IndexReader reader = w.getReader()) { diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregatorTests.java index 86ce455d372e7..c27bbc61f01e2 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregatorTests.java @@ -67,6 +67,7 @@ public void testEmpty() throws Exception { null, null, null, + false, Map.of() ); try (IndexReader reader = w.getReader()) { @@ -94,6 +95,7 @@ public void testUnmapped() throws Exception { null, null, null, + false, Map.of() ); InternalGeoCentroid result = searchAndReduce(reader, new AggTestConfig(aggBuilder, fieldType)); @@ -108,6 +110,7 @@ public void testUnmapped() throws Exception { null, null, null, + false, Map.of() ); result = searchAndReduce(reader, new AggTestConfig(aggBuilder, fieldType)); @@ -138,6 +141,7 @@ public void testUnmappedWithMissing() throws Exception { null, null, null, + false, Map.of() ); InternalGeoCentroid result = searchAndReduce(reader, new AggTestConfig(aggBuilder, fieldType)); @@ -214,6 +218,7 @@ private void assertCentroid(RandomIndexWriter w, GeoPoint expectedCentroid) thro null, null, null, + false, Map.of() ); GeoCentroidAggregationBuilder aggBuilder = new GeoCentroidAggregationBuilder("my_agg").field("field");