diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java index 9b2878ff7bfc8..cfdec40bf9190 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java @@ -40,7 +40,10 @@ public DocumentGenerator(DataGeneratorSpecification specification) { public Map generate(Template template, Mapping mapping) { var documentMap = new TreeMap(); for (var predefinedField : specification.predefinedFields()) { - documentMap.put(predefinedField.name(), predefinedField.generator(specification.dataSource()).generateValue()); + documentMap.put( + predefinedField.name(), + predefinedField.generator(specification.dataSource()).generateValue(predefinedField.mapping()) + ); } generateFields(documentMap, template.template(), new Context("", mapping.lookup())); @@ -53,16 +56,18 @@ private void generateFields(Map document, Map arrayLength = objectArrayGenerator.lengthGenerator().get(); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldDataGenerator.java index 7e28a0a0fab25..1d84c5adc4eda 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldDataGenerator.java @@ -9,11 +9,13 @@ package org.elasticsearch.logsdb.datageneration; +import java.util.Map; + /** * Entity responsible for generating a valid value for a field. * * Generator is expected to produce a different value on every call. */ public interface FieldDataGenerator { - Object generateValue(); + Object generateValue(Map fieldMapping); } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java index 3aad8e55c326c..beef9fb4dd799 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java @@ -58,6 +58,10 @@ default DataSourceResponse.RepeatingWrapper handle(DataSourceRequest.RepeatingWr return null; } + default DataSourceResponse.MalformedWrapper handle(DataSourceRequest.MalformedWrapper request) { + return null; + } + default DataSourceResponse.ChildFieldGenerator handle(DataSourceRequest.ChildFieldGenerator request) { return null; } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java index 999e9a8bf6c7b..0e6e796ff6d54 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java @@ -15,6 +15,7 @@ import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping; import java.util.Set; +import java.util.function.Supplier; public interface DataSourceRequest { TResponse accept(DataSourceHandler handler); @@ -91,6 +92,12 @@ public DataSourceResponse.RepeatingWrapper accept(DataSourceHandler handler) { } } + record MalformedWrapper(Supplier malformedValues) implements DataSourceRequest { + public DataSourceResponse.MalformedWrapper accept(DataSourceHandler handler) { + return handler.handle(this); + } + } + record ChildFieldGenerator(DataGeneratorSpecification specification) implements DataSourceRequest { diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java index 3722e75d916dc..e9f1adb98d248 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java @@ -41,6 +41,8 @@ record ArrayWrapper(Function, Supplier> wrapper) implem record RepeatingWrapper(Function, Supplier> wrapper) implements DataSourceResponse {} + record MalformedWrapper(Function, Supplier> wrapper) implements DataSourceResponse {} + interface ChildFieldGenerator extends DataSourceResponse { int generateChildFieldCount(); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java index 702145dd9a503..bf99ab71d0149 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java @@ -11,6 +11,7 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.ObjectMapper; +import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.test.ESTestCase; import java.util.HashMap; @@ -31,7 +32,7 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques return new DataSourceResponse.LeafMappingParametersGenerator(switch (request.fieldType()) { case KEYWORD -> keywordMapping(request, map); - case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, HALF_FLOAT, UNSIGNED_LONG -> plain(map); + case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, HALF_FLOAT, UNSIGNED_LONG -> numberMapping(map, request.fieldType()); case SCALED_FLOAT -> scaledFloatMapping(map); case COUNTED_KEYWORD -> plain(Map.of("index", ESTestCase.randomBoolean())); }); @@ -41,6 +42,30 @@ private Supplier> plain(Map injected) { return () -> injected; } + private Supplier> numberMapping(Map injected, FieldType fieldType) { + return () -> { + if (ESTestCase.randomBoolean()) { + injected.put("ignore_malformed", ESTestCase.randomBoolean()); + } + if (ESTestCase.randomDouble() <= 0.2) { + Number value = switch (fieldType) { + case LONG -> ESTestCase.randomLong(); + case UNSIGNED_LONG -> ESTestCase.randomNonNegativeLong(); + case INTEGER -> ESTestCase.randomInt(); + case SHORT -> ESTestCase.randomShort(); + case BYTE -> ESTestCase.randomByte(); + case DOUBLE -> ESTestCase.randomDouble(); + case FLOAT, HALF_FLOAT -> ESTestCase.randomFloat(); + default -> throw new IllegalStateException("Unexpected field type"); + }; + + injected.put("null_value", value); + } + + return injected; + }; + } + private Supplier> keywordMapping( DataSourceRequest.LeafMappingParametersGenerator request, Map injected @@ -75,6 +100,15 @@ private Supplier> keywordMapping( private Supplier> scaledFloatMapping(Map injected) { return () -> { injected.put("scaling_factor", ESTestCase.randomFrom(10, 1000, 100000, 100.5)); + + if (ESTestCase.randomDouble() <= 0.2) { + injected.put("null_value", ESTestCase.randomFloat()); + } + + if (ESTestCase.randomBoolean()) { + injected.put("ignore_malformed", ESTestCase.randomBoolean()); + } + return injected; }; } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultWrappersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultWrappersHandler.java index 3640f98e25b55..ac686e0201327 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultWrappersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultWrappersHandler.java @@ -32,6 +32,11 @@ public DataSourceResponse.RepeatingWrapper handle(DataSourceRequest.RepeatingWra return new DataSourceResponse.RepeatingWrapper(repeatValues()); } + @Override + public DataSourceResponse.MalformedWrapper handle(DataSourceRequest.MalformedWrapper request) { + return new DataSourceResponse.MalformedWrapper(injectMalformed(request.malformedValues())); + } + private static Function, Supplier> injectNulls() { // Inject some nulls but majority of data should be non-null (as it likely is in reality). return (values) -> () -> ESTestCase.randomDouble() <= 0.05 ? null : values.get(); @@ -62,4 +67,8 @@ private static Function, Supplier> repeatValues() { }; }; } + + private static Function, Supplier> injectMalformed(Supplier malformedValues) { + return (values) -> () -> ESTestCase.randomDouble() <= 0.1 ? malformedValues.get() : values.get(); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java index 4ead8ffd0b718..809354c01e94c 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java @@ -13,21 +13,28 @@ import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import java.util.Map; import java.util.function.Supplier; public class ByteFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; + private final Supplier valueGeneratorWithMalformed; public ByteFieldDataGenerator(String fieldName, DataSource dataSource) { - var bytes = dataSource.get(new DataSourceRequest.ByteGenerator()); - var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); - var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); + var bytes = dataSource.get(new DataSourceRequest.ByteGenerator()).generator(); - this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> bytes.generator().get()); + this.valueGenerator = Wrappers.defaults(bytes::get, dataSource); + + var strings = dataSource.get(new DataSourceRequest.StringGenerator()).generator(); + this.valueGeneratorWithMalformed = Wrappers.defaultsWithMalformed(bytes::get, strings::get, dataSource); } @Override - public Object generateValue() { + public Object generateValue(Map fieldMapping) { + if (fieldMapping != null && (Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) { + return valueGeneratorWithMalformed.get(); + } + return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/CountedKeywordFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/CountedKeywordFieldDataGenerator.java index f09ef2397bfc7..64a40704a64ce 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/CountedKeywordFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/CountedKeywordFieldDataGenerator.java @@ -14,6 +14,7 @@ import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import java.util.HashSet; +import java.util.Map; import java.util.Set; import java.util.function.Supplier; @@ -31,7 +32,7 @@ public CountedKeywordFieldDataGenerator(String fieldName, DataSource dataSource) } @Override - public Object generateValue() { + public Object generateValue(Map fieldMapping) { return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java index cf2c4f6abdbf4..8cd72c93cb6d1 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java @@ -13,21 +13,28 @@ import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import java.util.Map; import java.util.function.Supplier; public class DoubleFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; + private final Supplier valueGeneratorWithMalformed; public DoubleFieldDataGenerator(String fieldName, DataSource dataSource) { - var doubles = dataSource.get(new DataSourceRequest.DoubleGenerator()); - var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); - var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); + var doubles = dataSource.get(new DataSourceRequest.DoubleGenerator()).generator(); - this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> doubles.generator().get()); + this.valueGenerator = Wrappers.defaults(doubles::get, dataSource); + + var strings = dataSource.get(new DataSourceRequest.StringGenerator()).generator(); + this.valueGeneratorWithMalformed = Wrappers.defaultsWithMalformed(doubles::get, strings::get, dataSource); } @Override - public Object generateValue() { + public Object generateValue(Map fieldMapping) { + if (fieldMapping != null && (Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) { + return valueGeneratorWithMalformed.get(); + } + return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java index b59d5ceabb188..651c1e40e4100 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java @@ -13,21 +13,28 @@ import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import java.util.Map; import java.util.function.Supplier; public class FloatFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; + private final Supplier valueGeneratorWithMalformed; public FloatFieldDataGenerator(String fieldName, DataSource dataSource) { - var floats = dataSource.get(new DataSourceRequest.FloatGenerator()); - var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); - var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); + var floats = dataSource.get(new DataSourceRequest.FloatGenerator()).generator(); - this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> floats.generator().get()); + this.valueGenerator = Wrappers.defaults(floats::get, dataSource); + + var strings = dataSource.get(new DataSourceRequest.StringGenerator()).generator(); + this.valueGeneratorWithMalformed = Wrappers.defaultsWithMalformed(floats::get, strings::get, dataSource); } @Override - public Object generateValue() { + public Object generateValue(Map fieldMapping) { + if (fieldMapping != null && (Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) { + return valueGeneratorWithMalformed.get(); + } + return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java index e2ed299f1a4dc..35de92b093084 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java @@ -13,21 +13,28 @@ import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import java.util.Map; import java.util.function.Supplier; public class HalfFloatFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; + private final Supplier valueGeneratorWithMalformed; public HalfFloatFieldDataGenerator(String fieldName, DataSource dataSource) { - var halfFloats = dataSource.get(new DataSourceRequest.HalfFloatGenerator()); - var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); - var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); + var halfFloats = dataSource.get(new DataSourceRequest.HalfFloatGenerator()).generator(); - this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> halfFloats.generator().get()); + this.valueGenerator = Wrappers.defaults(halfFloats::get, dataSource); + + var strings = dataSource.get(new DataSourceRequest.StringGenerator()).generator(); + this.valueGeneratorWithMalformed = Wrappers.defaultsWithMalformed(halfFloats::get, strings::get, dataSource); } @Override - public Object generateValue() { + public Object generateValue(Map fieldMapping) { + if (fieldMapping != null && (Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) { + return valueGeneratorWithMalformed.get(); + } + return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java index f2fe8ed8362e5..0d7d8194c4001 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java @@ -13,21 +13,28 @@ import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import java.util.Map; import java.util.function.Supplier; public class IntegerFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; + private final Supplier valueGeneratorWithMalformed; public IntegerFieldDataGenerator(String fieldName, DataSource dataSource) { - var ints = dataSource.get(new DataSourceRequest.IntegerGenerator()); - var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); - var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); + var ints = dataSource.get(new DataSourceRequest.IntegerGenerator()).generator(); - this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> ints.generator().get()); + this.valueGenerator = Wrappers.defaults(ints::get, dataSource); + + var strings = dataSource.get(new DataSourceRequest.StringGenerator()).generator(); + this.valueGeneratorWithMalformed = Wrappers.defaultsWithMalformed(ints::get, strings::get, dataSource); } @Override - public Object generateValue() { + public Object generateValue(Map fieldMapping) { + if (fieldMapping != null && (Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) { + return valueGeneratorWithMalformed.get(); + } + return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java index 8dc4d8b8767c4..4dbbc6b740dc2 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java @@ -13,6 +13,7 @@ import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import java.util.Map; import java.util.function.Supplier; public class KeywordFieldDataGenerator implements FieldDataGenerator { @@ -27,7 +28,7 @@ public KeywordFieldDataGenerator(String fieldName, DataSource dataSource) { } @Override - public Object generateValue() { + public Object generateValue(Map fieldMapping) { return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java index f17610e501ed7..bd22d58af16a7 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/LongFieldDataGenerator.java @@ -13,21 +13,28 @@ import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import java.util.Map; import java.util.function.Supplier; public class LongFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; + private final Supplier valueGeneratorWithMalformed; public LongFieldDataGenerator(String fieldName, DataSource dataSource) { - var longs = dataSource.get(new DataSourceRequest.LongGenerator()); - var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); - var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); + var longs = dataSource.get(new DataSourceRequest.LongGenerator()).generator(); - this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> longs.generator().get()); + this.valueGenerator = Wrappers.defaults(longs::get, dataSource); + + var strings = dataSource.get(new DataSourceRequest.StringGenerator()).generator(); + this.valueGeneratorWithMalformed = Wrappers.defaultsWithMalformed(longs::get, strings::get, dataSource); } @Override - public Object generateValue() { + public Object generateValue(Map fieldMapping) { + if (fieldMapping != null && (Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) { + return valueGeneratorWithMalformed.get(); + } + return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java index 008dd04179dcd..117682e739461 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ScaledFloatFieldDataGenerator.java @@ -13,21 +13,28 @@ import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import java.util.Map; import java.util.function.Supplier; public class ScaledFloatFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; + private final Supplier valueGeneratorWithMalformed; public ScaledFloatFieldDataGenerator(String fieldName, DataSource dataSource) { - var doubles = dataSource.get(new DataSourceRequest.DoubleGenerator()); - var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); - var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); + var doubles = dataSource.get(new DataSourceRequest.DoubleGenerator()).generator(); - this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> doubles.generator().get()); + this.valueGenerator = Wrappers.defaults(doubles::get, dataSource); + + var strings = dataSource.get(new DataSourceRequest.StringGenerator()).generator(); + this.valueGeneratorWithMalformed = Wrappers.defaultsWithMalformed(doubles::get, strings::get, dataSource); } @Override - public Object generateValue() { + public Object generateValue(Map fieldMapping) { + if (fieldMapping != null && (Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) { + return valueGeneratorWithMalformed.get(); + } + return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java index 85bff2c85e538..a7e50ffc6d0f5 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ShortFieldDataGenerator.java @@ -13,21 +13,28 @@ import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import java.util.Map; import java.util.function.Supplier; public class ShortFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; + private final Supplier valueGeneratorWithMalformed; public ShortFieldDataGenerator(String fieldName, DataSource dataSource) { - var shorts = dataSource.get(new DataSourceRequest.ShortGenerator()); - var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); - var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); + var shorts = dataSource.get(new DataSourceRequest.ShortGenerator()).generator(); - this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> shorts.generator().get()); + this.valueGenerator = Wrappers.defaults(shorts::get, dataSource); + + var strings = dataSource.get(new DataSourceRequest.StringGenerator()).generator(); + this.valueGeneratorWithMalformed = Wrappers.defaultsWithMalformed(shorts::get, strings::get, dataSource); } @Override - public Object generateValue() { + public Object generateValue(Map fieldMapping) { + if (fieldMapping != null && (Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) { + return valueGeneratorWithMalformed.get(); + } + return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java index 329f684bef70d..18dd61f0c8ff1 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/UnsignedLongFieldDataGenerator.java @@ -13,21 +13,28 @@ import org.elasticsearch.logsdb.datageneration.datasource.DataSource; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; +import java.util.Map; import java.util.function.Supplier; public class UnsignedLongFieldDataGenerator implements FieldDataGenerator { private final Supplier valueGenerator; + private final Supplier valueGeneratorWithMalformed; public UnsignedLongFieldDataGenerator(String fieldName, DataSource dataSource) { - var unsignedLongs = dataSource.get(new DataSourceRequest.UnsignedLongGenerator()); - var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); - var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); + var unsignedLongs = dataSource.get(new DataSourceRequest.UnsignedLongGenerator()).generator(); - this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> unsignedLongs.generator().get()); + this.valueGenerator = Wrappers.defaults(unsignedLongs::get, dataSource); + + var strings = dataSource.get(new DataSourceRequest.StringGenerator()).generator(); + this.valueGeneratorWithMalformed = Wrappers.defaultsWithMalformed(unsignedLongs::get, strings::get, dataSource); } @Override - public Object generateValue() { + public Object generateValue(Map fieldMapping) { + if (fieldMapping != null && (Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) { + return valueGeneratorWithMalformed.get(); + } + return valueGenerator.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/Wrappers.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/Wrappers.java new file mode 100644 index 0000000000000..74be106620d46 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/Wrappers.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.logsdb.datageneration.fields.leaf; + +import org.elasticsearch.logsdb.datageneration.datasource.DataSource; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; + +import java.util.function.Supplier; + +public class Wrappers { + /** + * Applies default wrappers for raw values - adds nulls and wraps values in arrays. + * @return + */ + static Supplier defaults(Supplier rawValues, DataSource dataSource) { + var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); + var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); + + return arrays.wrapper().compose(nulls.wrapper()).apply(rawValues::get); + } + + /** + * Applies default wrappers for raw values and also adds malformed values. + * @return + */ + static Supplier defaultsWithMalformed(Supplier rawValues, Supplier malformedValues, DataSource dataSource) { + var nulls = dataSource.get(new DataSourceRequest.NullWrapper()); + var malformed = dataSource.get(new DataSourceRequest.MalformedWrapper(malformedValues)); + var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper()); + + return arrays.wrapper().compose(nulls.wrapper()).compose(malformed.wrapper()).apply(rawValues::get); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/DynamicFieldMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/DynamicFieldMatcher.java index 5bcf53cfa5c9a..7957ed1eebe4d 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/DynamicFieldMatcher.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/DynamicFieldMatcher.java @@ -10,6 +10,7 @@ package org.elasticsearch.logsdb.datageneration.matchers.source; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.logsdb.datageneration.matchers.ListEqualMatcher; import org.elasticsearch.logsdb.datageneration.matchers.MatchResult; import org.elasticsearch.xcontent.XContentBuilder; @@ -46,9 +47,12 @@ class DynamicFieldMatcher { * @return {#{@link MatchResult}} if field values need special treatment by this matcher. * If field values can be matched using generic mapper, returns {@link Optional#empty()}. */ - public Optional match(List actual, List expected) { + public MatchResult match(List actual, List expected) { if (expected == null) { - return Optional.empty(); + expected = List.of(); + } + if (actual == null) { + actual = List.of(); } // Floating point values are always mapped as float with dynamic mapping. @@ -59,7 +63,7 @@ public Optional match(List actual, List expected) { var normalizedActual = normalizeDoubles(actual); var normalizedExpected = normalizeDoubles(expected); - var matchResult = normalizedActual.equals(normalizedExpected) + return normalizedActual.equals(normalizedExpected) ? MatchResult.match() : MatchResult.noMatch( formatErrorMessage( @@ -71,10 +75,9 @@ public Optional match(List actual, List expected) { + prettyPrintCollections(normalizedActual, normalizedExpected) ) ); - return Optional.of(matchResult); } - return Optional.empty(); + return matchWithGenericMatcher(actual, expected); } private static Set normalizeDoubles(List values) { @@ -85,4 +88,18 @@ private static Set normalizeDoubles(List values) { Function toFloat = (o) -> o instanceof Number n ? n.floatValue() : Float.parseFloat((String) o); return values.stream().filter(Objects::nonNull).map(toFloat).collect(Collectors.toSet()); } + + private MatchResult matchWithGenericMatcher(List actualValues, List expectedValues) { + var genericListMatcher = new ListEqualMatcher( + actualMappings, + actualSettings, + expectedMappings, + expectedSettings, + SourceTransforms.normalizeValues(actualValues), + SourceTransforms.normalizeValues(expectedValues), + true + ); + + return genericListMatcher.match(); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java index df26b652a806d..e2acea4ad91de 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java @@ -15,12 +15,11 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.math.BigInteger; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.Function; +import java.util.TreeMap; import java.util.stream.Collectors; import static org.elasticsearch.logsdb.datageneration.matchers.Messages.formatErrorMessage; @@ -29,13 +28,13 @@ interface FieldSpecificMatcher { MatchResult match(List actual, List expected, Map actualMapping, Map expectedMapping); - class HalfFloatMatcher implements FieldSpecificMatcher { + class CountedKeywordMatcher implements FieldSpecificMatcher { private final XContentBuilder actualMappings; private final Settings.Builder actualSettings; private final XContentBuilder expectedMappings; private final Settings.Builder expectedSettings; - HalfFloatMatcher( + CountedKeywordMatcher( XContentBuilder actualMappings, Settings.Builder actualSettings, XContentBuilder expectedMappings, @@ -54,35 +53,83 @@ public MatchResult match( Map actualMapping, Map expectedMapping ) { - var actualHalfFloatBytes = normalize(actual); - var expectedHalfFloatBytes = normalize(expected); + var actualNormalized = normalize(actual); + var expectedNormalized = normalize(expected); - return actualHalfFloatBytes.equals(expectedHalfFloatBytes) - ? MatchResult.match() - : MatchResult.noMatch( + Map counts = new TreeMap<>(); + for (String value : actualNormalized) { + counts.put(value, counts.getOrDefault(value, 0) + 1); + } + for (String value : expectedNormalized) { + int newCount = counts.getOrDefault(value, 0) - 1; + if (newCount == 0) { + counts.remove(value); + } else { + counts.put(value, newCount); + } + } + + if (counts.isEmpty() == false) { + var extraValuesMessage = new StringBuilder("extra values: "); + for (var entry : counts.entrySet()) { + extraValuesMessage.append('\n').append(entry.getKey()).append(": ").append(entry.getValue()); + } + + return MatchResult.noMatch( formatErrorMessage( actualMappings, actualSettings, expectedMappings, expectedSettings, - "Values of type [half_float] don't match after normalization, normalized " - + prettyPrintCollections(actualHalfFloatBytes, expectedHalfFloatBytes) + "Values of type [counted_keyword] don't match, " + + extraValuesMessage + + ".\n" + + prettyPrintCollections(actualNormalized, expectedNormalized) ) ); + } + + return MatchResult.match(); } - private static Set normalize(List values) { - if (values == null) { - return Set.of(); - } + private static List normalize(List values) { + return values.stream().filter(Objects::nonNull).map(it -> (String) it).toList(); + } + } - Function toFloat = (o) -> o instanceof Number n ? n.floatValue() : Float.parseFloat((String) o); - return values.stream() - .filter(Objects::nonNull) - .map(toFloat) - // Based on logic in NumberFieldMapper - .map(HalfFloatPoint::halfFloatToSortableShort) - .collect(Collectors.toSet()); + class HalfFloatMatcher extends GenericMappingAwareMatcher { + HalfFloatMatcher( + XContentBuilder actualMappings, + Settings.Builder actualSettings, + XContentBuilder expectedMappings, + Settings.Builder expectedSettings + ) { + super("half_float", actualMappings, actualSettings, expectedMappings, expectedSettings); + } + + @Override + Object convert(Object value, Object nullValue) { + var nullValueShort = nullValue != null ? HalfFloatPoint.halfFloatToSortableShort(((Number) nullValue).floatValue()) : null; + + return switch (value) { + case null -> nullValueShort; + case Number n -> HalfFloatPoint.halfFloatToSortableShort(n.floatValue()); + case String s -> { + // Special case for number coercion from strings + if (s.isEmpty()) { + yield nullValueShort; + } + + try { + var f = Float.parseFloat(s); + yield HalfFloatPoint.halfFloatToSortableShort(f); + } catch (NumberFormatException e) { + // Malformed, leave it be and match as is + yield s; + } + } + default -> value; + }; } } @@ -111,23 +158,23 @@ public MatchResult match( Map actualMapping, Map expectedMapping ) { - var scalingFactor = actualMapping.get("scaling_factor"); - var expectedScalingFactor = expectedMapping.get("scaling_factor"); - if (Objects.equals(scalingFactor, expectedScalingFactor) == false) { - throw new IllegalStateException("Scaling factor for scaled_float field does not match between actual and expected mapping"); - } + var scalingFactor = FieldSpecificMatcher.getMappingParameter("scaling_factor", actualMapping, expectedMapping); assert scalingFactor instanceof Number; double scalingFactorDouble = ((Number) scalingFactor).doubleValue(); + + var nullValue = (Number) FieldSpecificMatcher.getNullValue(actualMapping, expectedMapping); + // It is possible that we receive a mix of reduced precision values and original values. // F.e. in case of `synthetic_source_keep: "arrays"` in nested objects only arrays are preserved as is // and therefore any singleton values have reduced precision. // Therefore, we need to match either an exact value or a normalized value. - var expectedNormalized = normalizeValues(expected); - var actualNormalized = normalizeValues(actual); - for (var expectedValue : expectedNormalized) { - if (actualNormalized.contains(expectedValue) == false - && actualNormalized.contains(encodeDecodeWithPrecisionLoss(expectedValue, scalingFactorDouble)) == false) { + var expectedNumbers = numbers(expected, nullValue); + var actualNumbers = numbers(actual, nullValue); + + for (var expectedValue : expectedNumbers) { + if (actualNumbers.contains(expectedValue) == false + && actualNumbers.contains(encodeDecodeWithPrecisionLoss(expectedValue, scalingFactorDouble)) == false) { return MatchResult.noMatch( formatErrorMessage( actualMappings, @@ -135,7 +182,24 @@ public MatchResult match( expectedMappings, expectedSettings, "Values of type [scaled_float] don't match after normalization, normalized " - + prettyPrintCollections(actualNormalized, expectedNormalized) + + prettyPrintCollections(actualNumbers, expectedNumbers) + ) + ); + } + } + + var expectedNotNumbers = notNumbers(expected); + var actualNotNumbers = notNumbers(actual); + for (var expectedValue : expectedNotNumbers) { + if (actualNotNumbers.contains(expectedValue) == false) { + return MatchResult.noMatch( + formatErrorMessage( + actualMappings, + actualSettings, + expectedMappings, + expectedSettings, + "Malformed values of [scaled_float] field don't match, values:" + + prettyPrintCollections(actualNotNumbers, expectedNotNumbers) ) ); } @@ -144,18 +208,49 @@ public MatchResult match( return MatchResult.match(); } - private Double encodeDecodeWithPrecisionLoss(double value, double scalingFactor) { - // Based on logic in ScaledFloatFieldMapper - var encoded = Math.round(value * scalingFactor); - return encoded / scalingFactor; + private Set numbers(List values, Number nullValue) { + if (values == null) { + return Set.of(); + } + + return values.stream() + .map(v -> convertNumber(v, nullValue)) + .filter(Objects::nonNull) + .map(ScaledFloatMatcher::toDouble) + .collect(Collectors.toSet()); } - private static Set normalizeValues(List values) { + private static Object convertNumber(Object value, Number nullValue) { + if (value == null) { + return nullValue; + } + // Special case for number coercion from strings + if (value instanceof String s && s.isEmpty()) { + return nullValue; + } + if (value instanceof Number n) { + return n; + } + + return null; + } + + private Set notNumbers(List values) { if (values == null) { return Set.of(); } - return values.stream().filter(Objects::nonNull).map(ScaledFloatMatcher::toDouble).collect(Collectors.toSet()); + return values.stream() + .filter(Objects::nonNull) + .filter(v -> v instanceof Number == false) + .filter(v -> v instanceof String == false || ((String) v).isEmpty() == false) + .collect(Collectors.toSet()); + } + + private Double encodeDecodeWithPrecisionLoss(double value, double scalingFactor) { + // Based on logic in ScaledFloatFieldMapper + var encoded = Math.round(value * scalingFactor); + return encoded / scalingFactor; } private static double toDouble(Object value) { @@ -163,144 +258,118 @@ private static double toDouble(Object value) { } } - class UnsignedLongMatcher implements FieldSpecificMatcher { - private final XContentBuilder actualMappings; - private final Settings.Builder actualSettings; - private final XContentBuilder expectedMappings; - private final Settings.Builder expectedSettings; - + class UnsignedLongMatcher extends GenericMappingAwareMatcher { UnsignedLongMatcher( XContentBuilder actualMappings, Settings.Builder actualSettings, XContentBuilder expectedMappings, Settings.Builder expectedSettings ) { - this.actualMappings = actualMappings; - this.actualSettings = actualSettings; - this.expectedMappings = expectedMappings; - this.expectedSettings = expectedSettings; + super("unsigned_long", actualMappings, actualSettings, expectedMappings, expectedSettings); } @Override - public MatchResult match( - List actual, - List expected, - Map actualMapping, - Map expectedMapping - ) { - var expectedNormalized = normalize(expected); - var actualNormalized = normalize(actual); + Object convert(Object value, Object nullValue) { + var nullValueBigInt = nullValue != null ? BigInteger.valueOf(((Number) nullValue).longValue()) : null; + + return switch (value) { + case null -> nullValueBigInt; + case String s -> { + // Special case for number coercion from strings + if (s.isEmpty()) { + yield nullValueBigInt; + } + + yield s; + } + case Long l -> BigInteger.valueOf(l); + default -> value; + }; - return actualNormalized.equals(expectedNormalized) - ? MatchResult.match() - : MatchResult.noMatch( - formatErrorMessage( - actualMappings, - actualSettings, - expectedMappings, - expectedSettings, - "Values of type [unsigned_long] don't match after normalization, normalized " - + prettyPrintCollections(actualNormalized, expectedNormalized) - ) - ); } + } - private static Set normalize(List values) { - if (values == null) { - return Set.of(); - } - - return values.stream().filter(Objects::nonNull).map(UnsignedLongMatcher::toBigInteger).collect(Collectors.toSet()); + class KeywordMatcher extends GenericMappingAwareMatcher { + KeywordMatcher( + XContentBuilder actualMappings, + Settings.Builder actualSettings, + XContentBuilder expectedMappings, + Settings.Builder expectedSettings + ) { + super("keyword", actualMappings, actualSettings, expectedMappings, expectedSettings); } - private static BigInteger toBigInteger(Object value) { - if (value instanceof String s) { - return new BigInteger(s, 10); - } - if (value instanceof Long l) { - return BigInteger.valueOf(l); + @Override + Object convert(Object value, Object nullValue) { + if (value == null) { + return nullValue; } - return (BigInteger) value; + return value; } } - class CountedKeywordMatcher implements FieldSpecificMatcher { - private final XContentBuilder actualMappings; - private final Settings.Builder actualSettings; - private final XContentBuilder expectedMappings; - private final Settings.Builder expectedSettings; - - CountedKeywordMatcher( + class NumberMatcher extends GenericMappingAwareMatcher { + NumberMatcher( + String fieldType, XContentBuilder actualMappings, Settings.Builder actualSettings, XContentBuilder expectedMappings, Settings.Builder expectedSettings ) { - this.actualMappings = actualMappings; - this.actualSettings = actualSettings; - this.expectedMappings = expectedMappings; - this.expectedSettings = expectedSettings; - } - - private static List normalize(List values) { - return values.stream().filter(Objects::nonNull).map(it -> (String) it).toList(); + super(fieldType, actualMappings, actualSettings, expectedMappings, expectedSettings); } - private static boolean matchCountsEqualExact(List actualNormalized, List expectedNormalized) { - HashMap counts = new HashMap<>(); - for (String value : actualNormalized) { - counts.put(value, counts.getOrDefault(value, 0) + 1); + @Override + Object convert(Object value, Object nullValue) { + if (value == null) { + return nullValue; } - for (String value : expectedNormalized) { - int newCount = counts.getOrDefault(value, 0) - 1; - if (newCount == 0) { - counts.remove(value); - } else { - counts.put(value, newCount); - } + // Special case for number coercion from strings + if (value instanceof String s && s.isEmpty()) { + return nullValue; } - return counts.isEmpty(); + return value; } + } - @Override - public MatchResult match( - List actual, - List expected, - Map actualMapping, - Map expectedMapping + // TODO basic implementation only right now + class DateMatcher extends GenericMappingAwareMatcher { + DateMatcher( + XContentBuilder actualMappings, + Settings.Builder actualSettings, + XContentBuilder expectedMappings, + Settings.Builder expectedSettings ) { - var actualNormalized = normalize(actual); - var expectedNormalized = normalize(expected); + super("date", actualMappings, actualSettings, expectedMappings, expectedSettings); + } - return matchCountsEqualExact(actualNormalized, expectedNormalized) - ? MatchResult.match() - : MatchResult.noMatch( - formatErrorMessage( - actualMappings, - actualSettings, - expectedMappings, - expectedSettings, - "Values of type [counted_keyword] don't match after normalization, normalized" - + prettyPrintCollections(actualNormalized, expectedNormalized) - ) - ); + @Override + Object convert(Object value, Object nullValue) { + return value; } } - class KeywordMatcher implements FieldSpecificMatcher { + /** + * Generic matcher that supports common matching logic like null values. + */ + abstract class GenericMappingAwareMatcher implements FieldSpecificMatcher { + private final String fieldType; + private final XContentBuilder actualMappings; private final Settings.Builder actualSettings; private final XContentBuilder expectedMappings; private final Settings.Builder expectedSettings; - KeywordMatcher( + GenericMappingAwareMatcher( + String fieldType, XContentBuilder actualMappings, Settings.Builder actualSettings, XContentBuilder expectedMappings, Settings.Builder expectedSettings ) { + this.fieldType = fieldType; this.actualMappings = actualMappings; this.actualSettings = actualSettings; this.expectedMappings = expectedMappings; @@ -308,22 +377,17 @@ class KeywordMatcher implements FieldSpecificMatcher { } @Override + @SuppressWarnings("unchecked") public MatchResult match( List actual, List expected, Map actualMapping, Map expectedMapping ) { - var nullValue = actualMapping.get("null_value"); - var expectedNullValue = expectedMapping.get("null_value"); - if (Objects.equals(nullValue, expectedNullValue) == false) { - throw new IllegalStateException( - "[null_value] parameter for [keyword] field does not match between actual and expected mapping" - ); - } + var nullValue = getNullValue(actualMapping, expectedMapping); - var expectedNormalized = normalize(expected, (String) nullValue); - var actualNormalized = normalize(actual, (String) nullValue); + var expectedNormalized = normalize(expected, nullValue); + var actualNormalized = normalize(actual, nullValue); return actualNormalized.equals(expectedNormalized) ? MatchResult.match() @@ -333,18 +397,35 @@ public MatchResult match( actualSettings, expectedMappings, expectedSettings, - "Values of type [keyword] don't match after normalization, normalized " + "Values of type [" + + fieldType + + "] don't match after normalization, normalized " + prettyPrintCollections(actualNormalized, expectedNormalized) ) ); } - private static Set normalize(List values, String nullValue) { + private Set normalize(List values, Object nullValue) { if (values == null) { return Set.of(); } - return values.stream().map(v -> v == null ? nullValue : (String) v).filter(Objects::nonNull).collect(Collectors.toSet()); + return values.stream().map(v -> convert(v, nullValue)).filter(Objects::nonNull).collect(Collectors.toSet()); + } + + abstract Object convert(Object value, Object nullValue); + } + + private static Object getNullValue(Map actualMapping, Map expectedMapping) { + return getMappingParameter("null_value", actualMapping, expectedMapping); + } + + private static Object getMappingParameter(String name, Map actualMapping, Map expectedMapping) { + var actualValue = actualMapping.get(name); + var expectedValue = expectedMapping.get(name); + if (Objects.equals(actualValue, expectedValue) == false) { + throw new IllegalStateException("[" + name + "] parameter does not match between actual and expected mapping"); } + return actualValue; } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java index 57c7a92bfa55a..7390f846b017a 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java @@ -13,14 +13,13 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.logsdb.datageneration.matchers.GenericEqualsMatcher; -import org.elasticsearch.logsdb.datageneration.matchers.ListEqualMatcher; import org.elasticsearch.logsdb.datageneration.matchers.MatchResult; import org.elasticsearch.xcontent.XContentBuilder; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import static org.elasticsearch.logsdb.datageneration.matchers.Messages.formatErrorMessage; import static org.elasticsearch.logsdb.datageneration.matchers.Messages.prettyPrintCollections; @@ -51,18 +50,52 @@ public SourceMatcher( .v2(); this.expectedNormalizedMapping = MappingTransforms.normalizeMapping(expectedMappingAsMap); - this.fieldSpecificMatchers = Map.of( - "half_float", - new FieldSpecificMatcher.HalfFloatMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings), - "scaled_float", - new FieldSpecificMatcher.ScaledFloatMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings), - "unsigned_long", - new FieldSpecificMatcher.UnsignedLongMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings), - "counted_keyword", - new FieldSpecificMatcher.CountedKeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings), - "keyword", - new FieldSpecificMatcher.KeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings) - ); + this.fieldSpecificMatchers = new HashMap<>() { + { + put("keyword", new FieldSpecificMatcher.KeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); + put("date", new FieldSpecificMatcher.DateMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); + put( + "long", + new FieldSpecificMatcher.NumberMatcher("long", actualMappings, actualSettings, expectedMappings, expectedSettings) + ); + put( + "unsigned_long", + new FieldSpecificMatcher.UnsignedLongMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings) + ); + put( + "integer", + new FieldSpecificMatcher.NumberMatcher("integer", actualMappings, actualSettings, expectedMappings, expectedSettings) + ); + put( + "short", + new FieldSpecificMatcher.NumberMatcher("short", actualMappings, actualSettings, expectedMappings, expectedSettings) + ); + put( + "byte", + new FieldSpecificMatcher.NumberMatcher("byte", actualMappings, actualSettings, expectedMappings, expectedSettings) + ); + put( + "double", + new FieldSpecificMatcher.NumberMatcher("double", actualMappings, actualSettings, expectedMappings, expectedSettings) + ); + put( + "float", + new FieldSpecificMatcher.NumberMatcher("float", actualMappings, actualSettings, expectedMappings, expectedSettings) + ); + put( + "half_float", + new FieldSpecificMatcher.HalfFloatMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings) + ); + put( + "scaled_float", + new FieldSpecificMatcher.ScaledFloatMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings) + ); + put( + "counted_keyword", + new FieldSpecificMatcher.CountedKeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings) + ); + } + }; this.dynamicFieldMatcher = new DynamicFieldMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings); } @@ -104,9 +137,7 @@ private MatchResult compareSource(Map> actual, Map matchWithGenericMatcher(actualValues, expectedValues) - ); + var matchIncludingFieldSpecificMatchers = matchWithFieldSpecificMatcher(name, actualValues, expectedValues); if (matchIncludingFieldSpecificMatchers.isMatch() == false) { var message = "Source documents don't match for field [" + name + "]: " + matchIncludingFieldSpecificMatchers.getMessage(); return MatchResult.noMatch(message); @@ -115,7 +146,7 @@ private MatchResult compareSource(Map> actual, Map matchWithFieldSpecificMatcher(String fieldName, List actualValues, List expectedValues) { + private MatchResult matchWithFieldSpecificMatcher(String fieldName, List actualValues, List expectedValues) { var actualFieldMapping = actualNormalizedMapping.get(fieldName); if (actualFieldMapping == null) { if (expectedNormalizedMapping.get(fieldName) != null @@ -155,30 +186,13 @@ private Optional matchWithFieldSpecificMatcher(String fieldName, Li } var fieldSpecificMatcher = fieldSpecificMatchers.get(actualFieldType); - if (fieldSpecificMatcher == null) { - return Optional.empty(); - } + assert fieldSpecificMatcher != null : "Missing matcher for field type [" + actualFieldType + "]"; - MatchResult matched = fieldSpecificMatcher.match( + return fieldSpecificMatcher.match( actualValues, expectedValues, actualFieldMapping.mappingParameters(), expectedFieldMapping.mappingParameters() ); - return Optional.of(matched); - } - - private MatchResult matchWithGenericMatcher(List actualValues, List expectedValues) { - var genericListMatcher = new ListEqualMatcher( - actualMappings, - actualSettings, - expectedMappings, - expectedSettings, - SourceTransforms.normalizeValues(actualValues), - SourceTransforms.normalizeValues(expectedValues), - true - ); - - return genericListMatcher.match(); } } diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/SourceMatcherTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/SourceMatcherTests.java index bc7a558a0b687..aba5cbc9878e1 100644 --- a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/SourceMatcherTests.java +++ b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/SourceMatcherTests.java @@ -62,8 +62,8 @@ public void testDynamicMismatch() throws IOException { public void testMappedMatch() throws IOException { List> values = List.of( - Map.of("aaa", 124, "bbb", false, "ccc", 12.34), - Map.of("aaa", 124, "bbb", false, "ccc", 12.34) + Map.of("aaa", 124, "bbb", "hey", "ccc", 12.34), + Map.of("aaa", 124, "bbb", "yeh", "ccc", 12.34) ); var mapping = XContentBuilder.builder(XContentType.JSON.xContent()); @@ -71,7 +71,7 @@ public void testMappedMatch() throws IOException { mapping.startObject("_doc"); { mapping.startObject("aaa").field("type", "long").endObject(); - mapping.startObject("bbb").field("type", "boolean").endObject(); + mapping.startObject("bbb").field("type", "keyword").endObject(); mapping.startObject("ccc").field("type", "half_float").endObject(); } mapping.endObject(); @@ -83,12 +83,12 @@ public void testMappedMatch() throws IOException { public void testMappedMismatch() throws IOException { List> actual = List.of( - Map.of("aaa", 124, "bbb", false, "ccc", 12.34), - Map.of("aaa", 124, "bbb", false, "ccc", 12.34) + Map.of("aaa", 124, "bbb", "hey", "ccc", 12.34), + Map.of("aaa", 124, "bbb", "yeh", "ccc", 12.34) ); List> expected = List.of( - Map.of("aaa", 124, "bbb", false, "ccc", 12.34), - Map.of("aaa", 124, "bbb", false, "ccc", 12.35) + Map.of("aaa", 124, "bbb", "hey", "ccc", 12.34), + Map.of("aaa", 124, "bbb", "yeh", "ccc", 12.35) ); var mapping = XContentBuilder.builder(XContentType.JSON.xContent()); @@ -96,7 +96,7 @@ public void testMappedMismatch() throws IOException { mapping.startObject("_doc"); { mapping.startObject("aaa").field("type", "long").endObject(); - mapping.startObject("bbb").field("type", "boolean").endObject(); + mapping.startObject("bbb").field("type", "keyword").endObject(); mapping.startObject("ccc").field("type", "half_float").endObject(); } mapping.endObject(); diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java index 8a5bb8d12cd3d..fb890f3ac7ae7 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java @@ -49,14 +49,14 @@ public DataGenerationHelper(Consumer builder "host.name", FieldType.KEYWORD, Map.of("type", "keyword"), - () -> ESTestCase.randomAlphaOfLength(5) + (ignored) -> ESTestCase.randomAlphaOfLength(5) ), // Needed for terms query new PredefinedField.WithGenerator( "method", FieldType.KEYWORD, Map.of("type", "keyword"), - () -> ESTestCase.randomFrom("put", "post", "get") + (ignored) -> ESTestCase.randomFrom("put", "post", "get") ), // Needed for histogram aggregation @@ -64,7 +64,7 @@ public DataGenerationHelper(Consumer builder "memory_usage_bytes", FieldType.LONG, Map.of("type", "long"), - () -> ESTestCase.randomLongBetween(1000, 2000) + (ignored) -> ESTestCase.randomLongBetween(1000, 2000) ) ) );