Add ignore_malformed and null_values to test data generation

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/DocumentGenerator.java

@@ -40,7 +40,10 @@ public DocumentGenerator(DataGeneratorSpecification specification) {
     public Map<String, Object> generate(Template template, Mapping mapping) {
         var documentMap = new TreeMap<String, Object>();
         for (var predefinedField : specification.predefinedFields()) {
-            documentMap.put(predefinedField.name(), predefinedField.generator(specification.dataSource()).generateValue());
+            documentMap.put(
+                predefinedField.name(),
+                predefinedField.generator(specification.dataSource()).generateValue(predefinedField.mapping())
+            );
         }
 
         generateFields(documentMap, template.template(), new Context("", mapping.lookup()));
@@ -53,16 +56,18 @@ private void generateFields(Map<String, Object> document, Map<String, Template.E
             Template.Entry templateEntry = entry.getValue();
 
             if (templateEntry instanceof Template.Leaf leaf) {
+                var fieldMapping = context.mappingLookup().get(context.pathTo(fieldName));
+
                 // Unsigned long does not play well when dynamically mapped because
                 // it gets mapped as just long and large values fail to index.
                 // Just skip it.
-                if (leaf.type() == FieldType.UNSIGNED_LONG && context.mappingLookup().get(context.pathTo(fieldName)) == null) {
+                if (leaf.type() == FieldType.UNSIGNED_LONG && fieldMapping == null) {
                     continue;
                 }
 
                 var generator = leaf.type().generator(fieldName, specification.dataSource());
 
-                document.put(fieldName, generator.generateValue());
+                document.put(fieldName, generator.generateValue(fieldMapping));
             } else if (templateEntry instanceof Template.Object object) {
                 Optional<Integer> arrayLength = objectArrayGenerator.lengthGenerator().get();
 

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldDataGenerator.java

@@ -9,11 +9,13 @@
 
 package org.elasticsearch.logsdb.datageneration;
 
+import java.util.Map;
+
 /**
  * Entity responsible for generating a valid value for a field.
  *
  * Generator is expected to produce a different value on every call.
  */
 public interface FieldDataGenerator {
-    Object generateValue();
+    Object generateValue(Map<String, Object> fieldMapping);
 }

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java

@@ -58,6 +58,10 @@ default DataSourceResponse.RepeatingWrapper handle(DataSourceRequest.RepeatingWr
         return null;
     }
 
+    default DataSourceResponse.MalformedWrapper handle(DataSourceRequest.MalformedWrapper request) {
+        return null;
+    }
+
     default DataSourceResponse.ChildFieldGenerator handle(DataSourceRequest.ChildFieldGenerator request) {
         return null;
     }

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping;
 
 import java.util.Set;
+import java.util.function.Supplier;
 
 public interface DataSourceRequest<TResponse extends DataSourceResponse> {
     TResponse accept(DataSourceHandler handler);
@@ -91,6 +92,12 @@ public DataSourceResponse.RepeatingWrapper accept(DataSourceHandler handler) {
         }
     }
 
+    record MalformedWrapper(Supplier<Object> malformedValues) implements DataSourceRequest<DataSourceResponse.MalformedWrapper> {
+        public DataSourceResponse.MalformedWrapper accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
+
     record ChildFieldGenerator(DataGeneratorSpecification specification)
         implements
             DataSourceRequest<DataSourceResponse.ChildFieldGenerator> {

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java

@@ -41,6 +41,8 @@ record ArrayWrapper(Function<Supplier<Object>, Supplier<Object>> wrapper) implem
 
     record RepeatingWrapper(Function<Supplier<Object>, Supplier<Object>> wrapper) implements DataSourceResponse {}
 
+    record MalformedWrapper(Function<Supplier<Object>, Supplier<Object>> wrapper) implements DataSourceResponse {}
+
     interface ChildFieldGenerator extends DataSourceResponse {
         int generateChildFieldCount();
 

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java

@@ -11,6 +11,7 @@
 
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.ObjectMapper;
+import org.elasticsearch.logsdb.datageneration.FieldType;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
@@ -31,7 +32,7 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques
 
         return new DataSourceResponse.LeafMappingParametersGenerator(switch (request.fieldType()) {
             case KEYWORD -> keywordMapping(request, map);
-            case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, HALF_FLOAT, UNSIGNED_LONG -> plain(map);
+            case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, HALF_FLOAT, UNSIGNED_LONG -> numberMapping(map, request.fieldType());
             case SCALED_FLOAT -> scaledFloatMapping(map);
             case COUNTED_KEYWORD -> plain(Map.of("index", ESTestCase.randomBoolean()));
         });
@@ -41,6 +42,30 @@ private Supplier<Map<String, Object>> plain(Map<String, Object> injected) {
         return () -> injected;
     }
 
+    private Supplier<Map<String, Object>> numberMapping(Map<String, Object> injected, FieldType fieldType) {
+        return () -> {
+            if (ESTestCase.randomBoolean()) {
+                injected.put("ignore_malformed", ESTestCase.randomBoolean());
+            }
+            if (ESTestCase.randomDouble() <= 0.2) {
+                Number value = switch (fieldType) {
+                    case LONG -> ESTestCase.randomLong();
+                    case UNSIGNED_LONG -> ESTestCase.randomNonNegativeLong();
+                    case INTEGER -> ESTestCase.randomInt();
+                    case SHORT -> ESTestCase.randomShort();
+                    case BYTE -> ESTestCase.randomByte();
+                    case DOUBLE -> ESTestCase.randomDouble();
+                    case FLOAT, HALF_FLOAT -> ESTestCase.randomFloat();
+                    default -> throw new IllegalStateException("Unexpected field type");
+                };
+
+                injected.put("null_value", value);
+            }
+
+            return injected;
+        };
+    }
+
     private Supplier<Map<String, Object>> keywordMapping(
         DataSourceRequest.LeafMappingParametersGenerator request,
         Map<String, Object> injected
@@ -75,6 +100,15 @@ private Supplier<Map<String, Object>> keywordMapping(
     private Supplier<Map<String, Object>> scaledFloatMapping(Map<String, Object> injected) {
         return () -> {
             injected.put("scaling_factor", ESTestCase.randomFrom(10, 1000, 100000, 100.5));
+
+            if (ESTestCase.randomDouble() <= 0.2) {
+                injected.put("null_value", ESTestCase.randomFloat());
+            }
+
+            if (ESTestCase.randomBoolean()) {
+                injected.put("ignore_malformed", ESTestCase.randomBoolean());
+            }
+
             return injected;
         };
     }

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultWrappersHandler.java

@@ -32,6 +32,11 @@ public DataSourceResponse.RepeatingWrapper handle(DataSourceRequest.RepeatingWra
         return new DataSourceResponse.RepeatingWrapper(repeatValues());
     }
 
+    @Override
+    public DataSourceResponse.MalformedWrapper handle(DataSourceRequest.MalformedWrapper request) {
+        return new DataSourceResponse.MalformedWrapper(injectMalformed(request.malformedValues()));
+    }
+
     private static Function<Supplier<Object>, Supplier<Object>> injectNulls() {
         // Inject some nulls but majority of data should be non-null (as it likely is in reality).
         return (values) -> () -> ESTestCase.randomDouble() <= 0.05 ? null : values.get();
@@ -62,4 +67,8 @@ private static Function<Supplier<Object>, Supplier<Object>> repeatValues() {
             };
         };
     }
+
+    private static Function<Supplier<Object>, Supplier<Object>> injectMalformed(Supplier<Object> malformedValues) {
+        return (values) -> () -> ESTestCase.randomDouble() <= 0.1 ? malformedValues.get() : values.get();
+    }
 }

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/ByteFieldDataGenerator.java

@@ -13,21 +13,28 @@
 import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
 
+import java.util.Map;
 import java.util.function.Supplier;
 
 public class ByteFieldDataGenerator implements FieldDataGenerator {
     private final Supplier<Object> valueGenerator;
+    private final Supplier<Object> valueGeneratorWithMalformed;
 
     public ByteFieldDataGenerator(String fieldName, DataSource dataSource) {
-        var bytes = dataSource.get(new DataSourceRequest.ByteGenerator());
-        var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
-        var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
+        var bytes = dataSource.get(new DataSourceRequest.ByteGenerator()).generator();
 
-        this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> bytes.generator().get());
+        this.valueGenerator = Wrappers.defaults(bytes::get, dataSource);
+
+        var strings = dataSource.get(new DataSourceRequest.StringGenerator()).generator();
+        this.valueGeneratorWithMalformed = Wrappers.defaultsWithMalformed(bytes::get, strings::get, dataSource);
     }
 
     @Override
-    public Object generateValue() {
+    public Object generateValue(Map<String, Object> fieldMapping) {
+        if (fieldMapping != null && (Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) {
+            return valueGeneratorWithMalformed.get();
+        }
+
         return valueGenerator.get();
     }
 }

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/CountedKeywordFieldDataGenerator.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
 
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 import java.util.function.Supplier;
 
@@ -31,7 +32,7 @@ public CountedKeywordFieldDataGenerator(String fieldName, DataSource dataSource)
     }
 
     @Override
-    public Object generateValue() {
+    public Object generateValue(Map<String, Object> fieldMapping) {
         return valueGenerator.get();
     }
 }

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DoubleFieldDataGenerator.java

@@ -13,21 +13,28 @@
 import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
 
+import java.util.Map;
 import java.util.function.Supplier;
 
 public class DoubleFieldDataGenerator implements FieldDataGenerator {
     private final Supplier<Object> valueGenerator;
+    private final Supplier<Object> valueGeneratorWithMalformed;
 
     public DoubleFieldDataGenerator(String fieldName, DataSource dataSource) {
-        var doubles = dataSource.get(new DataSourceRequest.DoubleGenerator());
-        var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
-        var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
+        var doubles = dataSource.get(new DataSourceRequest.DoubleGenerator()).generator();
 
-        this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> doubles.generator().get());
+        this.valueGenerator = Wrappers.defaults(doubles::get, dataSource);
+
+        var strings = dataSource.get(new DataSourceRequest.StringGenerator()).generator();
+        this.valueGeneratorWithMalformed = Wrappers.defaultsWithMalformed(doubles::get, strings::get, dataSource);
     }
 
     @Override
-    public Object generateValue() {
+    public Object generateValue(Map<String, Object> fieldMapping) {
+        if (fieldMapping != null && (Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) {
+            return valueGeneratorWithMalformed.get();
+        }
+
         return valueGenerator.get();
     }
 }

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/FloatFieldDataGenerator.java

@@ -13,21 +13,28 @@
 import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
 
+import java.util.Map;
 import java.util.function.Supplier;
 
 public class FloatFieldDataGenerator implements FieldDataGenerator {
     private final Supplier<Object> valueGenerator;
+    private final Supplier<Object> valueGeneratorWithMalformed;
 
     public FloatFieldDataGenerator(String fieldName, DataSource dataSource) {
-        var floats = dataSource.get(new DataSourceRequest.FloatGenerator());
-        var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
-        var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
+        var floats = dataSource.get(new DataSourceRequest.FloatGenerator()).generator();
 
-        this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> floats.generator().get());
+        this.valueGenerator = Wrappers.defaults(floats::get, dataSource);
+
+        var strings = dataSource.get(new DataSourceRequest.StringGenerator()).generator();
+        this.valueGeneratorWithMalformed = Wrappers.defaultsWithMalformed(floats::get, strings::get, dataSource);
     }
 
     @Override
-    public Object generateValue() {
+    public Object generateValue(Map<String, Object> fieldMapping) {
+        if (fieldMapping != null && (Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) {
+            return valueGeneratorWithMalformed.get();
+        }
+
         return valueGenerator.get();
     }
 }

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/HalfFloatFieldDataGenerator.java

@@ -13,21 +13,28 @@
 import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
 
+import java.util.Map;
 import java.util.function.Supplier;
 
 public class HalfFloatFieldDataGenerator implements FieldDataGenerator {
     private final Supplier<Object> valueGenerator;
+    private final Supplier<Object> valueGeneratorWithMalformed;
 
     public HalfFloatFieldDataGenerator(String fieldName, DataSource dataSource) {
-        var halfFloats = dataSource.get(new DataSourceRequest.HalfFloatGenerator());
-        var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
-        var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
+        var halfFloats = dataSource.get(new DataSourceRequest.HalfFloatGenerator()).generator();
 
-        this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> halfFloats.generator().get());
+        this.valueGenerator = Wrappers.defaults(halfFloats::get, dataSource);
+
+        var strings = dataSource.get(new DataSourceRequest.StringGenerator()).generator();
+        this.valueGeneratorWithMalformed = Wrappers.defaultsWithMalformed(halfFloats::get, strings::get, dataSource);
     }
 
     @Override
-    public Object generateValue() {
+    public Object generateValue(Map<String, Object> fieldMapping) {
+        if (fieldMapping != null && (Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) {
+            return valueGeneratorWithMalformed.get();
+        }
+
         return valueGenerator.get();
     }
 }

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/IntegerFieldDataGenerator.java

@@ -13,21 +13,28 @@
 import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
 
+import java.util.Map;
 import java.util.function.Supplier;
 
 public class IntegerFieldDataGenerator implements FieldDataGenerator {
     private final Supplier<Object> valueGenerator;
+    private final Supplier<Object> valueGeneratorWithMalformed;
 
     public IntegerFieldDataGenerator(String fieldName, DataSource dataSource) {
-        var ints = dataSource.get(new DataSourceRequest.IntegerGenerator());
-        var nulls = dataSource.get(new DataSourceRequest.NullWrapper());
-        var arrays = dataSource.get(new DataSourceRequest.ArrayWrapper());
+        var ints = dataSource.get(new DataSourceRequest.IntegerGenerator()).generator();
 
-        this.valueGenerator = arrays.wrapper().compose(nulls.wrapper()).apply(() -> ints.generator().get());
+        this.valueGenerator = Wrappers.defaults(ints::get, dataSource);
+
+        var strings = dataSource.get(new DataSourceRequest.StringGenerator()).generator();
+        this.valueGeneratorWithMalformed = Wrappers.defaultsWithMalformed(ints::get, strings::get, dataSource);
     }
 
     @Override
-    public Object generateValue() {
+    public Object generateValue(Map<String, Object> fieldMapping) {
+        if (fieldMapping != null && (Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) {
+            return valueGeneratorWithMalformed.get();
+        }
+
         return valueGenerator.get();
     }
 }

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/KeywordFieldDataGenerator.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.logsdb.datageneration.datasource.DataSource;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
 
+import java.util.Map;
 import java.util.function.Supplier;
 
 public class KeywordFieldDataGenerator implements FieldDataGenerator {
@@ -27,7 +28,7 @@ public KeywordFieldDataGenerator(String fieldName, DataSource dataSource) {
     }
 
     @Override
-    public Object generateValue() {
+    public Object generateValue(Map<String, Object> fieldMapping) {
         return valueGenerator.get();
     }
 }