[TEST] Restore scaled_float and half_float data generation (#120756)

lkts · web-flow · commit 3d4349851bdc · 2025-01-24T13:45:42.000-08:00
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java
@@ -13,9 +13,11 @@
 import org.elasticsearch.logsdb.datageneration.fields.leaf.ByteFieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.fields.leaf.DoubleFieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.fields.leaf.FloatFieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.fields.leaf.HalfFloatFieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.fields.leaf.IntegerFieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.fields.leaf.KeywordFieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.fields.leaf.LongFieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.fields.leaf.ScaledFloatFieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.fields.leaf.ShortFieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.fields.leaf.UnsignedLongFieldDataGenerator;
 
@@ -30,7 +32,9 @@ public enum FieldType {
     SHORT("short"),
     BYTE("byte"),
     DOUBLE("double"),
-    FLOAT("float");
+    FLOAT("float"),
+    HALF_FLOAT("half_float"),
+    SCALED_FLOAT("scaled_float");
 
     private final String name;
 
@@ -48,6 +52,8 @@ public FieldDataGenerator generator(String fieldName, DataSource dataSource) {
             case BYTE -> new ByteFieldDataGenerator(fieldName, dataSource);
             case DOUBLE -> new DoubleFieldDataGenerator(fieldName, dataSource);
             case FLOAT -> new FloatFieldDataGenerator(fieldName, dataSource);
+            case HALF_FLOAT -> new HalfFloatFieldDataGenerator(fieldName, dataSource);
+            case SCALED_FLOAT -> new ScaledFloatFieldDataGenerator(fieldName, dataSource);
         };
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java
@@ -32,7 +32,8 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques
 
         return new DataSourceResponse.LeafMappingParametersGenerator(switch (request.fieldType()) {
             case KEYWORD -> keywordMapping(request, map);
-            case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, UNSIGNED_LONG -> plain(map);
+            case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, HALF_FLOAT, UNSIGNED_LONG -> plain(map);
+            case SCALED_FLOAT -> scaledFloatMapping(map);
         });
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/ArrayEqualMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/ArrayEqualMatcher.java
@@ -1,20 +1,22 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.xpack.logsdb.qa.matchers;
+package org.elasticsearch.logsdb.datageneration.matchers;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.util.Arrays;
 import java.util.List;
 
-import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.formatErrorMessage;
-import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.prettyPrintArrays;
+import static org.elasticsearch.logsdb.datageneration.matchers.Messages.formatErrorMessage;
+import static org.elasticsearch.logsdb.datageneration.matchers.Messages.prettyPrintArrays;
 
 class ArrayEqualMatcher extends GenericEqualsMatcher<Object[]> {
     ArrayEqualMatcher(
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/GenericEqualsMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/GenericEqualsMatcher.java
@@ -1,18 +1,20 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.xpack.logsdb.qa.matchers;
+package org.elasticsearch.logsdb.datageneration.matchers;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.util.List;
 
-import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.formatErrorMessage;
+import static org.elasticsearch.logsdb.datageneration.matchers.Messages.formatErrorMessage;
 
 public class GenericEqualsMatcher<T> extends Matcher {
     protected final XContentBuilder actualMappings;
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/ListEqualMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/ListEqualMatcher.java
@@ -1,19 +1,21 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.xpack.logsdb.qa.matchers;
+package org.elasticsearch.logsdb.datageneration.matchers;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.util.List;
 
-import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.formatErrorMessage;
-import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.prettyPrintCollections;
+import static org.elasticsearch.logsdb.datageneration.matchers.Messages.formatErrorMessage;
+import static org.elasticsearch.logsdb.datageneration.matchers.Messages.prettyPrintCollections;
 
 public class ListEqualMatcher extends GenericEqualsMatcher<List<?>> {
     public ListEqualMatcher(
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/MatchResult.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/MatchResult.java
@@ -1,11 +1,13 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.xpack.logsdb.qa.matchers;
+package org.elasticsearch.logsdb.datageneration.matchers;
 
 import java.util.Objects;
 
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/Matcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/Matcher.java
@@ -1,15 +1,17 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.xpack.logsdb.qa.matchers;
+package org.elasticsearch.logsdb.datageneration.matchers;
 
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.logsdb.datageneration.matchers.source.SourceMatcher;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.logsdb.qa.matchers.source.SourceMatcher;
 
 import java.util.List;
 import java.util.Map;
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/Messages.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/Messages.java
@@ -1,11 +1,13 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.xpack.logsdb.qa.matchers;
+package org.elasticsearch.logsdb.datageneration.matchers;
 
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/ObjectMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/ObjectMatcher.java
@@ -1,16 +1,18 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.xpack.logsdb.qa.matchers;
+package org.elasticsearch.logsdb.datageneration.matchers;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xcontent.XContentBuilder;
 
-import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.formatErrorMessage;
+import static org.elasticsearch.logsdb.datageneration.matchers.Messages.formatErrorMessage;
 
 public class ObjectMatcher extends GenericEqualsMatcher<Object> {
     ObjectMatcher(
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/DynamicFieldMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/DynamicFieldMatcher.java
@@ -1,15 +1,17 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.xpack.logsdb.qa.matchers.source;
+package org.elasticsearch.logsdb.datageneration.matchers.source;
 
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.logsdb.datageneration.matchers.MatchResult;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.logsdb.qa.matchers.MatchResult;
 
 import java.util.List;
 import java.util.Objects;
@@ -18,8 +20,8 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.formatErrorMessage;
-import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.prettyPrintCollections;
+import static org.elasticsearch.logsdb.datageneration.matchers.Messages.formatErrorMessage;
+import static org.elasticsearch.logsdb.datageneration.matchers.Messages.prettyPrintCollections;
 
 class DynamicFieldMatcher {
     private final XContentBuilder actualMappings;
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java
@@ -1,16 +1,18 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.xpack.logsdb.qa.matchers.source;
+package org.elasticsearch.logsdb.datageneration.matchers.source;
 
 import org.apache.lucene.sandbox.document.HalfFloatPoint;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.logsdb.datageneration.matchers.MatchResult;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.logsdb.qa.matchers.MatchResult;
 
 import java.math.BigInteger;
 import java.util.List;
@@ -20,8 +22,8 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.formatErrorMessage;
-import static org.elasticsearch.xpack.logsdb.qa.matchers.Messages.prettyPrintCollections;
+import static org.elasticsearch.logsdb.datageneration.matchers.Messages.formatErrorMessage;
+import static org.elasticsearch.logsdb.datageneration.matchers.Messages.prettyPrintCollections;
 
 interface FieldSpecificMatcher {
     MatchResult match(List<Object> actual, List<Object> expected, Map<String, Object> actualMapping, Map<String, Object> expectedMapping);
@@ -115,40 +117,39 @@ public MatchResult match(
             }
 
             assert scalingFactor instanceof Number;
-            var expectedNormalized = normalizeExpected(expected, ((Number) scalingFactor).doubleValue());
-            var actualNormalized = normalizeActual(actual);
+            double scalingFactorDouble = ((Number) scalingFactor).doubleValue();
+            // It is possible that we receive a mix of reduced precision values and original values.
+            // F.e. in case of `synthetic_source_keep: "arrays"` in nested objects only arrays are preserved as is
+            // and therefore any singleton values have reduced precision.
+            // Therefore, we need to match either an exact value or a normalized value.
+            var expectedNormalized = normalizeValues(expected);
+            var actualNormalized = normalizeValues(actual);
+            for (var expectedValue : expectedNormalized) {
+                if (actualNormalized.contains(expectedValue) == false
+                    && actualNormalized.contains(encodeDecodeWithPrecisionLoss(expectedValue, scalingFactorDouble)) == false) {
+                    return MatchResult.noMatch(
+                        formatErrorMessage(
+                            actualMappings,
+                            actualSettings,
+                            expectedMappings,
+                            expectedSettings,
+                            "Values of type [scaled_float] don't match after normalization, normalized "
+                                + prettyPrintCollections(actualNormalized, expectedNormalized)
+                        )
+                    );
+                }
+            }
 
-            return actualNormalized.equals(expectedNormalized)
-                ? MatchResult.match()
-                : MatchResult.noMatch(
-                    formatErrorMessage(
-                        actualMappings,
-                        actualSettings,
-                        expectedMappings,
-                        expectedSettings,
-                        "Values of type [scaled_float] don't match after normalization, normalized "
-                            + prettyPrintCollections(actualNormalized, expectedNormalized)
-                    )
-                );
+            return MatchResult.match();
         }
 
-        private static Set<Double> normalizeExpected(List<Object> values, double scalingFactor) {
-            if (values == null) {
-                return Set.of();
-            }
-
-            return values.stream()
-                .filter(Objects::nonNull)
-                .map(ScaledFloatMatcher::toDouble)
-                // Based on logic in ScaledFloatFieldMapper
-                .map(v -> {
-                    var encoded = Math.round(v * scalingFactor);
-                    return encoded / scalingFactor;
-                })
-                .collect(Collectors.toSet());
+        private Double encodeDecodeWithPrecisionLoss(double value, double scalingFactor) {
+            // Based on logic in ScaledFloatFieldMapper
+            var encoded = Math.round(value * scalingFactor);
+            return encoded / scalingFactor;
         }
 
-        private static Set<Double> normalizeActual(List<Object> values) {
+        private static Set<Double> normalizeValues(List<Object> values) {
             if (values == null) {
                 return Set.of();
             }
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/MappingTransforms.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/MappingTransforms.java
@@ -1,11 +1,13 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.xpack.logsdb.qa.matchers.source;
+package org.elasticsearch.logsdb.datageneration.matchers.source;
 
 import java.util.ArrayList;
 import java.util.HashMap;
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java
diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceTransforms.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceTransforms.java
diff --git a/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/SourceMatcherTests.java b/test/framework/src/test/java/org/elasticsearch/logsdb/datageneration/SourceMatcherTests.java
diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java

Original file line number	Diff line number	Diff line change
`@@ -32,7 +32,8 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques`
`32`	`32`
`33`	`33`	`return new DataSourceResponse.LeafMappingParametersGenerator(switch (request.fieldType()) {`
`34`	`34`	`case KEYWORD -> keywordMapping(request, map);`
`35`		`- case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, UNSIGNED_LONG -> plain(map);`
	`35`	`+ case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, HALF_FLOAT, UNSIGNED_LONG -> plain(map);`
	`36`	`+ case SCALED_FLOAT -> scaledFloatMapping(map);`
`36`	`37`	`});`
`37`	`38`	`}`
`38`	`39`