Reapply ES|QL bfloat16 support tests (elastic#138499) (elastic#138584)

Add bfloat16 tests with a feature flag this time

Author: thecoop

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.codec.vectors.BFloat16;
+import org.elasticsearch.index.codec.vectors.es93.ES93GenericFlatVectorsFormat;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType;
 import org.elasticsearch.script.field.vectors.DenseVector;
@@ -61,16 +62,20 @@ private enum VectorSourceOptions {
         .collect(Collectors.toSet());
 
     public static final float DELTA = 1e-7F;
+    public static final float BFLOAT16_DELTA = 1e-2F;
 
     private final ElementType elementType;
     private final DenseVectorFieldMapper.VectorSimilarity similarity;
     private final VectorSourceOptions sourceOptions;
     private final boolean index;
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws Exception {
+    public static Iterable<Object[]> parameters() {
         List<Object[]> params = new ArrayList<>();
-        for (ElementType elementType : List.of(ElementType.BYTE, ElementType.FLOAT, ElementType.BIT)) {
+        ElementType[] elementTypes = ES93GenericFlatVectorsFormat.GENERIC_VECTOR_FORMAT.isEnabled()
+            ? ElementType.values()
+            : new ElementType[] { ElementType.BYTE, ElementType.FLOAT, ElementType.BIT };
+        for (ElementType elementType : elementTypes) {
             // Test all similarities
             for (DenseVectorFieldMapper.VectorSimilarity similarity : DenseVectorFieldMapper.VectorSimilarity.values()) {
                 if (elementType == ElementType.BIT && similarity != DenseVectorFieldMapper.VectorSimilarity.L2_NORM) {
@@ -137,8 +142,10 @@ public void testRetrieveTopNDenseVectorFieldData() {
                 } else {
                     assertNotNull(actualVector);
                     assertEquals(expectedVector.size(), actualVector.size());
+
+                    float delta = elementType == ElementType.BFLOAT16 ? BFLOAT16_DELTA : DELTA;
                     for (int i = 0; i < expectedVector.size(); i++) {
-                        assertEquals(expectedVector.get(i).floatValue(), actualVector.get(i).floatValue(), DELTA);
+                        assertEquals(expectedVector.get(i).floatValue(), actualVector.get(i).floatValue(), delta);
                     }
                 }
             });
@@ -167,12 +174,14 @@ public void testRetrieveDenseVectorFieldData() {
                 } else {
                     assertNotNull(actualVector);
                     assertEquals(expectedVector.size(), actualVector.size());
+
+                    float delta = elementType == ElementType.BFLOAT16 ? BFLOAT16_DELTA : DELTA;
                     for (int i = 0; i < actualVector.size(); i++) {
                         assertEquals(
                             "Actual: " + actualVector + "; expected: " + expectedVector,
                             expectedVector.get(i).floatValue(),
                             actualVector.get(i).floatValue(),
-                            DELTA
+                            delta
                         );
                     }
                 }
@@ -253,12 +262,13 @@ public void setup() throws IOException {
             } else {
                 for (int j = 0; j < numDims; j++) {
                     switch (elementType) {
-                        case FLOAT -> vector.add(randomFloatBetween(0F, 1F, true));
+                        case FLOAT, BFLOAT16 -> vector.add(randomFloatBetween(0F, 1F, true));
                         case BYTE, BIT -> vector.add((byte) randomIntBetween(-128, 127));
                         default -> throw new IllegalArgumentException("Unexpected element type: " + elementType);
                     }
                 }
-                if ((elementType == ElementType.FLOAT) && (similarity == DenseVectorFieldMapper.VectorSimilarity.DOT_PRODUCT || rarely())) {
+                if ((elementType == ElementType.FLOAT || elementType == ElementType.BFLOAT16)
+                    && (similarity == DenseVectorFieldMapper.VectorSimilarity.DOT_PRODUCT || rarely())) {
                     // Normalize the vector
                     float magnitude = DenseVector.getMagnitude(vector);
                     vector.replaceAll(number -> number.floatValue() / magnitude);

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.codec.vectors.es93.ES93GenericFlatVectorsFormat;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
@@ -57,6 +58,9 @@ public static Iterable<Object[]> parameters() throws Exception {
         List<Object[]> params = new ArrayList<>();
         for (String indexType : ALL_DENSE_VECTOR_INDEX_TYPES) {
             params.add(new Object[] { DenseVectorFieldMapper.ElementType.FLOAT, indexType });
+            if (ES93GenericFlatVectorsFormat.GENERIC_VECTOR_FORMAT.isEnabled()) {
+                params.add(new Object[] { DenseVectorFieldMapper.ElementType.BFLOAT16, indexType });
+            }
         }
         for (String indexType : NON_QUANTIZED_DENSE_VECTOR_INDEX_TYPES) {
             params.add(new Object[] { DenseVectorFieldMapper.ElementType.BYTE, indexType });
@@ -264,7 +268,7 @@ public void setup() throws IOException {
             List<Number> vector = new ArrayList<>(numDims);
             for (int j = 0; j < numDims; j++) {
                 switch (elementType) {
-                    case FLOAT -> vector.add(randomFloatBetween(0F, 1F, true));
+                    case FLOAT, BFLOAT16 -> vector.add(randomFloatBetween(0F, 1F, true));
                     case BYTE, BIT -> vector.add((byte) (randomFloatBetween(0F, 1F, true) * 127.0f));
                     default -> throw new IllegalArgumentException("Unexpected element type: " + elementType);
                 }