Merge branch 'non-issue/esql-dense-vector-support-normalization' into non-issue/esql-dense-vector-byte-element-support

# Conflicts:
#	server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java
#	x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java
#	x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java

Author: carlosdelest

server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java

@@ -536,11 +536,10 @@ public AllReader reader(LeafReaderContext context) throws IOException {
                     FloatVectorValues floatVectorValues = context.reader().getFloatVectorValues(fieldName);
                     if (floatVectorValues != null) {
                         if (fieldType.isNormalized()) {
-                            return new FloatDenseVectorNormalizedValuesBlockReader(
-                                floatVectorValues,
-                                dimensions,
-                                context.reader().getNumericDocValues(fieldType.name() + COSINE_MAGNITUDE_FIELD_SUFFIX)
-                            );
+                            NumericDocValues magnitudeDocValues = context.reader()
+                                .getNumericDocValues(fieldType.name() + COSINE_MAGNITUDE_FIELD_SUFFIX);
+                            return new FloatDenseVectorNormalizedValuesBlockReader(floatVectorValues, dimensions, magnitudeDocValues);
+                        }
                         }
                         return new FloatDenseVectorValuesBlockReader(floatVectorValues, dimensions);
                     }
@@ -644,8 +643,12 @@ protected void appendDoc(BlockLoader.FloatBuilder builder) throws IOException {
             assert floats.length == dimensions
                 : "unexpected dimensions for vector value; expected " + dimensions + " but got " + floats.length;
 
-            assert magnitudeDocValues.advanceExact(iterator.docID());
-            float magnitude = Float.intBitsToFloat((int) magnitudeDocValues.longValue());
+            float magnitude = 1.0f;
+            // If all vectors are normalized, no doc values will be present. The vector may be normalized already, so we may not have a
+            // stored magnitude for all docs
+            if ((magnitudeDocValues != null) && magnitudeDocValues.advanceExact(iterator.docID())) {
+                magnitude = Float.intBitsToFloat((int) magnitudeDocValues.longValue());
+            }
             for (float aFloat : floats) {
                 builder.appendFloat(aFloat * magnitude);
             }
@@ -654,6 +657,7 @@ protected void appendDoc(BlockLoader.FloatBuilder builder) throws IOException {
         @Override
         public String toString() {
             return "BlockDocValuesReader.FloatDenseVectorNormalizedValuesBlockReader";
+            return "BlockDocValuesReader.FloatDenseVectorNormalizedValuesBlockReader";
         }
     }
 

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java

@@ -219,7 +219,7 @@ public void setup() throws IOException {
                         default -> throw new IllegalArgumentException("Unexpected element type: " + elementType);
                     }
                 }
-                if (similarity == DenseVectorFieldMapper.VectorSimilarity.DOT_PRODUCT) {
+                if (similarity == DenseVectorFieldMapper.VectorSimilarity.DOT_PRODUCT || rarely()) {
                     // Normalize the vector
                     float magnitude = DenseVector.getMagnitude(vector);
                     switch (elementType) {
@@ -248,10 +248,7 @@ private void createIndexWithDenseVector(String indexName) throws IOException {
             .field("element_type", elementType.toString().toLowerCase(Locale.ROOT))
             .field("index", index);
         if (index) {
-            mapping.field(
-                "similarity",
-                similarity.name().toLowerCase(Locale.ROOT)
-            );
+            mapping.field("similarity", similarity.name().toLowerCase(Locale.ROOT));
         }
         if (indexType != null) {
             mapping.startObject("index_options").field("type", indexType).endObject();