Fix null values handling for dense vectors (elastic#131808)

Author: carlosdelest

server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java

@@ -548,11 +548,7 @@ public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset) throw
             // Doubles from doc values ensures that the values are in order
             try (BlockLoader.FloatBuilder builder = factory.denseVectors(docs.count() - offset, dimensions)) {
                 for (int i = offset; i < docs.count(); i++) {
-                    int doc = docs.get(i);
-                    if (doc < iterator.docID()) {
-                        throw new IllegalStateException("docs within same block must be in order");
-                    }
-                    read(doc, builder);
+                    read(docs.get(i), builder);
                 }
                 return builder.build();
             }
@@ -564,7 +560,9 @@ public void read(int docId, BlockLoader.StoredFields storedFields, Builder build
         }
 
         private void read(int doc, BlockLoader.FloatBuilder builder) throws IOException {
-            if (iterator.advance(doc) == doc) {
+            if (iterator.docID() > doc) {
+                builder.appendNull();
+            } else if (iterator.docID() == doc || iterator.advance(doc) == doc) {
                 builder.beginPositionEntry();
                 float[] floats = floatVectorValues.vectorValue(iterator.index());
                 assert floats.length == dimensions

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java

@@ -94,10 +94,14 @@ public void testRetrieveTopNDenseVectorFieldData() {
                 var values = valuesList.get(id);
                 assertEquals(id, values.get(0));
                 List<Float> vectors = (List<Float>) values.get(1);
-                assertNotNull(vectors);
-                assertEquals(vector.size(), vectors.size());
-                for (int i = 0; i < vector.size(); i++) {
-                    assertEquals(vector.get(i), vectors.get(i), 0F);
+                if (vector == null) {
+                    assertNull(vectors);
+                } else {
+                    assertNotNull(vectors);
+                    assertEquals(vector.size(), vectors.size());
+                    for (int i = 0; i < vector.size(); i++) {
+                        assertEquals(vector.get(i), vectors.get(i), 0F);
+                    }
                 }
             });
         }
@@ -117,12 +121,18 @@ public void testRetrieveDenseVectorFieldData() {
                 ;
                 assertEquals(2, value.size());
                 Integer id = (Integer) value.get(0);
-                List<Float> vector = (List<Float>) value.get(1);
-                assertNotNull(vector);
                 List<Float> expectedVector = indexedVectors.get(id);
-                assertNotNull(expectedVector);
-                for (int i = 0; i < vector.size(); i++) {
-                    assertEquals(expectedVector.get(i), vector.get(i), 0F);
+                List<Float> vector = (List<Float>) value.get(1);
+                if (expectedVector == null) {
+                    assertNull(vector);
+                } else {
+                    assertNotNull(vector);
+                    assertEquals(expectedVector.size(), vector.size());
+                    assertNotNull(vector);
+                    assertNotNull(expectedVector);
+                    for (int i = 0; i < vector.size(); i++) {
+                        assertEquals(expectedVector.get(i), vector.get(i), 0F);
+                    }
                 }
             });
         }
@@ -168,11 +178,16 @@ public void setup() throws IOException {
         IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs];
         for (int i = 0; i < numDocs; i++) {
             List<Float> vector = new ArrayList<>(numDims);
-            for (int j = 0; j < numDims; j++) {
-                vector.add(randomFloat());
+            if (rarely()) {
+                docs[i] = prepareIndex("test").setId("" + i).setSource("id", String.valueOf(i));
+                indexedVectors.put(i, null);
+            } else {
+                for (int j = 0; j < numDims; j++) {
+                    vector.add(randomFloat());
+                }
+                docs[i] = prepareIndex("test").setId("" + i).setSource("id", String.valueOf(i), "vector", vector);
+                indexedVectors.put(i, vector);
             }
-            docs[i] = prepareIndex("test").setId("" + i).setSource("id", String.valueOf(i), "vector", vector);
-            indexedVectors.put(i, vector);
         }
 
         indexRandom(true, docs);