Fixed BlockDocValuesReader to work with all dense_vector types

Author: carlosdelest

server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java

@@ -12,6 +12,7 @@
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.FloatVectorValues;
+import org.apache.lucene.index.KnnVectorValues;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.SortedDocValues;
@@ -529,18 +530,19 @@ public AllReader reader(LeafReaderContext context) throws IOException {
 
     private static class FloatVectorValuesBlockReader extends BlockDocValuesReader {
         private final FloatVectorValues floatVectorValues;
-        private int docId = -1;
+        private final KnnVectorValues.DocIndexIterator iterator;
 
         FloatVectorValuesBlockReader(FloatVectorValues floatVectorValues) {
             this.floatVectorValues = floatVectorValues;
+            iterator = floatVectorValues.iterator();
         }
 
         @Override
         public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException {
             try (BlockLoader.DoubleBuilder builder = factory.doubles(docs.count())) {
                 for (int i = 0; i < docs.count(); i++) {
                     int doc = docs.get(i);
-                    if (doc < docId) {
+                    if (doc < iterator.docID()) {
                         throw new IllegalStateException("docs within same block must be in order");
                     }
                     read(doc, builder);
@@ -555,22 +557,21 @@ public void read(int docId, BlockLoader.StoredFields storedFields, Builder build
         }
 
         private void read(int doc, DoubleBuilder builder) throws IOException {
-            float[] floats = floatVectorValues.vectorValue(doc);
-            if (floats != null) {
+            if (iterator.advance(doc) == doc) {
                 builder.beginPositionEntry();
+                float[] floats = floatVectorValues.vectorValue(iterator.index());
                 for (float aFloat : floats) {
                     builder.appendDouble(aFloat);
                 }
                 builder.endPositionEntry();
             } else {
                 builder.appendNull();
             }
-            docId = doc;
         }
 
         @Override
         public int docId() {
-            return docId;
+            return iterator.docID();
         }
 
         @Override

server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java

@@ -35,7 +35,6 @@
 import org.apache.lucene.search.join.BitSetProducer;
 import org.apache.lucene.util.BitUtil;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.NumericUtils;
 import org.apache.lucene.util.VectorUtil;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -54,7 +53,6 @@
 import org.elasticsearch.index.mapper.ArraySourceValueFetcher;
 import org.elasticsearch.index.mapper.BlockDocValuesReader;
 import org.elasticsearch.index.mapper.BlockLoader;
-import org.elasticsearch.index.mapper.BlockSourceReader;
 import org.elasticsearch.index.mapper.DocumentParserContext;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -2313,8 +2311,6 @@ ElementType getElementType() {
 
         @Override
         public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) {
-            // TODO Check synthetic source etc (see NumberFieldMapper)
-
             return new BlockDocValuesReader.DenseVectorBlockLoader(name());
         }
     }

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java

@@ -7,22 +7,48 @@
 
 package org.elasticsearch.xpack.esql;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase;
-import org.hamcrest.CoreMatchers;
 import org.junit.Before;
 
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 
 public class DenseVectorFieldTypeIT extends AbstractEsqlIntegTestCase {
 
+    private static final Set<String> DENSE_VECTOR_INDEX_TYPES = Set.of(
+        "int8_hnsw",
+        "hnsw",
+        "int4_hnsw",
+        "bbq_hnsw",
+        "int8_flat",
+        "int4_flat",
+        "bbq_flat",
+        "flat"
+    );
+
+    private final String indexType;
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return DENSE_VECTOR_INDEX_TYPES.stream().map(type -> new Object[] { type }).toList();
+    }
+
+    public DenseVectorFieldTypeIT(@Name("indexType") String indexType) {
+        this.indexType = indexType;
+    }
+
     private static Map<Integer, List<Float>> DOC_VALUES = new HashMap<>();
     static {
         DOC_VALUES.put(1, List.of(1.0f, 2.0f, 3.0f));
@@ -80,15 +106,15 @@ public void testSorted() {
     public void setup() {
         var indexName = "test";
         var client = client().admin().indices();
-        var mapping = """
+        var mapping = String.format(Locale.ROOT, """
                 "id": integer,
                 "vector": {
                     "type": "dense_vector",
                     "index_options": {
-                        "type": "hnsw"
+                        "type": "%s"
                     }
                 }
-            """;
+            """, indexType);
         var CreateRequest = client.prepareCreate(indexName)
             .setSettings(Settings.builder().put("index.number_of_shards", 1))
             .setMapping(mapping);