Fix default index options when dimensions are unset for legacy indices (elastic#130540) (elastic#130575)

Backport of elastic#130572 to 9.1

Author: jimczi

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml

@@ -605,6 +605,40 @@ setup:
   - match: { hits.hits.0._score: $knn_score0 }
   - match: { hits.hits.1._score: $knn_score1 }
   - match: { hits.hits.2._score: $knn_score2 }
+
+---
+"Dimensions are dynamically set":
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          mappings:
+            properties:
+              embedding:
+                type: dense_vector
+
+  - do:
+      index:
+        index: test_index
+        id: "0"
+        body:
+          embedding: [ 0.5, 111.3, -13.0, 14.8, -156.0 ]
+
+  - do:
+      indices.get_mapping:
+        index: test_index
+
+  - match: { test_index.mappings.properties.embedding.type: dense_vector }
+  - match: { test_index.mappings.properties.embedding.dims: 5 }
+
+  - do:
+      catch: bad_request
+      index:
+        index: test_index
+        id: "0"
+        body:
+          embedding: [ 0.5, 111.3 ]
+
 ---
 "Updating dim to null is not allowed":
   - requires:

server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java

@@ -327,7 +327,9 @@ public Builder(String name, IndexVersion indexVersionCreated) {
                         || previous.updatableTo(current)
                 );
             if (defaultInt8Hnsw || defaultBBQ8Hnsw) {
-                this.indexOptions.alwaysSerialize();
+                if (defaultBBQ8Hnsw == false || (dims != null && dims.isConfigured())) {
+                    this.indexOptions.alwaysSerialize();
+                }
             }
             this.indexed.addValidator(v -> {
                 if (v) {
@@ -350,21 +352,31 @@ public Builder(String name, IndexVersion indexVersionCreated) {
         }
 
         private DenseVectorIndexOptions defaultIndexOptions(boolean defaultInt8Hnsw, boolean defaultBBQHnsw) {
-            if (this.dims != null && this.dims.isConfigured() && elementType.getValue() == ElementType.FLOAT && this.indexed.getValue()) {
-                if (defaultBBQHnsw && this.dims.getValue() >= BBQ_DIMS_DEFAULT_THRESHOLD) {
-                    return new BBQHnswIndexOptions(
-                        Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
-                        Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
-                        new RescoreVector(DEFAULT_OVERSAMPLE)
-                    );
-                } else if (defaultInt8Hnsw) {
-                    return new Int8HnswIndexOptions(
-                        Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
-                        Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
-                        null,
-                        null
-                    );
-                }
+            if (elementType.getValue() != ElementType.FLOAT || indexed.getValue() == false) {
+                return null;
+            }
+
+            boolean dimIsConfigured = dims != null && dims.isConfigured();
+            if (defaultBBQHnsw && dimIsConfigured == false) {
+                // Delay selecting the default index options until dimensions are configured.
+                // This applies only to indices that are eligible to use BBQ as the default,
+                // since prior to this change, the default was selected eagerly.
+                return null;
+            }
+
+            if (defaultBBQHnsw && dimIsConfigured && dims.getValue() >= BBQ_DIMS_DEFAULT_THRESHOLD) {
+                return new BBQHnswIndexOptions(
+                    Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
+                    Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
+                    new RescoreVector(DEFAULT_OVERSAMPLE)
+                );
+            } else if (defaultInt8Hnsw) {
+                return new Int8HnswIndexOptions(
+                    Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
+                    Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
+                    null,
+                    null
+                );
             }
             return null;
         }
@@ -2713,47 +2725,9 @@ public void parse(DocumentParserContext context) throws IOException {
         }
         if (fieldType().dims == null) {
             int dims = fieldType().elementType.parseDimensionCount(context);
-            ;
-            final boolean defaultInt8Hnsw = indexCreatedVersion.onOrAfter(IndexVersions.DEFAULT_DENSE_VECTOR_TO_INT8_HNSW);
-            final boolean defaultBBQ8Hnsw = indexCreatedVersion.onOrAfter(IndexVersions.DEFAULT_DENSE_VECTOR_TO_BBQ_HNSW);
-            DenseVectorIndexOptions denseVectorIndexOptions = fieldType().indexOptions;
-            if (denseVectorIndexOptions == null && fieldType().getElementType() == ElementType.FLOAT && fieldType().isIndexed()) {
-                if (defaultBBQ8Hnsw && dims >= BBQ_DIMS_DEFAULT_THRESHOLD) {
-                    denseVectorIndexOptions = new BBQHnswIndexOptions(
-                        Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
-                        Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
-                        new RescoreVector(DEFAULT_OVERSAMPLE)
-                    );
-                } else if (defaultInt8Hnsw) {
-                    denseVectorIndexOptions = new Int8HnswIndexOptions(
-                        Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
-                        Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
-                        null,
-                        null
-                    );
-                }
-            }
-            if (denseVectorIndexOptions != null) {
-                denseVectorIndexOptions.validateDimension(dims);
-            }
-            DenseVectorFieldType updatedDenseVectorFieldType = new DenseVectorFieldType(
-                fieldType().name(),
-                indexCreatedVersion,
-                fieldType().elementType,
-                dims,
-                fieldType().indexed,
-                fieldType().similarity,
-                denseVectorIndexOptions,
-                fieldType().meta(),
-                fieldType().isSyntheticSource
-            );
-            Mapper update = new DenseVectorFieldMapper(
-                leafName(),
-                updatedDenseVectorFieldType,
-                builderParams,
-                denseVectorIndexOptions,
-                indexCreatedVersion
-            );
+            DenseVectorFieldMapper.Builder builder = (Builder) getMergeBuilder();
+            builder.dimensions(dims);
+            Mapper update = builder.build(context.createDynamicMapperBuilderContext());
             context.addDynamicMapper(update);
             return;
         }