Add a feature flag for the new formats

Author: thecoop

server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93GenericFlatVectorsFormat.java

@@ -15,6 +15,7 @@
 import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
+import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.index.codec.vectors.AbstractFlatVectorsFormat;
 import org.elasticsearch.index.codec.vectors.DirectIOCapableFlatVectorsFormat;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
@@ -24,6 +25,8 @@
 
 public class ES93GenericFlatVectorsFormat extends AbstractFlatVectorsFormat {
 
+    public static final FeatureFlag ES93_VECTOR_FORMATS = new FeatureFlag("es93_vector_formats");
+
     static final String NAME = "ES93GenericFlatVectorsFormat";
     static final String VECTOR_FORMAT_INFO_EXTENSION = "vfi";
     static final String META_CODEC_NAME = "ES93GenericFlatVectorsFormatMeta";

server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java

@@ -11,7 +11,9 @@
 
 import org.elasticsearch.features.FeatureSpecification;
 import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.index.codec.vectors.es93.ES93GenericFlatVectorsFormat;
 
+import java.util.HashSet;
 import java.util.Set;
 
 import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.RESCORE_VECTOR_QUANTIZED_VECTOR_MAPPING;
@@ -63,7 +65,7 @@ public class MapperFeatures implements FeatureSpecification {
 
     @Override
     public Set<NodeFeature> getTestFeatures() {
-        return Set.of(
+        var features = Set.of(
             RangeFieldMapper.DATE_RANGE_INDEXING_FIX,
             IgnoredSourceFieldMapper.DONT_EXPAND_DOTS_IN_IGNORED_SOURCE,
             SourceFieldMapper.REMOVE_SYNTHETIC_SOURCE_ONLY_VALIDATION,
@@ -102,8 +104,12 @@ public Set<NodeFeature> getTestFeatures() {
             PROVIDE_INDEX_SORT_SETTING_DEFAULTS,
             INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_FIELD_NAME_LIMIT,
             EXCLUDE_VECTORS_DOCVALUE_BUGFIX,
-            BASE64_DENSE_VECTORS,
-            HNSW_BFLOAT16_ON_DISK_RESCORING
+            BASE64_DENSE_VECTORS
         );
+        if (ES93GenericFlatVectorsFormat.ES93_VECTOR_FORMATS.isEnabled()) {
+            features = new HashSet<>(features);
+            features.add(HNSW_BFLOAT16_ON_DISK_RESCORING);
+        }
+        return features;
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java

@@ -54,7 +54,10 @@
 import org.elasticsearch.index.codec.vectors.ES815BitFlatVectorFormat;
 import org.elasticsearch.index.codec.vectors.ES815HnswBitVectorsFormat;
 import org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat;
+import org.elasticsearch.index.codec.vectors.es818.ES818BinaryQuantizedVectorsFormat;
+import org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsFormat;
 import org.elasticsearch.index.codec.vectors.es93.ES93BinaryQuantizedVectorsFormat;
+import org.elasticsearch.index.codec.vectors.es93.ES93GenericFlatVectorsFormat;
 import org.elasticsearch.index.codec.vectors.es93.ES93HnswBinaryQuantizedVectorsFormat;
 import org.elasticsearch.index.codec.vectors.es93.ES93HnswVectorsFormat;
 import org.elasticsearch.index.fielddata.FieldDataContext;
@@ -238,7 +241,8 @@ public static class Builder extends FieldMapper.Builder {
 
         private final Parameter<ElementType> elementType = new Parameter<>("element_type", false, () -> ElementType.FLOAT, (n, c, o) -> {
             ElementType elementType = namesToElementType.get((String) o);
-            if (elementType == null) {
+            if (elementType == null
+                || (elementType == ElementType.BFLOAT16 && ES93GenericFlatVectorsFormat.ES93_VECTOR_FORMATS.isEnabled() == false)) {
                 throw new MapperParsingException("invalid element_type [" + o + "]; available types are " + namesToElementType.keySet());
             }
             return elementType;
@@ -2145,7 +2149,14 @@ public static class HnswIndexOptions extends DenseVectorIndexOptions {
 
         @Override
         public KnnVectorsFormat getVectorsFormat(ElementType elementType) {
-            return new ES93HnswVectorsFormat(m, efConstruction, elementType);
+            if (ES93GenericFlatVectorsFormat.ES93_VECTOR_FORMATS.isEnabled()) {
+                return new ES93HnswVectorsFormat(m, efConstruction, elementType);
+            } else {
+                if (elementType == ElementType.BIT) {
+                    return new ES815HnswBitVectorsFormat(m, efConstruction);
+                }
+                return new Lucene99HnswVectorsFormat(m, efConstruction, 1, null);
+            }
         }
 
         @Override
@@ -2220,7 +2231,9 @@ public BBQHnswIndexOptions(int m, int efConstruction, boolean onDiskRescore, Res
         @Override
         KnnVectorsFormat getVectorsFormat(ElementType elementType) {
             assert elementType == ElementType.FLOAT || elementType == ElementType.BFLOAT16;
-            return new ES93HnswBinaryQuantizedVectorsFormat(m, efConstruction, elementType, onDiskRescore);
+            return ES93GenericFlatVectorsFormat.ES93_VECTOR_FORMATS.isEnabled()
+                ? new ES93HnswBinaryQuantizedVectorsFormat(m, efConstruction, elementType, onDiskRescore)
+                : new ES818HnswBinaryQuantizedVectorsFormat(m, efConstruction);
         }
 
         @Override
@@ -2286,7 +2299,9 @@ static class BBQFlatIndexOptions extends QuantizedIndexOptions {
         @Override
         KnnVectorsFormat getVectorsFormat(ElementType elementType) {
             assert elementType == ElementType.FLOAT || elementType == ElementType.BFLOAT16;
-            return new ES93BinaryQuantizedVectorsFormat(elementType, false);
+            return ES93GenericFlatVectorsFormat.ES93_VECTOR_FORMATS.isEnabled()
+                ? new ES93BinaryQuantizedVectorsFormat(elementType, false)
+                : new ES818BinaryQuantizedVectorsFormat();
         }
 
         @Override