Enable directIO and bfloat16 for bbq and unquantized HNSW vector field types

qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java

@@ -96,7 +96,8 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         // geohex_grid requires gold license
         .setting("xpack.license.self_generated.type", "trial")
         .feature(FeatureFlag.TIME_SERIES_MODE)
-        .feature(FeatureFlag.SYNTHETIC_VECTORS);
+        .feature(FeatureFlag.SYNTHETIC_VECTORS)
+        .feature(FeatureFlag.GENERIC_VECTOR_FORMAT);
 
     private static ElasticsearchCluster remoteCluster = ElasticsearchCluster.local()
         .name(REMOTE_CLUSTER_NAME)

qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java

@@ -96,6 +96,7 @@ public class RcsCcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         .setting("xpack.security.remote_cluster_client.ssl.enabled", "false")
         .feature(FeatureFlag.TIME_SERIES_MODE)
         .feature(FeatureFlag.SYNTHETIC_VECTORS)
+        .feature(FeatureFlag.GENERIC_VECTOR_FORMAT)
         .user("test_admin", "x-pack-test-password");
 
     private static ElasticsearchCluster fulfillingCluster = ElasticsearchCluster.local()

qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java

@@ -38,6 +38,7 @@ public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTe
         .feature(FeatureFlag.DOC_VALUES_SKIPPER)
         .feature(FeatureFlag.SYNTHETIC_VECTORS)
         .feature(FeatureFlag.RANDOM_SAMPLING)
+        .feature(FeatureFlag.GENERIC_VECTOR_FORMAT)
         .build();
 
     public SmokeTestMultiNodeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java

@@ -38,6 +38,7 @@ public class ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         .feature(FeatureFlag.DOC_VALUES_SKIPPER)
         .feature(FeatureFlag.SYNTHETIC_VECTORS)
         .feature(FeatureFlag.RANDOM_SAMPLING)
+        .feature(FeatureFlag.GENERIC_VECTOR_FORMAT)
         .build();
 
     public ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/200_dense_vector_docvalue_fields.yml

@@ -164,10 +164,7 @@ setup:
 "dense_vector docvalues with bfloat16":
   - requires:
       cluster_features: [ "mapper.vectors.generic_vector_format" ]
-      reason: Needs bfloat16 support
-  - skip:
-      awaits_fix: https://github.com/elastic/elasticsearch/issues/131109
-      reason:     Feature flag not enabled
+      reason: Needs generic vector support
   - do:
       indices.create:
         index: test-bfloat16

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search_bfloat16.yml

@@ -1,10 +1,7 @@
 setup:
   - requires:
-      cluster_features: ["mapper.vectors.hnsw_bfloat16_on_disk_rescoring"]
-      reason: 'bfloat16 needs to be supported'
-  - skip:
-      awaits_fix: https://github.com/elastic/elasticsearch/issues/131109
-      reason:     Feature flag not enabled
+      cluster_features: [ "mapper.vectors.generic_vector_format" ]
+      reason: Needs generic vector support
   - do:
       indices.create:
         index: test

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml

@@ -340,11 +340,8 @@ setup:
 ---
 "Test index configured rescore vector with on-disk rescoring":
   - requires:
-      cluster_features: ["mapper.vectors.generic_vector_format"]
-      reason: Needs on_disk_rescoring feature
-  - skip:
-      awaits_fix: https://github.com/elastic/elasticsearch/issues/131109
-      reason:     Feature flag not enabled
+      cluster_features: [ "mapper.vectors.generic_vector_format" ]
+      reason: Needs generic vector support
   - skip:
       features: "headers"
   - do:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw_bfloat16.yml

@@ -1,10 +1,7 @@
 setup:
   - requires:
-      cluster_features: "mapper.vectors.generic_vector_format"
-      reason: 'bfloat16 needs to be supported'
-  - skip:
-      awaits_fix: https://github.com/elastic/elasticsearch/issues/131109
-      reason:     Feature flag not enabled
+      cluster_features: [ "mapper.vectors.generic_vector_format" ]
+      reason: Needs generic vector support
   - do:
       indices.create:
         index: bbq_hnsw

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat_bfloat16.yml

@@ -1,10 +1,7 @@
 setup:
   - requires:
-      cluster_features: "mapper.vectors.generic_vector_format"
-      reason: 'bfloat16 needs to be supported'
-  - skip:
-      awaits_fix: https://github.com/elastic/elasticsearch/issues/131109
-      reason:     Feature flag not enabled
+      cluster_features: [ "mapper.vectors.generic_vector_format" ]
+      reason: Needs generic vector support
   - do:
       indices.create:
         index: bbq_flat

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/46_knn_search_bbq_ivf_bfloat16.yml

@@ -1,10 +1,7 @@
 setup:
   - requires:
-      cluster_features: ["mapper.vectors.hnsw_bfloat16_on_disk_rescoring"]
-      reason: 'bfloat16 needs to be supported'
-  - skip:
-      awaits_fix: https://github.com/elastic/elasticsearch/issues/131109
-      reason:     Feature flag not enabled
+      cluster_features: [ "mapper.vectors.generic_vector_format" ]
+      reason: Needs generic vector support
   - skip:
       features: "headers"
   - do:

server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java

@@ -1640,7 +1640,9 @@ public boolean supportsDimension(int dims) {
             public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map<String, ?> indexOptionsMap, IndexVersion indexVersion) {
                 Object mNode = indexOptionsMap.remove("m");
                 Object efConstructionNode = indexOptionsMap.remove("ef_construction");
-                Object onDiskRescoreNode = indexOptionsMap.remove("on_disk_rescore");
+                Object onDiskRescoreNode = ES93GenericFlatVectorsFormat.GENERIC_VECTOR_FORMAT.isEnabled()
+                    ? indexOptionsMap.remove("on_disk_rescore")
+                    : false;
 
                 int m = XContentMapValues.nodeIntegerValue(mNode, Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN);
                 int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode, Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH);
@@ -1731,7 +1733,9 @@ public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map<String, ?
                     }
                 }
 
-                Object onDiskRescoreNode = indexOptionsMap.remove("on_disk_rescore");
+                Object onDiskRescoreNode = ES93GenericFlatVectorsFormat.GENERIC_VECTOR_FORMAT.isEnabled()
+                    ? indexOptionsMap.remove("on_disk_rescore")
+                    : false;
                 boolean onDiskRescore = XContentMapValues.nodeBooleanValue(onDiskRescoreNode, false);
 
                 MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap);

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java

@@ -27,7 +27,8 @@ public enum FeatureFlag {
         null
     ),
     RANDOM_SAMPLING("es.random_sampling_feature_flag_enabled=true", Version.fromString("9.2.0"), null),
-    INFERENCE_API_CCM("es.inference_api_ccm_feature_flag_enabled=true", Version.fromString("9.3.0"), null);
+    INFERENCE_API_CCM("es.inference_api_ccm_feature_flag_enabled=true", Version.fromString("9.3.0"), null),
+    GENERIC_VECTOR_FORMAT("es.generic_vector_format_feature_flag_enabled=true", Version.fromString("9.3.0"), null);
 
     public final String systemProperty;
     public final Version from;