Semantic Text - Add YAML tests for inference metadata fields format (#118841)

Author: Mikep86

muted-tests.yml

@@ -148,6 +148,9 @@ tests:
 - class: org.elasticsearch.xpack.inference.InferenceRestIT
   method: test {p0=inference/30_semantic_text_inference/Calculates embeddings using the default ELSER 2 endpoint}
   issue: https://github.com/elastic/elasticsearch/issues/117349
+- class: org.elasticsearch.xpack.inference.InferenceRestIT
+  method: test {p0=inference/30_semantic_text_inference_bwc/Calculates embeddings using the default ELSER 2 endpoint}
+  issue: https://github.com/elastic/elasticsearch/issues/117349
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_reset/Test reset running transform}
   issue: https://github.com/elastic/elasticsearch/issues/117473

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestGetInferenceModelAction.java

@@ -28,7 +28,6 @@
 @ServerlessScope(Scope.PUBLIC)
 public class RestGetInferenceModelAction extends BaseRestHandler {
     public static final String DEFAULT_ELSER_2_CAPABILITY = "default_elser_2";
-    public static final String INFERENCE_METADATA_FIELDS_CAPABILITY = "inference_metadata_fields";
 
     @Override
     public String getName() {
@@ -67,6 +66,6 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
 
     @Override
     public Set<String> supportedCapabilities() {
-        return Set.of(DEFAULT_ELSER_2_CAPABILITY, INFERENCE_METADATA_FIELDS_CAPABILITY);
+        return Set.of(DEFAULT_ELSER_2_CAPABILITY);
     }
 }

x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml

@@ -7,6 +7,11 @@ setup:
       indices.create:
         index: test-index
         body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: false
           mappings:
             properties:
               sparse_field:
@@ -18,16 +23,6 @@ setup:
 
 ---
 "Indexes sparse vector document":
-  - requires:
-      test_runner_features: [capabilities]
-
-  - skip:
-      reason: Test targets semantic text without inference metadata fields
-      capabilities:
-        - method: GET
-          path: /_inference
-          capabilities: [inference_metadata_fields]
-
   # Checks mapping is not updated until first doc arrives
   - do:
       indices.get_mapping:
@@ -42,25 +37,28 @@ setup:
         index: test-index
         id: doc_1
         body:
-          sparse_field:
-            text: "these are not the droids you're looking for. He's free to go around"
+          sparse_field: "these are not the droids you're looking for. He's free to go around"
+          _inference_fields.sparse_field:
             inference:
               inference_id: sparse-inference-id
               model_settings:
                 task_type: sparse_embedding
               chunks:
-                - text: "these are not the droids you're looking for"
-                  embeddings:
-                    feature_0: 1.0
-                    feature_1: 2.0
-                    feature_2: 3.0
-                    feature_3: 4.0
-                - text: "He's free to go around"
-                  embeddings:
-                    feature_4: 0.1
-                    feature_5: 0.2
-                    feature_6: 0.3
-                    feature_7: 0.4
+                sparse_field:
+                  - start_offset: 0
+                    end_offset: 44
+                    embeddings:
+                      feature_0: 1.0
+                      feature_1: 2.0
+                      feature_2: 3.0
+                      feature_3: 4.0
+                  - start_offset: 44
+                    end_offset: 67
+                    embeddings:
+                      feature_4: 0.1
+                      feature_5: 0.2
+                      feature_6: 0.3
+                      feature_7: 0.4
 
   # Checks mapping is updated when first doc arrives
   - do:
@@ -77,14 +75,6 @@ setup:
   - requires:
       cluster_features: "gte_v8.16.0"
       reason: field_caps support for semantic_text added in 8.16.0
-      test_runner_features: [ capabilities ]
-
-  - skip:
-      reason: Test targets semantic text without inference metadata fields
-      capabilities:
-        - method: GET
-          path: /_inference
-          capabilities: [ inference_metadata_fields ]
 
   - do:
       field_caps:
@@ -111,25 +101,28 @@ setup:
         index: test-index
         id: doc_1
         body:
-          sparse_field:
-            text: "these are not the droids you're looking for. He's free to go around"
+          sparse_field: "these are not the droids you're looking for. He's free to go around"
+          _inference_fields.sparse_field:
             inference:
               inference_id: sparse-inference-id
               model_settings:
                 task_type: sparse_embedding
               chunks:
-                - text: "these are not the droids you're looking for"
-                  embeddings:
-                    feature_0: 1.0
-                    feature_1: 2.0
-                    feature_2: 3.0
-                    feature_3: 4.0
-                - text: "He's free to go around"
-                  embeddings:
-                    feature_4: 0.1
-                    feature_5: 0.2
-                    feature_6: 0.3
-                    feature_7: 0.4
+                sparse_field:
+                  - start_offset: 0
+                    end_offset: 44
+                    embeddings:
+                      feature_0: 1.0
+                      feature_1: 2.0
+                      feature_2: 3.0
+                      feature_3: 4.0
+                  - start_offset: 44
+                    end_offset: 67
+                    embeddings:
+                      feature_4: 0.1
+                      feature_5: 0.2
+                      feature_6: 0.3
+                      feature_7: 0.4
         refresh: true
 
   - do:
@@ -157,16 +150,6 @@ setup:
 
 ---
 "Indexes dense vector document":
-  - requires:
-      test_runner_features: [ capabilities ]
-
-  - skip:
-      reason: Test targets semantic text without inference metadata fields
-      capabilities:
-        - method: GET
-          path: /_inference
-          capabilities: [ inference_metadata_fields ]
-
   # Checks mapping is not updated until first doc arrives
   - do:
       indices.get_mapping:
@@ -181,8 +164,8 @@ setup:
         index: test-index
         id: doc_2
         body:
-          dense_field:
-            text: "these are not the droids you're looking for. He's free to go around"
+          dense_field: "these are not the droids you're looking for. He's free to go around"
+          _inference_fields.dense_field:
             inference:
               inference_id: dense-inference-id
               model_settings:
@@ -191,10 +174,13 @@ setup:
                 similarity: cosine
                 element_type: float
               chunks:
-                - text: "these are not the droids you're looking for"
-                  embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
-                - text: "He's free to go around"
-                  embeddings: [0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896]
+                dense_field:
+                  - start_offset: 0
+                    end_offset: 44
+                    embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
+                  - start_offset: 44
+                    end_offset: 67
+                    embeddings: [0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896]
 
   # Checks mapping is updated when first doc arrives
   - do:
@@ -211,14 +197,6 @@ setup:
   - requires:
       cluster_features: "gte_v8.16.0"
       reason: field_caps support for semantic_text added in 8.16.0
-      test_runner_features: [ capabilities ]
-
-  - skip:
-      reason: Test targets semantic text without inference metadata fields
-      capabilities:
-        - method: GET
-          path: /_inference
-          capabilities: [ inference_metadata_fields ]
 
   - do:
       field_caps:
@@ -245,8 +223,8 @@ setup:
         index: test-index
         id: doc_2
         body:
-          dense_field:
-            text: "these are not the droids you're looking for. He's free to go around"
+          dense_field: "these are not the droids you're looking for. He's free to go around"
+          _inference_fields.dense_field:
             inference:
               inference_id: dense-inference-id
               model_settings:
@@ -255,10 +233,13 @@ setup:
                 similarity: cosine
                 element_type: float
               chunks:
-                - text: "these are not the droids you're looking for"
-                  embeddings: [ 0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416 ]
-                - text: "He's free to go around"
-                  embeddings: [ 0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896 ]
+                dense_field:
+                  - start_offset: 0
+                    end_offset: 44
+                    embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416]
+                  - start_offset: 44
+                    end_offset: 67
+                    embeddings: [0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896]
         refresh: true
 
   - do: