Merge branch 'main' into lookupPrefilterPushable

Author: julian-elastic

docs/changelog/133675.yaml

@@ -0,0 +1,5 @@
+pr: 133675
+summary: Support using the semantic query across multiple inference IDs
+area: Vector Search
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -356,7 +356,8 @@ static TransportVersion def(int id) {
     public static final TransportVersion PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY = def(9_147_0_00);
     public static final TransportVersion STREAMS_ENDPOINT_PARAM_RESTRICTIONS = def(9_148_0_00);
     public static final TransportVersion RESOLVE_INDEX_MODE_FILTER = def(9_149_0_00);
-    public static final TransportVersion ESQL_LOOKUP_JOIN_PRE_JOIN_FILTER = def(9_150_0_00);
+    public static final TransportVersion SEMANTIC_QUERY_MULTIPLE_INFERENCE_IDS = def(9_150_0_00);
+    public static final TransportVersion ESQL_LOOKUP_JOIN_PRE_JOIN_FILTER = def(9_151_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/SemanticMatchTestCase.java

@@ -34,15 +34,28 @@ public abstract class SemanticMatchTestCase extends ESRestTestCase {
     public void testWithMultipleInferenceIds() throws IOException {
         assumeTrue("semantic text capability not available", EsqlCapabilities.Cap.SEMANTIC_TEXT_FIELD_CAPS.isEnabled());
 
+        var request1 = new Request("POST", "/test-semantic1/_doc/id-1");
+        request1.addParameter("refresh", "true");
+        request1.setJsonEntity("{\"semantic_text_field\": \"inference test 1\"}");
+        assertEquals(201, adminClient().performRequest(request1).getStatusLine().getStatusCode());
+
+        var request2 = new Request("POST", "/test-semantic2/_doc/id-2");
+        request2.addParameter("refresh", "true");
+        request2.setJsonEntity("{\"semantic_text_field\": \"inference test 2\"}");
+        assertEquals(201, adminClient().performRequest(request2).getStatusLine().getStatusCode());
+
         String query = """
             from test-semantic1,test-semantic2
             | where match(semantic_text_field, "something")
+            | SORT semantic_text_field ASC
             """;
-        ResponseException re = expectThrows(ResponseException.class, () -> runEsqlQuery(query));
-
-        assertThat(re.getMessage(), containsString("Field [semantic_text_field] has multiple inference IDs associated with it"));
+        Map<String, Object> result = runEsqlQuery(query);
 
-        assertEquals(400, re.getResponse().getStatusLine().getStatusCode());
+        assertResultMap(
+            result,
+            matchesList().item(matchesMap().entry("name", "semantic_text_field").entry("type", "text")),
+            matchesList(List.of(List.of("inference test 1"), List.of("inference test 2")))
+        );
     }
 
     public void testWithInferenceNotConfigured() {
@@ -128,6 +141,28 @@ public void setUpIndices() throws IOException {
         createIndex(adminClient(), "test-semantic4", settings, mapping4);
     }
 
+    @Before
+    public void setUpSparseEmbeddingInferenceEndpoint() throws IOException {
+        Request request = new Request("PUT", "_inference/sparse_embedding/test_sparse_inference");
+        request.setJsonEntity("""
+                  {
+                   "service": "test_service",
+                   "service_settings": {
+                     "model": "my_model",
+                     "api_key": "abc64"
+                   },
+                   "task_settings": {
+                   }
+                 }
+            """);
+        try {
+            adminClient().performRequest(request);
+        } catch (ResponseException exc) {
+            // in case the removal failed
+            assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(400));
+        }
+    }
+
     @Before
     public void setUpTextEmbeddingInferenceEndpoint() throws IOException {
         Request request = new Request("PUT", "_inference/text_embedding/test_dense_inference");
@@ -155,6 +190,15 @@ public void setUpTextEmbeddingInferenceEndpoint() throws IOException {
     public void wipeData() throws IOException {
         adminClient().performRequest(new Request("DELETE", "*"));
 
+        try {
+            adminClient().performRequest(new Request("DELETE", "_inference/test_sparse_inference"));
+        } catch (ResponseException e) {
+            // 404 here means the endpoint was not created
+            if (e.getResponse().getStatusLine().getStatusCode() != 404) {
+                throw e;
+            }
+        }
+
         try {
             adminClient().performRequest(new Request("DELETE", "_inference/test_dense_inference"));
         } catch (ResponseException e) {

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.xpack.inference.mapper.SemanticInferenceMetadataFieldsMapper;
 import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper;
+import org.elasticsearch.xpack.inference.queries.SemanticQueryBuilder;
 import org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankRetrieverBuilder;
 
 import java.util.HashSet;
@@ -82,7 +83,8 @@ public Set<NodeFeature> getTestFeatures() {
                 SEMANTIC_QUERY_REWRITE_INTERCEPTORS_PROPAGATE_BOOST_AND_QUERY_NAME_FIX,
                 SEMANTIC_TEXT_HIGHLIGHTING_FLAT,
                 SEMANTIC_TEXT_SPARSE_VECTOR_INDEX_OPTIONS,
-                SEMANTIC_TEXT_FIELDS_CHUNKS_FORMAT
+                SEMANTIC_TEXT_FIELDS_CHUNKS_FORMAT,
+                SemanticQueryBuilder.SEMANTIC_QUERY_MULTIPLE_INFERENCE_IDS
             )
         );
         if (RERANK_SNIPPETS.isEnabled()) {