diff --git a/docs/reference/query-languages/esql/_snippets/functions/description/knn.md b/docs/reference/query-languages/esql/_snippets/functions/description/knn.md index c39604bbf1fa6..5383b503d078b 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/description/knn.md +++ b/docs/reference/query-languages/esql/_snippets/functions/description/knn.md @@ -2,5 +2,5 @@ **Description** -Finds the k nearest vectors to a query vector, as measured by a similarity metric. knn function finds nearest vectors through approximate search on indexed dense_vectors. +Finds the k nearest vectors to a query vector, as measured by a similarity metric. knn function finds nearest vectors through approximate search on indexed dense_vectors or semantic_text fields. diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/knn.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/knn.md index fb1b98a1e8a7a..0a234d119fac0 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/parameters/knn.md +++ b/docs/reference/query-languages/esql/_snippets/functions/parameters/knn.md @@ -3,7 +3,7 @@ **Parameters** `field` -: Field that the query will target. +: Field that the query will target. knn function can be used with dense_vector or semantic_text fields. Other text fields are not allowed `query` : Vector value to find top nearest neighbours for. diff --git a/docs/reference/query-languages/esql/images/functions/knn.svg b/docs/reference/query-languages/esql/images/functions/knn.svg index 75a104a7cdcfa..501225abc9bfc 100644 --- a/docs/reference/query-languages/esql/images/functions/knn.svg +++ b/docs/reference/query-languages/esql/images/functions/knn.svg @@ -1 +1 @@ -KNN(field,query,options) \ No newline at end of file +KNN(field,query,options) \ No newline at end of file diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/knn.json b/docs/reference/query-languages/esql/kibana/definition/functions/knn.json index f4b77305a200b..cd5541f8e7bfe 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/knn.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/knn.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.", "type" : "scalar", "name" : "knn", - "description" : "Finds the k nearest vectors to a query vector, as measured by a similarity metric. knn function finds nearest vectors through approximate search on indexed dense_vectors.", + "description" : "Finds the k nearest vectors to a query vector, as measured by a similarity metric. knn function finds nearest vectors through approximate search on indexed dense_vectors or semantic_text fields.", "signatures" : [ ], "examples" : [ "from colors metadata _score\n| where knn(rgb_vector, [0, 120, 0])\n| sort _score desc, color asc" diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/knn.md b/docs/reference/query-languages/esql/kibana/docs/functions/knn.md index bea09b0bf50de..4ad34ae271381 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/knn.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/knn.md @@ -1,7 +1,7 @@ % This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. ### KNN -Finds the k nearest vectors to a query vector, as measured by a similarity metric. knn function finds nearest vectors through approximate search on indexed dense_vectors. +Finds the k nearest vectors to a query vector, as measured by a similarity metric. knn function finds nearest vectors through approximate search on indexed dense_vectors or semantic_text fields. ```esql from colors metadata _score diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java index b254612a700df..3b37a97daaf5e 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java @@ -55,6 +55,10 @@ public TypeResolution and(TypeResolution other) { return failed ? this : other; } + public TypeResolution or(TypeResolution other) { + return failed ? other : this; + } + public TypeResolution and(Supplier other) { return failed ? this : other.get(); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index b3f7acfe2eac4..6a54a038627a7 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -395,6 +395,9 @@ public enum DataType { // ES calls this 'point', but ESQL calls it 'cartesian_point' map.put("point", DataType.CARTESIAN_POINT); map.put("shape", DataType.CARTESIAN_SHAPE); + // semantic_text is returned as text by field_caps, but unit tests will retrieve it from the mapping + // so we need to map it here as well + map.put("semantic_text", DataType.TEXT); ES_TO_TYPE = Collections.unmodifiableMap(map); // DATETIME has different esType and typeName, add an entry in NAME_TO_TYPE with date as key map = TYPES.stream().collect(toMap(DataType::typeName, t -> t)); diff --git a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/KnnSemanticTextIT.java b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/KnnSemanticTextIT.java new file mode 100644 index 0000000000000..5b57b8edf3ca8 --- /dev/null +++ b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/KnnSemanticTextIT.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.qa.multi_node; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + +import org.elasticsearch.test.TestClustersThreadFilter; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.xpack.esql.qa.rest.KnnSemanticTextTestCase; +import org.junit.ClassRule; + +@ThreadLeakFilters(filters = TestClustersThreadFilter.class) +public class KnnSemanticTextIT extends KnnSemanticTextTestCase { + @ClassRule + public static ElasticsearchCluster cluster = Clusters.testCluster( + spec -> spec.module("x-pack-inference").plugin("inference-service-test") + ); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/KnnSemanticTextIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/KnnSemanticTextIT.java new file mode 100644 index 0000000000000..28db61abee562 --- /dev/null +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/KnnSemanticTextIT.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.qa.single_node; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + +import org.elasticsearch.test.TestClustersThreadFilter; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.xpack.esql.qa.rest.KnnSemanticTextTestCase; +import org.junit.ClassRule; + +@ThreadLeakFilters(filters = TestClustersThreadFilter.class) +public class KnnSemanticTextIT extends KnnSemanticTextTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = Clusters.testCluster(spec -> spec.plugin("inference-service-test")); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java index 7e0bd6031f455..8c6978e1423f8 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java @@ -73,6 +73,7 @@ import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.loadDataSetIntoEs; import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.COMPLETION; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.KNN_FUNCTION_V5; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METRICS_COMMAND; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.RERANK; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.SEMANTIC_TEXT_FIELD_CAPS; @@ -211,8 +212,12 @@ protected boolean supportsInferenceTestService() { } protected boolean requiresInferenceEndpoint() { - return Stream.of(SEMANTIC_TEXT_FIELD_CAPS.capabilityName(), RERANK.capabilityName(), COMPLETION.capabilityName()) - .anyMatch(testCase.requiredCapabilities::contains); + return Stream.of( + SEMANTIC_TEXT_FIELD_CAPS.capabilityName(), + RERANK.capabilityName(), + COMPLETION.capabilityName(), + KNN_FUNCTION_V5.capabilityName() + ).anyMatch(testCase.requiredCapabilities::contains); } protected boolean supportsIndexModeLookup() throws IOException { diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/KnnSemanticTextTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/KnnSemanticTextTestCase.java new file mode 100644 index 0000000000000..19f1327d15599 --- /dev/null +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/KnnSemanticTextTestCase.java @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.qa.rest; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.esql.AssertWarnings; +import org.elasticsearch.xpack.esql.CsvTestsDataLoader; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.rest.RestStatus.BAD_REQUEST; +import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.requestObjectBuilder; +import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.runEsqlSync; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.core.StringContains.containsString; + +/** + * Tests kNN queries on semantic_text fields. Mostly checks errors on the data node that can't be checked in other tests. + */ +public class KnnSemanticTextTestCase extends ESRestTestCase { + + @Rule(order = Integer.MIN_VALUE) + public ProfileLogger profileLogger = new ProfileLogger(); + + @Before + public void checkCapability() { + assumeTrue("knn with semantic text not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); + } + + @SuppressWarnings("unchecked") + public void testKnnQueryWithSemanticText() throws IOException { + String knnQuery = """ + FROM semantic-test METADATA _score + | WHERE knn(dense_semantic, [0, 1, 2]) + | KEEP id, _score, dense_semantic + | SORT _score DESC + | LIMIT 10 + """; + + Map response = runEsqlQuery(knnQuery); + List> columns = (List>) response.get("columns"); + assertThat(columns.size(), is(3)); + List> rows = (List>) response.get("values"); + assertThat(rows.size(), is(3)); + for (int row = 0; row < rows.size(); row++) { + List rowData = rows.get(row); + Integer id = (Integer) rowData.get(0); + assertThat(id, is(3 - row)); + } + } + + public void testKnnQueryOnTextField() throws IOException { + String knnQuery = """ + FROM semantic-test METADATA _score + | WHERE knn(text, [0, 1, 2]) + | KEEP id, _score, dense_semantic + | SORT _score DESC + | LIMIT 10 + """; + + ResponseException re = expectThrows(ResponseException.class, () -> runEsqlQuery(knnQuery)); + assertThat(re.getResponse().getStatusLine().getStatusCode(), is(BAD_REQUEST.getStatus())); + assertThat(re.getMessage(), containsString("[knn] queries are only supported on [dense_vector] fields")); + } + + public void testKnnQueryOnSparseSemanticTextField() throws IOException { + String knnQuery = """ + FROM semantic-test METADATA _score + | WHERE knn(sparse_semantic, [0, 1, 2]) + | KEEP id, _score, sparse_semantic + | SORT _score DESC + | LIMIT 10 + """; + + ResponseException re = expectThrows(ResponseException.class, () -> runEsqlQuery(knnQuery)); + assertThat(re.getResponse().getStatusLine().getStatusCode(), is(BAD_REQUEST.getStatus())); + assertThat(re.getMessage(), containsString("[knn] queries are only supported on [dense_vector] fields")); + } + + @Before + public void setUp() throws Exception { + super.setUp(); + setupInferenceEndpoints(); + setupIndex(); + } + + private void setupIndex() throws IOException { + Request request = new Request("PUT", "/semantic-test"); + request.setJsonEntity(""" + { + "mappings": { + "properties": { + "id": { + "type": "integer" + }, + "dense_semantic": { + "type": "semantic_text", + "inference_id": "test_dense_inference" + }, + "sparse_semantic": { + "type": "semantic_text", + "inference_id": "test_sparse_inference" + }, + "text": { + "type": "text", + "copy_to": ["dense_semantic", "sparse_semantic"] + } + } + } + } + """); + assertEquals(200, client().performRequest(request).getStatusLine().getStatusCode()); + + request = new Request("POST", "/_bulk?index=semantic-test&refresh=true"); + request.setJsonEntity(""" + {"index": {"_id": "1"}} + {"id": 1, "text": "sample text"} + {"index": {"_id": "2"}} + {"id": 2, "text": "another sample text"} + {"index": {"_id": "3"}} + {"id": 3, "text": "yet another sample text"} + """); + assertEquals(200, client().performRequest(request).getStatusLine().getStatusCode()); + } + + private void setupInferenceEndpoints() throws IOException { + CsvTestsDataLoader.createTextEmbeddingInferenceEndpoint(client()); + CsvTestsDataLoader.createSparseEmbeddingInferenceEndpoint(client()); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + client().performRequest(new Request("DELETE", "semantic-test")); + + if (CsvTestsDataLoader.clusterHasTextEmbeddingInferenceEndpoint(client())) { + CsvTestsDataLoader.deleteTextEmbeddingInferenceEndpoint(client()); + } + if (CsvTestsDataLoader.clusterHasSparseEmbeddingInferenceEndpoint(client())) { + CsvTestsDataLoader.deleteSparseEmbeddingInferenceEndpoint(client()); + } + } + + private Map runEsqlQuery(String query) throws IOException { + RestEsqlTestCase.RequestObjectBuilder builder = requestObjectBuilder().query(query); + return runEsqlSync(builder, new AssertWarnings.NoWarnings(), profileLogger); + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index c56ed4d489843..765e9d9173e87 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -415,6 +415,10 @@ public static void createInferenceEndpoints(RestClient client) throws IOExceptio createSparseEmbeddingInferenceEndpoint(client); } + if (clusterHasTextEmbeddingInferenceEndpoint(client) == false) { + createTextEmbeddingInferenceEndpoint(client); + } + if (clusterHasRerankInferenceEndpoint(client) == false) { createRerankInferenceEndpoint(client); } @@ -426,11 +430,12 @@ public static void createInferenceEndpoints(RestClient client) throws IOExceptio public static void deleteInferenceEndpoints(RestClient client) throws IOException { deleteSparseEmbeddingInferenceEndpoint(client); + deleteTextEmbeddingInferenceEndpoint(client); deleteRerankInferenceEndpoint(client); deleteCompletionInferenceEndpoint(client); } - /** The semantic_text mapping type require an inference endpoint that needs to be setup before creating the index. */ + /** The semantic_text mapping type requires inference endpoints that need to be setup before creating the index. */ public static void createSparseEmbeddingInferenceEndpoint(RestClient client) throws IOException { createInferenceEndpoint(client, TaskType.SPARSE_EMBEDDING, "test_sparse_inference", """ { @@ -441,14 +446,38 @@ public static void createSparseEmbeddingInferenceEndpoint(RestClient client) thr """); } + public static void createTextEmbeddingInferenceEndpoint(RestClient client) throws IOException { + createInferenceEndpoint(client, TaskType.TEXT_EMBEDDING, "test_dense_inference", """ + { + "service": "text_embedding_test_service", + "service_settings": { + "model": "my_model", + "api_key": "abc64", + "dimensions": 3, + "similarity": "l2_norm", + "element_type": "float" + }, + "task_settings": { } + } + """); + } + public static void deleteSparseEmbeddingInferenceEndpoint(RestClient client) throws IOException { deleteInferenceEndpoint(client, "test_sparse_inference"); } + public static void deleteTextEmbeddingInferenceEndpoint(RestClient client) throws IOException { + deleteInferenceEndpoint(client, "test_dense_inference"); + } + public static boolean clusterHasSparseEmbeddingInferenceEndpoint(RestClient client) throws IOException { return clusterHasInferenceEndpoint(client, TaskType.SPARSE_EMBEDDING, "test_sparse_inference"); } + public static boolean clusterHasTextEmbeddingInferenceEndpoint(RestClient client) throws IOException { + return clusterHasInferenceEndpoint(client, TaskType.TEXT_EMBEDDING, "test_dense_inference"); + } + public static void createRerankInferenceEndpoint(RestClient client) throws IOException { createInferenceEndpoint(client, TaskType.RERANK, "test_reranker", """ { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/semantic_text.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/semantic_text.csv index f79e44ab67ca3..fae7715a860c2 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/semantic_text.csv +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/semantic_text.csv @@ -1,4 +1,4 @@ -_id:keyword,semantic_text_field:semantic_text,st_bool:semantic_text,st_cartesian_point:semantic_text,st_cartesian_shape:semantic_text,st_datetime:semantic_text,st_double:semantic_text,st_geopoint:semantic_text,st_geoshape:semantic_text,st_integer:semantic_text,st_ip:semantic_text,st_long:semantic_text,st_unsigned_long:semantic_text,st_version:semantic_text,st_multi_value:semantic_text,st_unicode:semantic_text,host:keyword,description:text,value:long,st_base64:semantic_text,st_logs:semantic_text,language_name:keyword -1,live long and prosper,false,"POINT(4297.11 -1475.53)",,1953-09-02T00:00:00.000Z,5.20128E11,"POINT(42.97109630194 14.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",23,1.1.1.1,2147483648,2147483648,1.2.3,["Hello there!", "This is a random value", "for testing purposes"],你吃饭了吗,"host1","some description1",1001,ZWxhc3RpYw==,"2024-12-23T12:15:00.000Z 1.2.3.4 example@example.com 4553",English -2,all we have to decide is what to do with the time that is given to us,true,"POINT(7580.93 2272.77)",,2023-09-24T15:57:00.000Z,4541.11,"POINT(37.97109630194 21.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",122,1.1.2.1,123,2147483648.2,9.0.0,["nice to meet you", "bye bye!"],["谢谢", "对不起我的中文不好"],"host2","some description2",1002,aGVsbG8=,"2024-01-23T12:15:00.000Z 1.2.3.4 foo@example.com 42",French -3,be excellent to each other,,,,,,,,,,,,,,,"host3","some description3",1003,,"2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42",Spanish +_id:keyword,semantic_text_field:semantic_text,semantic_text_dense_field:semantic_text,st_bool:semantic_text,st_cartesian_point:semantic_text,st_cartesian_shape:semantic_text,st_datetime:semantic_text,st_double:semantic_text,st_geopoint:semantic_text,st_geoshape:semantic_text,st_integer:semantic_text,st_ip:semantic_text,st_long:semantic_text,st_unsigned_long:semantic_text,st_version:semantic_text,st_multi_value:semantic_text,st_unicode:semantic_text,host:keyword,description:text,value:long,st_base64:semantic_text,st_logs:semantic_text,language_name:keyword +1,live long and prosper,live long and prosper,false,"POINT(4297.11 -1475.53)",,1953-09-02T00:00:00.000Z,5.20128E11,"POINT(42.97109630194 14.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",23,1.1.1.1,2147483648,2147483648,1.2.3,["Hello there!", "This is a random value", "for testing purposes"],你吃饭了吗,"host1","some description1",1001,ZWxhc3RpYw==,"2024-12-23T12:15:00.000Z 1.2.3.4 example@example.com 4553",English +2,all we have to decide is what to do with the time that is given to us,all we have to decide is what to do with the time that is given to us,true,"POINT(7580.93 2272.77)",,2023-09-24T15:57:00.000Z,4541.11,"POINT(37.97109630194 21.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",122,1.1.2.1,123,2147483648.2,9.0.0,["nice to meet you", "bye bye!"],["谢谢", "对不起我的中文不好"],"host2","some description2",1002,aGVsbG8=,"2024-01-23T12:15:00.000Z 1.2.3.4 foo@example.com 42",French +3,be excellent to each other,be excellent to each other,,,,,,,,,,,,,,,"host3","some description3",1003,,"2023-01-23T12:15:00.000Z 127.0.0.1 some.email@foo.com 42",Spanish diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec index 7a0e854f63f90..e65d65f414cd1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec @@ -3,7 +3,7 @@ # top-n query at the shard level knnSearch -required_capability: knn_function_v4 +required_capability: knn_function_v5 // tag::knn-function[] from colors metadata _score @@ -30,7 +30,7 @@ chartreuse | [127.0, 255.0, 0.0] ; knnSearchWithSimilarityOption -required_capability: knn_function_v4 +required_capability: knn_function_v5 from colors metadata _score | where knn(rgb_vector, [255,192,203], {"similarity": 40}) @@ -46,7 +46,7 @@ wheat | [245.0, 222.0, 179.0] ; knnHybridSearch -required_capability: knn_function_v4 +required_capability: knn_function_v5 from colors metadata _score | where match(color, "blue") or knn(rgb_vector, [65,105,225]) @@ -69,7 +69,7 @@ yellow | [255.0, 255.0, 0.0] ; knnWithPrefilter -required_capability: knn_function_v4 +required_capability: knn_function_v5 from colors | where knn(rgb_vector, [120,180,0]) and (match(color, "olive") or match(color, "green")) @@ -83,7 +83,7 @@ olive ; knnWithNegatedPrefilter -required_capability: knn_function_v4 +required_capability: knn_function_v5 from colors metadata _score | where knn(rgb_vector, [128,128,0]) and not (match(color, "olive") or match(color, "chocolate")) @@ -106,7 +106,7 @@ orange | [255.0, 165.0, 0.0] ; knnAfterKeep -required_capability: knn_function_v4 +required_capability: knn_function_v5 from colors metadata _score | keep rgb_vector, color, _score @@ -125,7 +125,7 @@ rgb_vector:dense_vector ; knnAfterDrop -required_capability: knn_function_v4 +required_capability: knn_function_v5 from colors metadata _score | drop primary @@ -144,7 +144,7 @@ lime | [0.0, 255.0, 0.0] ; knnAfterEval -required_capability: knn_function_v4 +required_capability: knn_function_v5 from colors metadata _score | eval composed_name = locate(color, " ") > 0 @@ -163,7 +163,7 @@ golden rod | true ; knnWithConjunction -required_capability: knn_function_v4 +required_capability: knn_function_v5 from colors metadata _score | where knn(rgb_vector, [255,255,238]) and hex_code like "#FFF*" @@ -183,7 +183,7 @@ yellow | #FFFF00 | [255.0, 255.0, 0.0] ; knnWithDisjunctionAndFiltersConjunction -required_capability: knn_function_v4 +required_capability: knn_function_v5 from colors metadata _score | where (knn(rgb_vector, [0,255,255]) or knn(rgb_vector, [128, 0, 255])) and primary == true @@ -206,7 +206,7 @@ yellow | [255.0, 255.0, 0.0] ; knnWithNegationsAndFiltersConjunction -required_capability: knn_function_v4 +required_capability: knn_function_v5 from colors metadata _score | where (knn(rgb_vector, [0,255,255]) and not(primary == true and match(color, "blue"))) @@ -229,7 +229,7 @@ azure | [240.0, 255.0, 255.0] ; knnWithNonPushableConjunction -required_capability: knn_function_v4 +required_capability: knn_function_v5 from colors metadata _score | eval composed_name = locate(color, " ") > 0 @@ -253,7 +253,7 @@ maroon | false ; testKnnWithNonPushableDisjunctions -required_capability: knn_function_v4 +required_capability: knn_function_v5 from colors metadata _score | where knn(rgb_vector, [128,128,0]) or length(color) > 10 @@ -276,7 +276,7 @@ green ; testKnnWithNonPushableDisjunctionsAndMinCandidates -required_capability: knn_function_v4 +required_capability: knn_function_v5 from colors metadata _score | where (knn(rgb_vector, [128,128,0], {"min_candidates": 2}) and length(color) > 10) or (knn(rgb_vector, [128,0,128], {"min_candidates": 2}) and primary == true) @@ -300,7 +300,7 @@ cyan | true ; testKnnWithStats -required_capability: knn_function_v4 +required_capability: knn_function_v5 from colors metadata _score | where knn(rgb_vector, [128,128,0]) @@ -314,7 +314,7 @@ c:long ; testKnnWithRerank -required_capability: knn_function_v4 +required_capability: knn_function_v5 required_capability: rerank from colors metadata _score @@ -338,3 +338,75 @@ chocolate firebrick golden rod ; + +testKnnWithSemanticText +required_capability: knn_function_v5 + +from semantic_text +| where knn(semantic_text_dense_field, [0, 1, 2]) +| keep semantic_text_dense_field +| sort semantic_text_dense_field asc +; + +semantic_text_dense_field:text +all we have to decide is what to do with the time that is given to us +be excellent to each other +live long and prosper +; + +testKnnWithSemanticTextAndKeyword +required_capability: knn_function_v5 + +from semantic_text +| where knn(semantic_text_dense_field, [0, 1, 2]) +| keep semantic_text_dense_field, host +| sort host asc +; + +semantic_text_dense_field:text | host:keyword +live long and prosper | host1 +all we have to decide is what to do with the time that is given to us | host2 +be excellent to each other | host3 + +; + +testKnnWithSemanticTextMultiValueField +required_capability: knn_function_v5 + +from semantic_text metadata _id +| where match(st_multi_value, "something") AND match(host, "host1") +| keep _id, st_multi_value +; + +_id: keyword | st_multi_value:text +1 | ["Hello there!", "This is a random value", "for testing purposes"] +; + +testKnnWithSemanticTextWithEvalsAndOtherFunctionsAndStats +required_capability: knn_function_v5 + +from semantic_text +| where qstr("description:some*") +| eval size = mv_count(st_multi_value) +| where knn(semantic_text_dense_field, [0, 1, 2]) +| limit 100 +| stats result = count(*) +; + +result:long +3 +; + +testKnnWithSemanticTextAndKql +required_capability: knn_function_v5 +required_capability: kql_function + +from semantic_text +| where kql("host:host1") AND knn(semantic_text_dense_field, [0, 1, 2]) +| KEEP host, semantic_text_dense_field +; + +host:keyword | semantic_text_dense_field:text +"host1" | live long and prosper +; + diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-full_text_search.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-full_text_search.json index 160f285d792d1..a5458bc3f3074 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-full_text_search.json +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-full_text_search.json @@ -21,6 +21,10 @@ "vector": { "type": "dense_vector", "similarity": "l2_norm" + }, + "semantic": { + "type": "semantic_text", + "inference_id": "test_inference" } } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json index 5fa25e01ef0e4..d4a6fff384287 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json @@ -4,6 +4,10 @@ "type": "semantic_text", "inference_id": "test_sparse_inference" }, + "semantic_text_dense_field": { + "type": "semantic_text", + "inference_id": "test_dense_inference" + }, "st_bool": { "type": "semantic_text", "inference_id": "test_sparse_inference" diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java index 21ec240d9f8f4..70356cd36f34e 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java @@ -196,7 +196,7 @@ public void testKnnWithLookupJoin() { @Before public void setup() throws IOException { - assumeTrue("Needs KNN support", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("Needs KNN support", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); var indexName = "test"; var client = client().admin().indices(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index e7b6ce320fc55..083fcf04bf150 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1291,7 +1291,7 @@ public enum Cap { /** * Support knn function */ - KNN_FUNCTION_V4(Build.current().isSnapshot()), + KNN_FUNCTION_V5(Build.current().isSnapshot()), /** * Support for the LIKE operator with a list of wildcards. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java index 9add14da034b5..79930ef057837 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java @@ -66,6 +66,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.DENSE_VECTOR; import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT; import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; +import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.expression.Foldables.TypeResolutionValidator.forPreOptimizationValidation; import static org.elasticsearch.xpack.esql.expression.Foldables.resolveTypeQuery; @@ -76,6 +77,8 @@ public class Knn extends FullTextFunction PostAnalysisPlanVerificationAware, PostOptimizationVerificationAware { + private static final String[] ACCEPTED_FIELD_TYPES = { "dense_vector", "semantic_text" }; + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Knn", Knn::readFrom); private final Expression field; @@ -98,13 +101,18 @@ public class Knn extends FullTextFunction returnType = "boolean", preview = true, description = "Finds the k nearest vectors to a query vector, as measured by a similarity metric. " - + "knn function finds nearest vectors through approximate search on indexed dense_vectors.", + + "knn function finds nearest vectors through approximate search on indexed dense_vectors or semantic_text fields.", examples = { @Example(file = "knn-function", tag = "knn-function") }, appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.DEVELOPMENT) } ) public Knn( Source source, - @Param(name = "field", type = { "dense_vector" }, description = "Field that the query will target.") Expression field, + @Param( + name = "field", + type = { "dense_vector", "text" }, + description = "Field that the query will target. " + + "knn function can be used with dense_vector or semantic_text fields. Other text fields are not allowed" + ) Expression field, @Param( name = "query", type = { "dense_vector" }, @@ -205,7 +213,12 @@ protected TypeResolution resolveParams() { } private TypeResolution resolveField() { - return isNotNull(field(), sourceText(), FIRST).and(isType(field(), dt -> dt == DENSE_VECTOR, sourceText(), FIRST, "dense_vector")); + return isNotNull(field(), sourceText(), FIRST).and( + // It really should be semantic_text instead of text, but field_caps retrieves semantic_text fields as text + isType(field(), dt -> dt == TEXT, sourceText(), FIRST, ACCEPTED_FIELD_TYPES).or( + isType(field(), dt -> dt == DENSE_VECTOR, sourceText(), FIRST, ACCEPTED_FIELD_TYPES) + ) + ); } private TypeResolution resolveQuery() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorWritables.java index ab41201ceb328..d43ec3e92981d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorWritables.java @@ -27,7 +27,7 @@ private VectorWritables() { public static List getNamedWritables() { List entries = new ArrayList<>(); - if (EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()) { + if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { entries.add(Knn.ENTRY); } if (EsqlCapabilities.Cap.COSINE_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 97429ea091053..2941355eac39c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -305,7 +305,7 @@ public final void test() throws Throwable { ); assumeFalse( "can't use KNN function in csv tests", - testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.KNN_FUNCTION_V4.capabilityName()) + testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.KNN_FUNCTION_V5.capabilityName()) ); assumeFalse( "lookup join disabled for csv tests", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 3e595869d8f88..a2d6f770e8915 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -2349,7 +2349,7 @@ public void testImplicitCasting() { public void testDenseVectorImplicitCastingKnn() { assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE.isEnabled()); - assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); checkDenseVectorCastingKnn("float_vector"); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 815ae4bae6b89..a83dae458934e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1268,7 +1268,7 @@ public void testFieldBasedFullTextFunctions() throws Exception { checkFieldBasedWithNonIndexedColumn("Term", "term(text, \"cat\")", "function"); checkFieldBasedFunctionNotAllowedAfterCommands("Term", "function", "term(title, \"Meditation\")"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()) { + if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { checkFieldBasedFunctionNotAllowedAfterCommands("KNN", "function", "knn(vector, [1, 2, 3])"); } } @@ -1401,7 +1401,7 @@ public void testFullTextFunctionsOnlyAllowedInWhere() throws Exception { if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { checkFullTextFunctionsOnlyAllowedInWhere("MultiMatch", "multi_match(\"Meditation\", title, body)", "function"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()) { + if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { checkFullTextFunctionsOnlyAllowedInWhere("KNN", "knn(vector, [0, 1, 2])", "function"); } @@ -1456,7 +1456,7 @@ public void testFullTextFunctionsDisjunctions() { if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { checkWithFullTextFunctionsDisjunctions("term(title, \"Meditation\")"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()) { + if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { checkWithFullTextFunctionsDisjunctions("knn(vector, [1, 2, 3])"); } } @@ -1521,7 +1521,7 @@ public void testFullTextFunctionsWithNonBooleanFunctions() { if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { checkFullTextFunctionsWithNonBooleanFunctions("Term", "term(title, \"Meditation\")", "function"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()) { + if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { checkFullTextFunctionsWithNonBooleanFunctions("KNN", "knn(vector, [1, 2, 3])", "function"); } } @@ -1592,7 +1592,7 @@ public void testFullTextFunctionsTargetsExistingField() throws Exception { if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { testFullTextFunctionTargetsExistingField("term(fist_name, \"Meditation\")"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()) { + if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { testFullTextFunctionTargetsExistingField("knn(vector, [0, 1, 2], 10)"); } } @@ -2189,7 +2189,7 @@ public void testFullTextFunctionOptions() { if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { checkOptionDataTypes(MultiMatch.OPTIONS, "FROM test | WHERE MULTI_MATCH(\"Jean\", title, body, {\"%s\": %s})"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()) { + if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { checkOptionDataTypes(Knn.ALLOWED_OPTIONS, "FROM test | WHERE KNN(vector, [0.1, 0.2, 0.3], {\"%s\": %s})"); } } @@ -2282,7 +2282,7 @@ public void testFullTextFunctionsNullArgs() throws Exception { checkFullTextFunctionNullArgs("term(null, \"query\")", "first"); checkFullTextFunctionNullArgs("term(title, null)", "second"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()) { + if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { checkFullTextFunctionNullArgs("knn(null, [0, 1, 2])", "first"); checkFullTextFunctionNullArgs("knn(vector, null)", "second"); } @@ -2313,7 +2313,7 @@ public void testFullTextFunctionsInStats() { if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { checkFullTextFunctionsInStats("multi_match(\"Meditation\", title, body)"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()) { + if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { checkFullTextFunctionsInStats("knn(vector, [0, 1, 2])"); } } @@ -2375,6 +2375,16 @@ public void testVectorSimilarityFunctionsNullArgs() throws Exception { } } + public void testFullTextFunctionsWithSemanticText() { + checkFullTextFunctionsWithSemanticText("knn(semantic, [0, 1, 2])"); + checkFullTextFunctionsWithSemanticText("match(semantic, \"hello world\")"); + checkFullTextFunctionsWithSemanticText("semantic:\"hello world\""); + } + + public void checkFullTextFunctionsWithSemanticText(String functionInvocation) { + query("from test | where " + functionInvocation, fullTextAnalyzer); + } + public void testToIPInvalidOptions() { String query = "ROW result = to_ip(\"127.0.0.1\", 123)"; assertThat(error(query), containsString("second argument of [to_ip(\"127.0.0.1\", 123)] must be a map expression, received [123]")); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KnnTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KnnTests.java index f87e278bd4238..49a6d3c904203 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KnnTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KnnTests.java @@ -52,7 +52,7 @@ public static Iterable parameters() { @Before public void checkCapability() { - assumeTrue("KNN is not enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("KNN is not enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); } private static List testCaseSuppliers() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index 2ca2ab09f73c0..43c4dd449b045 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -1385,7 +1385,7 @@ public void testMultiMatchOptionsPushDown() { public void testKnnOptionsPushDown() { assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE.isEnabled()); - assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ from test @@ -1411,7 +1411,7 @@ public void testKnnOptionsPushDown() { public void testKnnUsesLimitForK() { assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE.isEnabled()); - assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ from test @@ -1430,7 +1430,7 @@ public void testKnnUsesLimitForK() { public void testKnnKAndMinCandidatesLowerK() { assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE.isEnabled()); - assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ from test @@ -1449,7 +1449,7 @@ public void testKnnKAndMinCandidatesLowerK() { public void testKnnKAndMinCandidatesHigherK() { assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE.isEnabled()); - assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ from test @@ -1908,7 +1908,7 @@ public void testFullTextFunctionWithStatsBy(FullTextFunctionTestCase testCase) { } public void testKnnPrefilters() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ from test @@ -1940,7 +1940,7 @@ public void testKnnPrefilters() { } public void testKnnPrefiltersWithMultipleFilters() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ from test @@ -1976,7 +1976,7 @@ public void testKnnPrefiltersWithMultipleFilters() { } public void testPushDownConjunctionsToKnnPrefilter() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ from test @@ -2013,7 +2013,7 @@ public void testPushDownConjunctionsToKnnPrefilter() { } public void testPushDownNegatedConjunctionsToKnnPrefilter() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ from test @@ -2050,7 +2050,7 @@ public void testPushDownNegatedConjunctionsToKnnPrefilter() { } public void testNotPushDownDisjunctionsToKnnPrefilter() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ from test @@ -2079,7 +2079,7 @@ public void testNotPushDownDisjunctionsToKnnPrefilter() { } public void testNotPushDownKnnWithNonPushablePrefilters() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ from test @@ -2113,7 +2113,7 @@ public void testNotPushDownKnnWithNonPushablePrefilters() { } public void testPushDownComplexNegationsToKnnPrefilter() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ from test @@ -2163,7 +2163,7 @@ and NOT ((keyword == "test") or knn(dense_vector, [4, 5, 6]))) } public void testMultipleKnnQueriesInPrefilters() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ from test diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 9964cbb89b99b..82119fb7baa82 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -8506,7 +8506,7 @@ public void testSampleNoPushDownChangePoint() { } public void testPushDownConjunctionsToKnnPrefilter() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); var query = """ from test @@ -8526,7 +8526,7 @@ public void testPushDownConjunctionsToKnnPrefilter() { } public void testPushDownMultipleFiltersToKnnPrefilter() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); var query = """ from test @@ -8549,7 +8549,7 @@ public void testPushDownMultipleFiltersToKnnPrefilter() { } public void testNotPushDownDisjunctionsToKnnPrefilter() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); var query = """ from test @@ -8566,7 +8566,7 @@ public void testNotPushDownDisjunctionsToKnnPrefilter() { } public void testPushDownConjunctionsAndNotDisjunctionsToKnnPrefilter() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); /* and @@ -8601,7 +8601,7 @@ public void testPushDownConjunctionsAndNotDisjunctionsToKnnPrefilter() { } public void testMorePushDownConjunctionsAndNotDisjunctionsToKnnPrefilter() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); /* or @@ -8633,7 +8633,7 @@ public void testMorePushDownConjunctionsAndNotDisjunctionsToKnnPrefilter() { } public void testMultipleKnnQueriesInPrefilters() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); /* and @@ -8676,7 +8676,7 @@ public void testMultipleKnnQueriesInPrefilters() { } public void testKnnImplicitLimit() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); var query = """ from test @@ -8691,7 +8691,7 @@ public void testKnnImplicitLimit() { } public void testKnnWithLimit() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); var query = """ from test @@ -8707,7 +8707,7 @@ public void testKnnWithLimit() { } public void testKnnWithTopN() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); var query = """ from test metadata _score @@ -8724,7 +8724,7 @@ public void testKnnWithTopN() { } public void testKnnWithMultipleLimitsAfterTopN() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); var query = """ from test metadata _score @@ -8744,7 +8744,7 @@ public void testKnnWithMultipleLimitsAfterTopN() { } public void testKnnWithMultipleLimitsCombined() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); var query = """ from test metadata _score @@ -8762,7 +8762,7 @@ public void testKnnWithMultipleLimitsCombined() { } public void testKnnWithMultipleClauses() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); var query = """ from test metadata _score @@ -8785,7 +8785,7 @@ public void testKnnWithMultipleClauses() { } public void testKnnWithStats() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); assertThat( typesError("from test | where knn(dense_vector, [0, 1, 2]) | stats c = count(*)"), @@ -8794,7 +8794,7 @@ public void testKnnWithStats() { } public void testKnnWithRerankAmdTopN() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); assertThat(typesError(""" from test metadata _score @@ -8806,7 +8806,7 @@ public void testKnnWithRerankAmdTopN() { } public void testKnnWithRerankAmdLimit() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V4.isEnabled()); + assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); var query = """ from test metadata _score