From f943ceb8248c013ad7e62c71a55e3a22fdfefc02 Mon Sep 17 00:00:00 2001 From: cdelgado Date: Mon, 29 Sep 2025 11:34:26 +0200 Subject: [PATCH 01/35] ES|QL - dense_vector field type and to_dense_vector function are out of snapshot --- .../esql/images/functions/to_dense_vector.svg | 2 +- .../definition/functions/to_dense_vector.json | 55 ++++++++++++++++++- .../xpack/esql/core/type/DataType.java | 3 +- .../xpack/esql/DenseVectorFieldTypeIT.java | 3 - .../vector/VectorSimilarityFunctionsIT.java | 2 - .../xpack/esql/action/EsqlCapabilities.java | 4 +- .../esql/expression/ExpressionWritables.java | 5 +- .../function/EsqlFunctionRegistry.java | 2 +- .../scalar/convert/ToDenseVector.java | 1 + .../esql/type/EsqlDataTypeConverter.java | 6 +- .../xpack/esql/analysis/AnalyzerTests.java | 8 --- .../scalar/convert/ToDenseVectorTests.java | 7 --- .../LocalPhysicalPlanOptimizerTests.java | 4 -- 13 files changed, 60 insertions(+), 42 deletions(-) diff --git a/docs/reference/query-languages/esql/images/functions/to_dense_vector.svg b/docs/reference/query-languages/esql/images/functions/to_dense_vector.svg index 54304ee44b11f..36a8f66b3af2a 100644 --- a/docs/reference/query-languages/esql/images/functions/to_dense_vector.svg +++ b/docs/reference/query-languages/esql/images/functions/to_dense_vector.svg @@ -1 +1 @@ -TO_DENSE_VECTOR(field) \ No newline at end of file +TO_DENSE_VECTOR(field) \ No newline at end of file diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/to_dense_vector.json b/docs/reference/query-languages/esql/kibana/definition/functions/to_dense_vector.json index 932937bf10c6c..56949532a872c 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/to_dense_vector.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/to_dense_vector.json @@ -3,10 +3,59 @@ "type" : "scalar", "name" : "to_dense_vector", "description" : "Converts a multi-valued input of numbers, or a hexadecimal string, to a dense_vector.", - "signatures" : [ ], + "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "double", + "optional" : false, + "description" : "multi-valued input of numbers or hexadecimal string to convert." + } + ], + "variadic" : false, + "returnType" : "dense_vector" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "multi-valued input of numbers or hexadecimal string to convert." + } + ], + "variadic" : false, + "returnType" : "dense_vector" + }, + { + "params" : [ + { + "name" : "field", + "type" : "keyword", + "optional" : false, + "description" : "multi-valued input of numbers or hexadecimal string to convert." + } + ], + "variadic" : false, + "returnType" : "dense_vector" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "multi-valued input of numbers or hexadecimal string to convert." + } + ], + "variadic" : false, + "returnType" : "dense_vector" + } + ], "examples" : [ "row ints = [1, 2, 3]\n| eval vector = to_dense_vector(ints)\n| keep vector" ], - "preview" : false, - "snapshot_only" : true + "preview" : true, + "snapshot_only" : false } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index be96a95d6710d..a1d62e4d7328d 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -349,8 +349,7 @@ public enum DataType implements Writeable { * */ public static final Map UNDER_CONSTRUCTION = Map.ofEntries( - Map.entry(AGGREGATE_METRIC_DOUBLE, EsqlCorePlugin.AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG), - Map.entry(DENSE_VECTOR, EsqlCorePlugin.DENSE_VECTOR_FEATURE_FLAG) + Map.entry(AGGREGATE_METRIC_DOUBLE, EsqlCorePlugin.AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG) ); private final String typeName; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java index 3014e1d4ba48a..bb819eaa6ed5e 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java @@ -19,7 +19,6 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.junit.Before; import java.io.IOException; @@ -197,8 +196,6 @@ public void testNonIndexedDenseVectorField() throws IOException { @Before public void setup() throws IOException { - assumeTrue("Dense vector type is disabled", EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE.isEnabled()); - createIndexWithDenseVector("test"); int numDims = randomIntBetween(32, 64) * 2; // min 64, even number diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/vector/VectorSimilarityFunctionsIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/vector/VectorSimilarityFunctionsIT.java index e886e2baf06c4..f0fc8955f0b39 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/vector/VectorSimilarityFunctionsIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/vector/VectorSimilarityFunctionsIT.java @@ -192,8 +192,6 @@ private static float[] readVector(List leftVector) { @Before public void setup() throws IOException { - assumeTrue("Dense vector type is disabled", EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE.isEnabled()); - createIndexWithDenseVector("test"); numDims = randomIntBetween(32, 64) * 2; // min 64, even number diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 3628eceb4593c..0ef4feb0b6fa9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1286,7 +1286,7 @@ public enum Cap { /** * Dense vector field type support */ - DENSE_VECTOR_FIELD_TYPE(EsqlCorePlugin.DENSE_VECTOR_FEATURE_FLAG), + DENSE_VECTOR_FIELD_TYPE, /** * Enable support for index aliases in lookup joins @@ -1516,7 +1516,7 @@ public enum Cap { /** * TO_DENSE_VECTOR function. */ - TO_DENSE_VECTOR_FUNCTION(Build.current().isSnapshot()), + TO_DENSE_VECTOR_FUNCTION, /** * Support present_over_time aggregation that gets evaluated per time-series diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java index 9a8824b16b1d7..ed7e8ebb57003 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java @@ -10,7 +10,6 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder; import org.elasticsearch.plugins.SearchPlugin; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.ExpressionCoreWritables; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateWritables; @@ -216,9 +215,7 @@ public static List unaryScalars() { entries.add(ToDatetime.ENTRY); entries.add(ToDateNanos.ENTRY); entries.add(ToDegrees.ENTRY); - if (EsqlCapabilities.Cap.TO_DENSE_VECTOR_FUNCTION.isEnabled()) { - entries.add(ToDenseVector.ENTRY); - } + entries.add(ToDenseVector.ENTRY); entries.add(ToDouble.ENTRY); entries.add(ToGeoShape.ENTRY); entries.add(ToCartesianShape.ENTRY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 45e7849b6a603..a648c2f77d9c6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -466,6 +466,7 @@ private static FunctionDefinition[][] functions() { def(ToDatetime.class, ToDatetime::new, "to_datetime", "to_dt"), def(ToDateNanos.class, ToDateNanos::new, "to_date_nanos", "to_datenanos"), def(ToDegrees.class, ToDegrees::new, "to_degrees"), + def(ToDenseVector.class, ToDenseVector::new, "to_dense_vector"), def(ToDouble.class, ToDouble::new, "to_double", "to_dbl"), def(ToGeohash.class, ToGeohash::new, "to_geohash"), def(ToGeotile.class, ToGeotile::new, "to_geotile"), @@ -539,7 +540,6 @@ private static FunctionDefinition[][] snapshotFunctions() { def(Last.class, bi(Last::new), "last"), def(Score.class, uni(Score::new), Score.NAME), def(Term.class, bi(Term::new), "term"), - def(ToDenseVector.class, ToDenseVector::new, "to_dense_vector"), def(Knn.class, tri(Knn::new), "knn"), def(CosineSimilarity.class, CosineSimilarity::new, "v_cosine"), def(DotProduct.class, DotProduct::new, "v_dot_product"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDenseVector.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDenseVector.java index f70c0a59b2ece..168a08e8b835e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDenseVector.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDenseVector.java @@ -49,6 +49,7 @@ public class ToDenseVector extends AbstractConvertFunction { @FunctionInfo( returnType = "dense_vector", description = "Converts a multi-valued input of numbers, or a hexadecimal string, to a dense_vector.", + preview = true, examples = @Example(file = "dense_vector", tag = "to_dense_vector-ints") ) public ToDenseVector( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java index 3997edb8d6872..ba7f3121f101b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -28,7 +28,6 @@ import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.InvalidArgumentException; import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -159,10 +158,7 @@ public class EsqlDataTypeConverter { typeToConverter.put(VERSION, ToVersion::new); typeToConverter.put(DATE_PERIOD, ToDatePeriod::new); typeToConverter.put(TIME_DURATION, ToTimeDuration::new); - - if (EsqlCapabilities.Cap.TO_DENSE_VECTOR_FUNCTION.isEnabled()) { - typeToConverter.put(DENSE_VECTOR, ToDenseVector::new); - } + typeToConverter.put(DENSE_VECTOR, ToDenseVector::new); TYPE_TO_CONVERTER_FUNCTION = Collections.unmodifiableMap(typeToConverter); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index a1c348000dc5b..766d30fd578dd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -2357,9 +2357,7 @@ public void testImplicitCasting() { } public void testDenseVectorImplicitCastingKnn() { - assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE.isEnabled()); assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - assumeTrue("dense vector casting must be enabled", EsqlCapabilities.Cap.TO_DENSE_VECTOR_FUNCTION.isEnabled()); if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { checkDenseVectorCastingHexKnn("float_vector"); @@ -2436,8 +2434,6 @@ private void checkDenseVectorCastingKnnQueryParams(String fieldName) { } public void testDenseVectorImplicitCastingSimilarityFunctions() { - assumeTrue("dense vector casting must be enabled", EsqlCapabilities.Cap.TO_DENSE_VECTOR_FUNCTION.isEnabled()); - if (EsqlCapabilities.Cap.COSINE_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { checkDenseVectorImplicitCastingSimilarityFunction( "v_cosine(float_vector, [0.342, 0.164, 0.234])", @@ -2500,8 +2496,6 @@ private void checkDenseVectorImplicitCastingSimilarityFunction(String similarity } public void testDenseVectorEvalCastingSimilarityFunctions() { - assumeTrue("dense vector casting must be enabled", EsqlCapabilities.Cap.TO_DENSE_VECTOR_FUNCTION.isEnabled()); - if (EsqlCapabilities.Cap.COSINE_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { checkDenseVectorEvalCastingSimilarityFunction("v_cosine(float_vector, query)"); checkDenseVectorEvalCastingSimilarityFunction("v_cosine(byte_vector, query)"); @@ -2542,8 +2536,6 @@ private void checkDenseVectorEvalCastingSimilarityFunction(String similarityFunc } public void testVectorFunctionHexImplicitCastingError() { - assumeTrue("dense vector casting must be enabled", EsqlCapabilities.Cap.TO_DENSE_VECTOR_FUNCTION.isEnabled()); - if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { checkVectorFunctionHexImplicitCastingError("where knn(float_vector, \"notcorrect\")"); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDenseVectorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDenseVectorTests.java index e4e153d25bf8f..19b1b8bf264ae 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDenseVectorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDenseVectorTests.java @@ -12,13 +12,11 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; -import org.junit.BeforeClass; import java.util.ArrayList; import java.util.Arrays; @@ -31,11 +29,6 @@ public class ToDenseVectorTests extends AbstractScalarFunctionTestCase { - @BeforeClass - public static void checkCapability() { - assumeTrue("To_DenseVector function capability", EsqlCapabilities.Cap.TO_DENSE_VECTOR_FUNCTION.isEnabled()); - } - public ToDenseVectorTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index 3a915bb937ce4..d53c370a572d5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -1384,7 +1384,6 @@ public void testMultiMatchOptionsPushDown() { } public void testKnnOptionsPushDown() { - assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE.isEnabled()); assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ @@ -1411,7 +1410,6 @@ public void testKnnOptionsPushDown() { } public void testKnnUsesLimitForK() { - assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE.isEnabled()); assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ @@ -1430,7 +1428,6 @@ public void testKnnUsesLimitForK() { } public void testKnnKAndMinCandidatesLowerK() { - assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE.isEnabled()); assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ @@ -1449,7 +1446,6 @@ public void testKnnKAndMinCandidatesLowerK() { } public void testKnnKAndMinCandidatesHigherK() { - assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE.isEnabled()); assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ From 3df6ea37187ecdcc8e14d1d6029d26e46c9a4be6 Mon Sep 17 00:00:00 2001 From: cdelgado Date: Mon, 29 Sep 2025 18:50:52 +0200 Subject: [PATCH 02/35] Remove capability usage and feature flag --- .../esql/core/expression/TypeResolutions.java | 14 +++++++++++--- .../xpack/esql/core/plugin/EsqlCorePlugin.java | 1 - .../xpack/esql/action/EsqlCapabilities.java | 5 ++--- .../xpack/esql/analysis/AnalyzerTests.java | 7 ++----- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java index c0ca5d18dc63d..aaf95cf824524 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java @@ -20,8 +20,10 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DENSE_VECTOR; import static org.elasticsearch.xpack.esql.core.type.DataType.IP; import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; +import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable; import static org.elasticsearch.xpack.esql.core.type.DataType.isSpatialOrGrid; public final class TypeResolutions { @@ -76,16 +78,22 @@ public static TypeResolution isDate(Expression e, String operationName, ParamOrd * @see DataType#isRepresentable(DataType) */ public static TypeResolution isRepresentableExceptCounters(Expression e, String operationName, ParamOrdinal paramOrd) { - return isType(e, DataType::isRepresentable, operationName, paramOrd, "any type except counter types"); + return isType( + e, + dt -> isRepresentable(dt) && dt != DENSE_VECTOR, + operationName, + paramOrd, + "any type except counter types or dense_vector" + ); } public static TypeResolution isRepresentableExceptCountersAndSpatial(Expression e, String operationName, ParamOrdinal paramOrd) { return isType( e, - (t) -> isSpatialOrGrid(t) == false && DataType.isRepresentable(t), + (t) -> isSpatialOrGrid(t) == false && DataType.isRepresentable(t) && t != DENSE_VECTOR, operationName, paramOrd, - "any type except counter and spatial types" + "any type except counter, spatial types or dense_vector" ); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java index b9f55f7985e84..4bd722992f524 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java @@ -14,5 +14,4 @@ public class EsqlCorePlugin extends Plugin implements ExtensiblePlugin { public static final FeatureFlag AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG = new FeatureFlag("esql_aggregate_metric_double"); - public static final FeatureFlag DENSE_VECTOR_FEATURE_FLAG = new FeatureFlag("esql_dense_vector"); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 0ef4feb0b6fa9..b5d12c5163bac 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -12,7 +12,6 @@ import org.elasticsearch.compute.lucene.read.ValuesSourceReaderOperator; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.rest.action.admin.cluster.RestNodesCapabilitiesAction; -import org.elasticsearch.xpack.esql.core.plugin.EsqlCorePlugin; import org.elasticsearch.xpack.esql.plugin.EsqlFeatures; import java.util.ArrayList; @@ -1443,12 +1442,12 @@ public enum Cap { /** * Byte elements dense vector field type support. */ - DENSE_VECTOR_FIELD_TYPE_BYTE_ELEMENTS(EsqlCorePlugin.DENSE_VECTOR_FEATURE_FLAG), + DENSE_VECTOR_FIELD_TYPE_BYTE_ELEMENTS, /** * Bit elements dense vector field type support. */ - DENSE_VECTOR_FIELD_TYPE_BIT_ELEMENTS(EsqlCorePlugin.DENSE_VECTOR_FEATURE_FLAG), + DENSE_VECTOR_FIELD_TYPE_BIT_ELEMENTS, /** * Support null elements on vector similarity functions diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 766d30fd578dd..daa0ebd5741f5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -135,7 +135,6 @@ import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.randomInferenceIdOtherThan; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.tsdbIndexResolution; -import static org.elasticsearch.xpack.esql.core.plugin.EsqlCorePlugin.DENSE_VECTOR_FEATURE_FLAG; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; import static org.elasticsearch.xpack.esql.core.type.DataType.AGGREGATE_METRIC_DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; @@ -3339,7 +3338,7 @@ public void testResolveDenseVector() { IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, true, true)); var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); assertThat(plan.output(), hasSize(1)); - assertThat(plan.output().getFirst().dataType(), equalTo(DENSE_VECTOR_FEATURE_FLAG.isEnabled() ? DENSE_VECTOR : UNSUPPORTED)); + assertThat(plan.output().getFirst().dataType(), equalTo(DENSE_VECTOR)); } { IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, true, false)); @@ -4149,9 +4148,7 @@ public void testRerankInvalidQueryTypes() { } public void testRerankFieldsInvalidTypes() { - List invalidFieldNames = DENSE_VECTOR_FEATURE_FLAG.isEnabled() - ? List.of("date", "date_nanos", "ip", "version", "dense_vector") - : List.of("date", "date_nanos", "ip", "version"); + List invalidFieldNames = List.of("date", "date_nanos", "ip", "version", "dense_vector"); for (String fieldName : invalidFieldNames) { LogManager.getLogger(AnalyzerTests.class).warn("[{}]", fieldName); From 48221c2f663f1d31ea4d7d7daf5b9dd2bec38fe1 Mon Sep 17 00:00:00 2001 From: cdelgado Date: Mon, 29 Sep 2025 18:51:22 +0200 Subject: [PATCH 03/35] Fix multivalued tests --- .../scalar/multivalue/MvAppendErrorTests.java | 26 ++++++++++++------- .../multivalue/MvContainsErrorTests.java | 26 ++++++++++++------- .../scalar/multivalue/MvCountErrorTests.java | 14 ++-------- .../scalar/multivalue/MvDedupeErrorTests.java | 10 ++----- .../scalar/multivalue/MvFirstErrorTests.java | 15 ++--------- .../scalar/multivalue/MvLastErrorTests.java | 14 ++-------- .../scalar/multivalue/MvMaxErrorTests.java | 4 ++- .../scalar/multivalue/MvMinErrorTests.java | 4 ++- .../scalar/multivalue/MvSliceErrorTests.java | 2 +- 9 files changed, 49 insertions(+), 66 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendErrorTests.java index df9ab4764c879..7cddacf1aec9c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendErrorTests.java @@ -17,6 +17,7 @@ import java.util.List; import java.util.Set; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class MvAppendErrorTests extends ErrorsForCasesWithoutExamplesTestCase { @@ -32,14 +33,21 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo( - "second argument of [" - + sourceForSignature(signature) - + "] must be [" - + signature.get(0).noText().typeName() - + "], found value [] type [" - + signature.get(1).typeName() - + "]" - ); + if (signature.getFirst() == DataType.DENSE_VECTOR + || signature.getFirst() == DataType.NULL && signature.get(1) == DataType.DENSE_VECTOR) { + return containsString( + typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter types or dense_vector") + ); + } else { + return equalTo( + "second argument of [" + + sourceForSignature(signature) + + "] must be [" + + signature.get(0).noText().typeName() + + "], found value [] type [" + + signature.get(1).typeName() + + "]" + ); + } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsErrorTests.java index c64cb8eef8d6f..0549a72a10131 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsErrorTests.java @@ -17,6 +17,7 @@ import java.util.List; import java.util.Set; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class MvContainsErrorTests extends ErrorsForCasesWithoutExamplesTestCase { @@ -32,14 +33,21 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo( - "second argument of [" - + sourceForSignature(signature) - + "] must be [" - + signature.get(0).noText().typeName() - + "], found value [] type [" - + signature.get(1).typeName() - + "]" - ); + if (signature.getFirst() == DataType.DENSE_VECTOR + || signature.getFirst() == DataType.NULL && signature.get(1) == DataType.DENSE_VECTOR) { + return containsString( + typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter types or dense_vector") + ); + } else { + return equalTo( + "second argument of [" + + sourceForSignature(signature) + + "] must be [" + + signature.get(0).noText().typeName() + + "], found value [] type [" + + signature.get(1).typeName() + + "]" + ); + } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountErrorTests.java index d59a1aa2eb098..0634781d5492f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountErrorTests.java @@ -32,22 +32,12 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> { - /* - * In general MvCount should support all signatures. While building a - * new type you may we to temporarily remove this. - */ - throw new UnsupportedOperationException("all signatures should be supported"); - })); + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter types or dense_vector")); } @Override protected void assertNumberOfCheckedSignatures(int checked) { - /* - * In general MvCount should support all signatures. While building a - * new type you may we to temporarily relax this. - */ - assertThat("all signatures should be supported", checked, equalTo(0)); + assertThat("all signatures except dense_vector should be supported", checked, equalTo(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeErrorTests.java index 55f34d9a72f41..d16347efbfd77 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeErrorTests.java @@ -32,13 +32,7 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> { - /* - * In general MvDedupe should support all signatures. While building a - * new type you may we to temporarily remove this. - */ - throw new UnsupportedOperationException("all signatures should be supported"); - })); + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter types or dense_vector")); } @Override @@ -47,7 +41,7 @@ protected void assertNumberOfCheckedSignatures(int checked) { * In general MvDedupe should support all signatures. While building a * new type you may we to temporarily relax this. */ - assertThat("all signatures should be supported", checked, equalTo(0)); + assertThat("all signatures except dense_vector should be supported", checked, equalTo(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstErrorTests.java index 7ca829a7629c5..8b296ae5f32b4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstErrorTests.java @@ -32,22 +32,11 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> { - /* - * In general MvFirst should support all signatures. While building a - * new type you may we to temporarily remove this. - */ - throw new UnsupportedOperationException("all signatures should be supported"); - })); + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter types or dense_vector")); } @Override protected void assertNumberOfCheckedSignatures(int checked) { - /* - * In general MvFirst should support all signatures. While building a - * new type you may we to temporarily relax this. - */ - assertThat("all signatures should be supported", checked, equalTo(0)); + assertThat("all signatures except dense_vector should be supported", checked, equalTo(1)); } - } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastErrorTests.java index 3db13f0368a88..9e5e6de7b7534 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastErrorTests.java @@ -32,21 +32,11 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> { - /* - * In general MvLast should support all signatures. While building a - * new type you may we to temporarily remove this. - */ - throw new UnsupportedOperationException("all signatures should be supported"); - })); + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter types or dense_vector")); } @Override protected void assertNumberOfCheckedSignatures(int checked) { - /* - * In general MvLast should support all signatures. While building a - * new type you may we to temporarily relax this. - */ - assertThat("all signatures should be supported", checked, equalTo(0)); + assertThat("all signatures except dense_vector should be supported", checked, equalTo(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxErrorTests.java index 6eee825de85fe..e020bd8b6562c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxErrorTests.java @@ -32,6 +32,8 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter and spatial types")); + return equalTo( + typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter, spatial types or dense_vector") + ); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinErrorTests.java index b9d97d7bb4cb8..4751abcd00891 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinErrorTests.java @@ -32,6 +32,8 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter and spatial types")); + return equalTo( + typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter, spatial types or dense_vector") + ); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceErrorTests.java index 83d0e4fcf3d75..d0daf48fd9a12 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceErrorTests.java @@ -34,7 +34,7 @@ protected Expression build(Source source, List args) { protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { case 1, 2 -> "integer"; - default -> throw new UnsupportedOperationException(); + default -> "any type except counter types or dense_vector"; })); } } From 11f2cac244138e2828bcf21b0c33c2fdef2ffe60 Mon Sep 17 00:00:00 2001 From: cdelgado Date: Mon, 29 Sep 2025 19:58:45 +0200 Subject: [PATCH 04/35] Fix aggregation and comparisons with dense_vectors --- .../esql/expression/function/aggregate/Absent.java | 8 +++++++- .../esql/expression/function/aggregate/Count.java | 8 +++++++- .../esql/expression/function/aggregate/Present.java | 8 +++++++- .../function/aggregate/AbsentErrorTests.java | 7 ++++--- .../function/aggregate/CountErrorTests.java | 7 ++++--- .../function/aggregate/PresentErrorTests.java | 7 ++++--- .../function/aggregate/SampleErrorTests.java | 12 ++++++++++++ .../function/aggregate/ValuesErrorTests.java | 7 ++++--- .../operator/comparison/EqualsErrorTests.java | 4 ++-- .../operator/comparison/NotEqualsErrorTests.java | 4 ++-- 10 files changed, 53 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Absent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Absent.java index f2661e764a9e0..7337d55b8f7df 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Absent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Absent.java @@ -127,7 +127,13 @@ public Nullability nullable() { @Override protected TypeResolution resolveType() { - return isType(field(), dt -> dt.isCounter() == false, sourceText(), DEFAULT, "any type except counter types"); + return isType( + field(), + dt -> dt.isCounter() == false && dt != DataType.DENSE_VECTOR, + sourceText(), + DEFAULT, + "any type except counter types or dense_vector" + ); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java index 15810e151e623..5ee5f371ca0fa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java @@ -144,7 +144,13 @@ public Nullability nullable() { @Override protected TypeResolution resolveType() { - return isType(field(), dt -> dt.isCounter() == false, sourceText(), DEFAULT, "any type except counter types"); + return isType( + field(), + dt -> dt.isCounter() == false && dt != DataType.DENSE_VECTOR, + sourceText(), + DEFAULT, + "any type except counter types or dense_vector" + ); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Present.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Present.java index 345d2cde7ef15..c65601211b544 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Present.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Present.java @@ -133,6 +133,12 @@ public Nullability nullable() { @Override protected TypeResolution resolveType() { - return isType(field(), dt -> dt.isCounter() == false, sourceText(), DEFAULT, "any type except counter types"); + return isType( + field(), + dt -> dt.isCounter() == false && dt != DataType.DENSE_VECTOR, + sourceText(), + DEFAULT, + "any type except counter types or dense_vector" + ); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AbsentErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AbsentErrorTests.java index 42920212a61bd..0e19d0c49953d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AbsentErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AbsentErrorTests.java @@ -32,12 +32,13 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - assert false : "All checked types must work"; - return null; + return equalTo( + "argument of [dense_vector] must be [any type except counter types or dense_vector], found value [] type [dense_vector]" + ); } @Override protected void assertNumberOfCheckedSignatures(int checked) { - assertThat(checked, equalTo(0)); + assertThat(checked, equalTo(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountErrorTests.java index e634994572680..c66c4aa9aaeec 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountErrorTests.java @@ -32,12 +32,13 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - assert false : "All checked types must work"; - return null; + return equalTo( + "argument of [dense_vector] must be [any type except counter types or dense_vector], found value [] type [dense_vector]" + ); } @Override protected void assertNumberOfCheckedSignatures(int checked) { - assertThat(checked, equalTo(0)); + assertThat(checked, equalTo(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PresentErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PresentErrorTests.java index b394fa803f7ce..0be345a9799e3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PresentErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PresentErrorTests.java @@ -32,12 +32,13 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - assert false : "All checked types must work"; - return null; + return equalTo( + "argument of [dense_vector] must be [any type except counter types or dense_vector], found value [] type [dense_vector]" + ); } @Override protected void assertNumberOfCheckedSignatures(int checked) { - assertThat(checked, equalTo(0)); + assertThat(checked, equalTo(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SampleErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SampleErrorTests.java index a343faa50f1ad..2b6c955020312 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SampleErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SampleErrorTests.java @@ -17,6 +17,7 @@ import java.util.List; import java.util.Set; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class SampleErrorTests extends ErrorsForCasesWithoutExamplesTestCase { @@ -32,6 +33,17 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + if (signature.getFirst() == DataType.DENSE_VECTOR) { + return equalTo( + "first argument of [" + + sourceForSignature(signature) + + "] must be [any type except counter types" + + " or dense_vector], found value [] type [dense_vector]" + ); + } + if (signature.getFirst() == DataType.NULL && signature.get(1) == DataType.DENSE_VECTOR) { + return containsString(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "integer")); + } if (signature.get(1).equals(DataType.NULL)) { return equalTo("second argument of [" + sourceForSignature(signature) + "] cannot be null, received []"); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesErrorTests.java index 1467aa129b467..74afcd0482407 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesErrorTests.java @@ -32,12 +32,13 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - assert false : "All checked types must work"; - return null; + return equalTo( + "argument of [dense_vector] must be [any type except counter types or dense_vector], found value [] type [dense_vector]" + ); } @Override protected void assertNumberOfCheckedSignatures(int checked) { - assertThat(checked, equalTo(0)); + assertThat(checked, equalTo(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsErrorTests.java index cecb6a2987fd4..16eae27a35d6b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsErrorTests.java @@ -36,6 +36,6 @@ protected Matcher expectedTypeErrorMatcher(List> validPerP } private static final String TYPE_ERROR = - "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long," - + " semantic_text, text, unsigned_long or version"; + "boolean, cartesian_point, cartesian_shape, date_nanos, datetime, double, geo_point, geo_shape, geohash, geohex, geotile, " + + "integer, ip, keyword, long, text, unsigned_long or version"; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsErrorTests.java index ed0a477ec613c..a748948783e5d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsErrorTests.java @@ -36,6 +36,6 @@ protected Matcher expectedTypeErrorMatcher(List> validPerP } private static final String TYPE_ERROR = - "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long, text, " - + "unsigned_long or version"; + "boolean, cartesian_point, cartesian_shape, date_nanos, datetime, double, geo_point, geo_shape, geohash, geohex, geotile, " + + "integer, ip, keyword, long, text, unsigned_long or version"; } From 8a103bfc3963a7b86a5405ae718c9f4bb2f6308b Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 29 Sep 2025 18:16:39 +0000 Subject: [PATCH 05/35] [CI] Auto commit changes from spotless --- .../xpack/esql/core/expression/TypeResolutions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java index aaf95cf824524..07c9bf32db22e 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java @@ -90,7 +90,7 @@ public static TypeResolution isRepresentableExceptCounters(Expression e, String public static TypeResolution isRepresentableExceptCountersAndSpatial(Expression e, String operationName, ParamOrdinal paramOrd) { return isType( e, - (t) -> isSpatialOrGrid(t) == false && DataType.isRepresentable(t) && t != DENSE_VECTOR, + (t) -> isSpatialOrGrid(t) == false && DataType.isRepresentable(t) && t != DENSE_VECTOR, operationName, paramOrd, "any type except counter, spatial types or dense_vector" From bea08d7cff50f06039dc738fda80d4cf3a07ae45 Mon Sep 17 00:00:00 2001 From: cdelgado Date: Mon, 29 Sep 2025 20:51:04 +0200 Subject: [PATCH 06/35] Added to_dense_vector docs --- .../esql/_snippets/functions/types/to_dense_vector.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 docs/reference/query-languages/esql/_snippets/functions/types/to_dense_vector.md diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/to_dense_vector.md b/docs/reference/query-languages/esql/_snippets/functions/types/to_dense_vector.md new file mode 100644 index 0000000000000..6a7d025f6a43d --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/to_dense_vector.md @@ -0,0 +1,11 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| field | result | +| --- | --- | +| double | dense_vector | +| integer | dense_vector | +| keyword | dense_vector | +| long | dense_vector | + From 972ad3ef330f3d73dbdcd542a3dec4503ef88664 Mon Sep 17 00:00:00 2001 From: cdelgado Date: Mon, 29 Sep 2025 21:10:55 +0200 Subject: [PATCH 07/35] Add docs for vector similarity functions --- .../_snippets/functions/types/v_cosine.md | 8 +++++++ .../functions/types/v_dot_product.md | 8 +++++++ .../_snippets/functions/types/v_hamming.md | 8 +++++++ .../_snippets/functions/types/v_l1_norm.md | 8 +++++++ .../_snippets/functions/types/v_l2_norm.md | 8 +++++++ .../_snippets/functions/types/v_magnitude.md | 8 +++++++ .../kibana/definition/functions/v_cosine.json | 21 ++++++++++++++++++- .../definition/functions/v_dot_product.json | 21 ++++++++++++++++++- .../definition/functions/v_hamming.json | 21 ++++++++++++++++++- .../definition/functions/v_l1_norm.json | 21 ++++++++++++++++++- .../definition/functions/v_l2_norm.json | 21 ++++++++++++++++++- .../definition/functions/v_magnitude.json | 15 ++++++++++++- 12 files changed, 162 insertions(+), 6 deletions(-) create mode 100644 docs/reference/query-languages/esql/_snippets/functions/types/v_cosine.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/types/v_dot_product.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/types/v_hamming.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/types/v_l1_norm.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/types/v_l2_norm.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/types/v_magnitude.md diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/v_cosine.md b/docs/reference/query-languages/esql/_snippets/functions/types/v_cosine.md new file mode 100644 index 0000000000000..4d9804aae2395 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/v_cosine.md @@ -0,0 +1,8 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| left | right | result | +| --- | --- | --- | +| dense_vector | dense_vector | double | + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/v_dot_product.md b/docs/reference/query-languages/esql/_snippets/functions/types/v_dot_product.md new file mode 100644 index 0000000000000..4d9804aae2395 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/v_dot_product.md @@ -0,0 +1,8 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| left | right | result | +| --- | --- | --- | +| dense_vector | dense_vector | double | + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/v_hamming.md b/docs/reference/query-languages/esql/_snippets/functions/types/v_hamming.md new file mode 100644 index 0000000000000..4d9804aae2395 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/v_hamming.md @@ -0,0 +1,8 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| left | right | result | +| --- | --- | --- | +| dense_vector | dense_vector | double | + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/v_l1_norm.md b/docs/reference/query-languages/esql/_snippets/functions/types/v_l1_norm.md new file mode 100644 index 0000000000000..4d9804aae2395 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/v_l1_norm.md @@ -0,0 +1,8 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| left | right | result | +| --- | --- | --- | +| dense_vector | dense_vector | double | + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/v_l2_norm.md b/docs/reference/query-languages/esql/_snippets/functions/types/v_l2_norm.md new file mode 100644 index 0000000000000..4d9804aae2395 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/v_l2_norm.md @@ -0,0 +1,8 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| left | right | result | +| --- | --- | --- | +| dense_vector | dense_vector | double | + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/v_magnitude.md b/docs/reference/query-languages/esql/_snippets/functions/types/v_magnitude.md new file mode 100644 index 0000000000000..664cb86a5d11e --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/v_magnitude.md @@ -0,0 +1,8 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| input | result | +| --- | --- | +| dense_vector | double | + diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/v_cosine.json b/docs/reference/query-languages/esql/kibana/definition/functions/v_cosine.json index f3b3df1d88c6a..aac52a2385ce3 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/v_cosine.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/v_cosine.json @@ -3,7 +3,26 @@ "type" : "scalar", "name" : "v_cosine", "description" : "Calculates the cosine similarity between two dense_vectors.", - "signatures" : [ ], + "signatures" : [ + { + "params" : [ + { + "name" : "left", + "type" : "dense_vector", + "optional" : false, + "description" : "first dense_vector to calculate cosine similarity" + }, + { + "name" : "right", + "type" : "dense_vector", + "optional" : false, + "description" : "second dense_vector to calculate cosine similarity" + } + ], + "variadic" : false, + "returnType" : "double" + } + ], "examples" : [ " from colors\n | where color != \"black\"\n | eval similarity = v_cosine(rgb_vector, [0, 255, 255])\n | sort similarity desc, color asc" ], diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/v_dot_product.json b/docs/reference/query-languages/esql/kibana/definition/functions/v_dot_product.json index 05b8f0aa1c3f7..a00eaf6dc8387 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/v_dot_product.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/v_dot_product.json @@ -3,7 +3,26 @@ "type" : "scalar", "name" : "v_dot_product", "description" : "Calculates the dot product between two dense_vectors.", - "signatures" : [ ], + "signatures" : [ + { + "params" : [ + { + "name" : "left", + "type" : "dense_vector", + "optional" : false, + "description" : "first dense_vector to calculate dot product similarity" + }, + { + "name" : "right", + "type" : "dense_vector", + "optional" : false, + "description" : "second dense_vector to calculate dot product similarity" + } + ], + "variadic" : false, + "returnType" : "double" + } + ], "examples" : [ " from colors\n | eval similarity = v_dot_product(rgb_vector, [0, 255, 255])\n | sort similarity desc, color asc" ], diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/v_hamming.json b/docs/reference/query-languages/esql/kibana/definition/functions/v_hamming.json index 51e3660ae8650..6d9885918f5df 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/v_hamming.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/v_hamming.json @@ -3,7 +3,26 @@ "type" : "scalar", "name" : "v_hamming", "description" : "Calculates the Hamming distance between two dense vectors.", - "signatures" : [ ], + "signatures" : [ + { + "params" : [ + { + "name" : "left", + "type" : "dense_vector", + "optional" : false, + "description" : "First dense_vector to use to calculate the Hamming distance" + }, + { + "name" : "right", + "type" : "dense_vector", + "optional" : false, + "description" : "Second dense_vector to use to calculate the Hamming distance" + } + ], + "variadic" : false, + "returnType" : "double" + } + ], "examples" : [ " from colors\n | eval similarity = v_hamming(rgb_vector, [0, 255, 255])\n | sort similarity desc, color asc" ], diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/v_l1_norm.json b/docs/reference/query-languages/esql/kibana/definition/functions/v_l1_norm.json index 2dcd2ac95c980..6cdf7ba32aacd 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/v_l1_norm.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/v_l1_norm.json @@ -3,7 +3,26 @@ "type" : "scalar", "name" : "v_l1_norm", "description" : "Calculates the l1 norm between two dense_vectors.", - "signatures" : [ ], + "signatures" : [ + { + "params" : [ + { + "name" : "left", + "type" : "dense_vector", + "optional" : false, + "description" : "first dense_vector to calculate l1 norm similarity" + }, + { + "name" : "right", + "type" : "dense_vector", + "optional" : false, + "description" : "second dense_vector to calculate l1 norm similarity" + } + ], + "variadic" : false, + "returnType" : "double" + } + ], "examples" : [ " from colors\n | eval similarity = v_l1_norm(rgb_vector, [0, 255, 255])\n | sort similarity desc, color asc" ], diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/v_l2_norm.json b/docs/reference/query-languages/esql/kibana/definition/functions/v_l2_norm.json index 573de7891d893..344d0c1746456 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/v_l2_norm.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/v_l2_norm.json @@ -3,7 +3,26 @@ "type" : "scalar", "name" : "v_l2_norm", "description" : "Calculates the l2 norm between two dense_vectors.", - "signatures" : [ ], + "signatures" : [ + { + "params" : [ + { + "name" : "left", + "type" : "dense_vector", + "optional" : false, + "description" : "first dense_vector to calculate l2 norm similarity" + }, + { + "name" : "right", + "type" : "dense_vector", + "optional" : false, + "description" : "second dense_vector to calculate l2 norm similarity" + } + ], + "variadic" : false, + "returnType" : "double" + } + ], "examples" : [ " from colors\n | eval similarity = v_l2_norm(rgb_vector, [0, 255, 255])\n | sort similarity desc, color asc" ], diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/v_magnitude.json b/docs/reference/query-languages/esql/kibana/definition/functions/v_magnitude.json index 2835d403e656e..1f69905176ed8 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/v_magnitude.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/v_magnitude.json @@ -3,7 +3,20 @@ "type" : "scalar", "name" : "v_magnitude", "description" : "Calculates the magnitude of a dense_vector.", - "signatures" : [ ], + "signatures" : [ + { + "params" : [ + { + "name" : "input", + "type" : "dense_vector", + "optional" : false, + "description" : "dense_vector for which to compute the magnitude" + } + ], + "variadic" : false, + "returnType" : "double" + } + ], "examples" : [ " from colors\n | eval magnitude = v_magnitude(rgb_vector)\n | sort magnitude desc, color asc" ], From 3383f1af99cbe6e253a6f6e9d9714fda6e4655b0 Mon Sep 17 00:00:00 2001 From: cdelgado Date: Mon, 29 Sep 2025 22:46:59 +0200 Subject: [PATCH 08/35] Add docs --- .../_snippets/operators/types/is_not_null.md | 1 + .../esql/kibana/definition/functions/knn.json | 53 ++++++++++++++++++- .../definition/operators/is_not_null.json | 12 +++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/docs/reference/query-languages/esql/_snippets/operators/types/is_not_null.md b/docs/reference/query-languages/esql/_snippets/operators/types/is_not_null.md index 0a1179d8b1218..333818f6d05d0 100644 --- a/docs/reference/query-languages/esql/_snippets/operators/types/is_not_null.md +++ b/docs/reference/query-languages/esql/_snippets/operators/types/is_not_null.md @@ -12,6 +12,7 @@ | counter_long | boolean | | date | boolean | | date_nanos | boolean | +| dense_vector | boolean | | double | boolean | | geo_point | boolean | | geo_shape | boolean | diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/knn.json b/docs/reference/query-languages/esql/kibana/definition/functions/knn.json index cd5541f8e7bfe..33845ebd490ce 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/knn.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/knn.json @@ -3,7 +3,58 @@ "type" : "scalar", "name" : "knn", "description" : "Finds the k nearest vectors to a query vector, as measured by a similarity metric. knn function finds nearest vectors through approximate search on indexed dense_vectors or semantic_text fields.", - "signatures" : [ ], + "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "dense_vector", + "optional" : false, + "description" : "Field that the query will target. knn function can be used with dense_vector or semantic_text fields. Other text fields are not allowed" + }, + { + "name" : "query", + "type" : "dense_vector", + "optional" : false, + "description" : "Vector value to find top nearest neighbours for." + }, + { + "name" : "options", + "type" : "function_named_parameters", + "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.Defaults to 1.0.'}, {name='min_candidates', values=[10], description='The minimum number of nearest neighbor candidates to consider per shard while doing knn search. KNN may use a higher number of candidates in case the query can't use a approximate results. Cannot exceed 10,000. Increasing min_candidates tends to improve the accuracy of the final results. Defaults to 1.5 * LIMIT used for the query.'}, {name='rescore_oversample', values=[3.5], description='Applies the specified oversampling for rescoring quantized vectors. See [oversampling and rescoring quantized vectors](docs-content://solutions/search/vector/knn.md#dense-vector-knn-search-rescoring) for details.'}, {name='similarity', values=[0.01], description='The minimum similarity required for a document to be considered a match. The similarity value calculated relates to the raw similarity used, not the document score.'}", + "optional" : true, + "description" : "(Optional) kNN additional options as <>. See <> for more information." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "text", + "optional" : false, + "description" : "Field that the query will target. knn function can be used with dense_vector or semantic_text fields. Other text fields are not allowed" + }, + { + "name" : "query", + "type" : "dense_vector", + "optional" : false, + "description" : "Vector value to find top nearest neighbours for." + }, + { + "name" : "options", + "type" : "function_named_parameters", + "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.Defaults to 1.0.'}, {name='min_candidates', values=[10], description='The minimum number of nearest neighbor candidates to consider per shard while doing knn search. KNN may use a higher number of candidates in case the query can't use a approximate results. Cannot exceed 10,000. Increasing min_candidates tends to improve the accuracy of the final results. Defaults to 1.5 * LIMIT used for the query.'}, {name='rescore_oversample', values=[3.5], description='Applies the specified oversampling for rescoring quantized vectors. See [oversampling and rescoring quantized vectors](docs-content://solutions/search/vector/knn.md#dense-vector-knn-search-rescoring) for details.'}, {name='similarity', values=[0.01], description='The minimum similarity required for a document to be considered a match. The similarity value calculated relates to the raw similarity used, not the document score.'}", + "optional" : true, + "description" : "(Optional) kNN additional options as <>. See <> for more information." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ], "examples" : [ "from colors metadata _score\n| where knn(rgb_vector, [0, 120, 0])\n| sort _score desc, color asc" ], diff --git a/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json b/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json index c3cc8fae2a14e..2243d4979ba69 100644 --- a/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json +++ b/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json @@ -102,6 +102,18 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "field", + "type" : "dense_vector", + "optional" : false, + "description" : "Value to check. It can be a single- or multi-valued column or an expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { From e4b791c9ea85588d16591c22ae56fc90c52125df Mon Sep 17 00:00:00 2001 From: cdelgado Date: Mon, 29 Sep 2025 22:47:36 +0200 Subject: [PATCH 09/35] dense_vector for Case and IsNotNull --- .../esql/expression/function/scalar/conditional/Case.java | 2 ++ .../xpack/esql/expression/predicate/nulls/IsNotNull.java | 6 ++++-- .../expression/function/scalar/conditional/CaseTests.java | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java index 3f5cfea2e7443..32198b0f42d17 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java @@ -68,6 +68,7 @@ ConditionEvaluatorSupplier toEvaluator(ToEvaluator toEvaluator) { "cartesian_shape", "date", "date_nanos", + "dense_vector", "double", "geo_point", "geo_shape", @@ -111,6 +112,7 @@ public Case( "cartesian_shape", "date", "date_nanos", + "dense_vector", "double", "geo_point", "geo_shape", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java index 52efaa79fd846..f9a0eb058acd1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java @@ -56,7 +56,8 @@ public class IsNotNull extends UnaryScalarFunction implements EvaluatorMapper, N "unsigned_long", "counter_long", "counter_integer", - "counter_double" }, + "counter_double", + "dense_vector" }, examples = { @Example(file = "null", tag = "is-not-null") } ) public IsNotNull( @@ -75,7 +76,8 @@ public IsNotNull( "unsigned_long", "counter_long", "counter_integer", - "counter_double" } + "counter_double", + "dense_vector", } ) Expression field ) { super(source, field); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java index a538f17969bce..400dd7d09e841 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java @@ -46,6 +46,7 @@ public class CaseTests extends AbstractScalarFunctionTestCase { DataType.BOOLEAN, DataType.DATETIME, DataType.DATE_NANOS, + DataType.DENSE_VECTOR, DataType.DOUBLE, DataType.INTEGER, DataType.LONG, From 7af178d6e164f1e51e719107753a3a6d09e6cf56 Mon Sep 17 00:00:00 2001 From: cdelgado Date: Mon, 29 Sep 2025 22:48:02 +0200 Subject: [PATCH 10/35] Fix tests --- .../xpack/esql/analysis/VerifierTests.java | 2 +- .../function/fulltext/KnnTests.java | 28 +++++++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 95e2e15da7d91..305d495b21e62 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1135,7 +1135,7 @@ public void testAggregateOnCounter() { assertThat( error("FROM tests | STATS count(network.bytes_out)", tsdb), equalTo( - "1:20: argument of [count(network.bytes_out)] must be [any type except counter types]," + "1:20: argument of [count(network.bytes_out)] must be [any type except counter types or dense_vector]," + " found value [network.bytes_out] type [counter_long]" ) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KnnTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KnnTests.java index 49a6d3c904203..9871b9ce2e25e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KnnTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KnnTests.java @@ -35,6 +35,7 @@ import static org.elasticsearch.xpack.esql.SerializationTestUtils.serializeDeserialize; import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; import static org.elasticsearch.xpack.esql.core.type.DataType.DENSE_VECTOR; +import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; import static org.elasticsearch.xpack.esql.planner.TranslatorHandler.TRANSLATOR_HANDLER; import static org.hamcrest.Matchers.equalTo; @@ -60,7 +61,7 @@ private static List testCaseSuppliers() { suppliers.add( new TestCaseSupplier( - List.of(DENSE_VECTOR, DENSE_VECTOR, DataType.INTEGER), + List.of(DENSE_VECTOR, DENSE_VECTOR), () -> new TestCaseSupplier.TestCase( List.of( new TestCaseSupplier.TypedData( @@ -72,8 +73,29 @@ private static List testCaseSuppliers() { DENSE_VECTOR, "dense_vector field" ), - new TestCaseSupplier.TypedData(randomDenseVector(), DENSE_VECTOR, "query"), - new TestCaseSupplier.TypedData(randomIntBetween(1, 1000), DataType.INTEGER, "k") + new TestCaseSupplier.TypedData(randomDenseVector(), DENSE_VECTOR, "query") + ), + equalTo("KnnEvaluator" + KnnTests.class.getSimpleName()), + BOOLEAN, + equalTo(true) + ) + ) + ); + suppliers.add( + new TestCaseSupplier( + List.of(TEXT, DENSE_VECTOR), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData( + new FieldAttribute( + Source.EMPTY, + randomIdentifier(), + new EsField(randomIdentifier(), TEXT, Map.of(), false, EsField.TimeSeriesFieldType.NONE) + ), + TEXT, + "text field" + ), + new TestCaseSupplier.TypedData(randomDenseVector(), DENSE_VECTOR, "query") ), equalTo("KnnEvaluator" + KnnTests.class.getSimpleName()), BOOLEAN, From ffbdafcb8b7cdae5c9f018d0cff1c3a4b0040da8 Mon Sep 17 00:00:00 2001 From: cdelgado Date: Mon, 29 Sep 2025 22:50:07 +0200 Subject: [PATCH 11/35] Fix tests --- .../scalar/convert/ToStringErrorTests.java | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringErrorTests.java index 31088139b8911..b46cf4a5fe2ef 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringErrorTests.java @@ -32,21 +32,15 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> { - /* - * In general ToString should support all signatures. While building a - * new type you may we to temporarily remove this. - */ - throw new UnsupportedOperationException("all signatures should be supported"); - })); + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> TYPE_ERROR)); } + private static final String TYPE_ERROR = + "aggregate_metric_double or boolean or cartesian_point or cartesian_shape or date_nanos or datetime or geo_point or geo_shape " + + "or geohash or geohex or geotile or ip or numeric or string or version"; + @Override protected void assertNumberOfCheckedSignatures(int checked) { - /* - * In general ToString should support all signatures. While building a - * new type you may we to temporarily relax this. - */ - assertThat("all signatures should be supported", checked, equalTo(0)); + assertThat(checked, equalTo(1)); } } From 3a408495cdeae42a1c2e6fb06c704bc344446dcb Mon Sep 17 00:00:00 2001 From: cdelgado Date: Mon, 29 Sep 2025 22:50:16 +0200 Subject: [PATCH 12/35] Add docs --- .../esql/_snippets/functions/types/knn.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docs/reference/query-languages/esql/_snippets/functions/types/knn.md diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/knn.md b/docs/reference/query-languages/esql/_snippets/functions/types/knn.md new file mode 100644 index 0000000000000..8b7a555d81665 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/knn.md @@ -0,0 +1,9 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| field | query | options | result | +| --- | --- | --- | --- | +| dense_vector | dense_vector | named parameters | boolean | +| text | dense_vector | named parameters | boolean | + From 37caab410e709aa99e13378a7cff6b4ed540c4e0 Mon Sep 17 00:00:00 2001 From: cdelgado Date: Mon, 29 Sep 2025 23:25:03 +0200 Subject: [PATCH 13/35] Add docs --- .../esql/_snippets/functions/types/case.md | 2 + .../esql/_snippets/operators/types/is_null.md | 1 + .../kibana/definition/functions/case.json | 42 +++++++++++++++++++ .../kibana/definition/operators/is_null.json | 12 ++++++ 4 files changed, 57 insertions(+) diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/case.md b/docs/reference/query-languages/esql/_snippets/functions/types/case.md index bb20353ebe7c7..dcc18f4f73660 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/case.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/case.md @@ -14,6 +14,8 @@ | boolean | date | | date | | boolean | date_nanos | date_nanos | date_nanos | | boolean | date_nanos | | date_nanos | +| boolean | dense_vector | dense_vector | dense_vector | +| boolean | dense_vector | | dense_vector | | boolean | double | double | double | | boolean | double | | double | | boolean | geo_point | geo_point | geo_point | diff --git a/docs/reference/query-languages/esql/_snippets/operators/types/is_null.md b/docs/reference/query-languages/esql/_snippets/operators/types/is_null.md index 0a1179d8b1218..333818f6d05d0 100644 --- a/docs/reference/query-languages/esql/_snippets/operators/types/is_null.md +++ b/docs/reference/query-languages/esql/_snippets/operators/types/is_null.md @@ -12,6 +12,7 @@ | counter_long | boolean | | date | boolean | | date_nanos | boolean | +| dense_vector | boolean | | double | boolean | | geo_point | boolean | | geo_shape | boolean | diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/case.json b/docs/reference/query-languages/esql/kibana/definition/functions/case.json index 96a3089e7f63c..b37f20a21dff9 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/case.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/case.json @@ -214,6 +214,48 @@ "variadic" : true, "returnType" : "date_nanos" }, + { + "params" : [ + { + "name" : "condition", + "type" : "boolean", + "optional" : false, + "description" : "A condition." + }, + { + "name" : "trueValue", + "type" : "dense_vector", + "optional" : false, + "description" : "The value that’s returned when the corresponding condition is the first to evaluate to `true`. The default value is returned when no condition matches." + } + ], + "variadic" : true, + "returnType" : "dense_vector" + }, + { + "params" : [ + { + "name" : "condition", + "type" : "boolean", + "optional" : false, + "description" : "A condition." + }, + { + "name" : "trueValue", + "type" : "dense_vector", + "optional" : false, + "description" : "The value that’s returned when the corresponding condition is the first to evaluate to `true`. The default value is returned when no condition matches." + }, + { + "name" : "elseValue", + "type" : "dense_vector", + "optional" : true, + "description" : "The value that’s returned when no condition evaluates to `true`." + } + ], + "variadic" : true, + "returnType" : "dense_vector" + }, { "params" : [ { diff --git a/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json b/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json index 08c98810ddede..11b7291b34afb 100644 --- a/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json +++ b/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json @@ -102,6 +102,18 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "field", + "type" : "dense_vector", + "optional" : false, + "description" : "Value to check. It can be a single- or multi-valued column or an expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { From 9538a33c4eced9207b7501a57fc80f6227d5c71c Mon Sep 17 00:00:00 2001 From: cdelgado Date: Mon, 29 Sep 2025 23:25:50 +0200 Subject: [PATCH 14/35] Fix CBE for CaseTests when CBEs are thrown on beginPositionEntry --- .../java/org/elasticsearch/compute/data/BlockUtils.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java index abc35d68a3d36..ed5519c11a79e 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java @@ -9,6 +9,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder.AggregateMetricDoubleLiteral; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; @@ -52,7 +53,12 @@ public BuilderWrapper(Block.Builder builder, Consumer append) { } public void accept(Object object) { - append.accept(object); + try { + append.accept(object); + } catch (CircuitBreakingException e) { + close(); + throw e; + } } @Override From 00ed4c5165455ccd128e239b4761232692ee091f Mon Sep 17 00:00:00 2001 From: cdelgado Date: Tue, 30 Sep 2025 08:43:31 +0200 Subject: [PATCH 15/35] Add ToString support --- .../_snippets/functions/types/to_string.md | 1 + .../definition/functions/to_string.json | 12 ++++++++ .../function/scalar/convert/ToString.java | 8 ++++++ .../esql/type/EsqlDataTypeConverter.java | 1 + .../expression/function/TestCaseSupplier.java | 28 +++++++++++++++++++ .../scalar/convert/ToStringTests.java | 9 ++++++ 6 files changed, 59 insertions(+) diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/to_string.md b/docs/reference/query-languages/esql/_snippets/functions/types/to_string.md index 77fd3adaf8c56..1c63eb7b367e8 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/to_string.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/to_string.md @@ -9,6 +9,7 @@ | cartesian_shape | keyword | | date | keyword | | date_nanos | keyword | +| dense_vector | keyword | | double | keyword | | geo_point | keyword | | geo_shape | keyword | diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/to_string.json b/docs/reference/query-languages/esql/kibana/definition/functions/to_string.json index 7e81f967db12e..e95e4fc682a83 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/to_string.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/to_string.json @@ -64,6 +64,18 @@ "variadic" : false, "returnType" : "keyword" }, + { + "params" : [ + { + "name" : "field", + "type" : "dense_vector", + "optional" : false, + "description" : "Input value. The input can be a single- or multi-valued column or an expression." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, { "params" : [ { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java index 9cf14aea6b643..9b05e37774bb8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java @@ -31,6 +31,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; +import static org.elasticsearch.xpack.esql.core.type.DataType.DENSE_VECTOR; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.GEOHASH; import static org.elasticsearch.xpack.esql.core.type.DataType.GEOHEX; @@ -62,6 +63,7 @@ public class ToString extends AbstractConvertFunction implements EvaluatorMapper Map.entry(DATETIME, ToStringFromDatetimeEvaluator.Factory::new), Map.entry(DATE_NANOS, ToStringFromDateNanosEvaluator.Factory::new), Map.entry(IP, ToStringFromIPEvaluator.Factory::new), + Map.entry(DENSE_VECTOR, ToStringFromFloatEvaluator.Factory::new), Map.entry(DOUBLE, ToStringFromDoubleEvaluator.Factory::new), Map.entry(LONG, ToStringFromLongEvaluator.Factory::new), Map.entry(INTEGER, ToStringFromIntEvaluator.Factory::new), @@ -96,6 +98,7 @@ public ToString( "cartesian_shape", "date", "date_nanos", + "dense_vector", "double", "geo_point", "geo_shape", @@ -169,6 +172,11 @@ static BytesRef fromDouble(double dbl) { return numericBooleanToString(dbl); } + @ConvertEvaluator(extraName = "FromFloat") + static BytesRef fromFloat(float flt) { + return numericBooleanToString(flt); + } + @ConvertEvaluator(extraName = "FromLong") static BytesRef fromDouble(long lng) { return numericBooleanToString(lng); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java index ba7f3121f101b..059cb20b8a08c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -143,6 +143,7 @@ public class EsqlDataTypeConverter { typeToConverter.put(DATETIME, ToDatetime::new); typeToConverter.put(DATE_NANOS, ToDateNanos::new); // ToDegrees, typeless + typeToConverter.put(DENSE_VECTOR, ToDenseVector::new); typeToConverter.put(DOUBLE, ToDouble::new); typeToConverter.put(GEO_POINT, ToGeoPoint::new); typeToConverter.put(GEO_SHAPE, ToGeoShape::new); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index 930bfc7cf7203..5107c55503e8a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -53,6 +53,7 @@ import java.util.function.UnaryOperator; import java.util.stream.Collectors; +import static org.elasticsearch.test.ESTestCase.randomFloatBetween; import static org.elasticsearch.test.ESTestCase.randomIntBetween; import static org.elasticsearch.xpack.esql.core.util.NumericUtils.UNSIGNED_LONG_MAX; import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN; @@ -647,6 +648,33 @@ public static void forUnaryDouble( ); } + /** + * Generate positive test cases for a unary function operating on a {@link DataType#DENSE_VECTOR}. + */ + @SuppressWarnings("unchecked") + public static void forUnaryDenseVector( + List suppliers, + String expectedEvaluatorToString, + DataType expectedType, + Function, Object> expectedValue, + float lowerBound, + float upperBound + ) { + List cases = new ArrayList<>(); + cases.add(new TypedDataSupplier("", () -> randomDenseVector(lowerBound, upperBound), DataType.DENSE_VECTOR)); + + unary(suppliers, expectedEvaluatorToString, cases, expectedType, v -> expectedValue.apply((List) v), List.of()); + } + + private static List randomDenseVector(float lower, float upper) { + int dimensions = randomIntBetween(64, 128); + List vector = new ArrayList<>(); + for (int i = 0; i < dimensions; i++) { + vector.add(randomFloatBetween(lower, upper, true)); + } + return vector; + } + /** * Generate positive test cases for a unary function operating on an {@link DataType#BOOLEAN}. */ diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java index 8d8f59b78114d..dac09579611b9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java @@ -35,6 +35,7 @@ public ToStringTests(@Name("TestCase") Supplier testC this.testCase = testCaseSupplier.get(); } + @SuppressWarnings("unchecked") @ParametersFactory public static Iterable parameters() { // TODO multivalue fields @@ -76,6 +77,14 @@ public static Iterable parameters() { Double.POSITIVE_INFINITY, List.of() ); + TestCaseSupplier.forUnaryDenseVector( + suppliers, + "ToStringFromFloatEvaluator[flt=" + read + "]", + DataType.KEYWORD, + d -> d.stream().map(f -> new BytesRef(f.toString())).toList(), + -1.0f, + 1.0f + ); TestCaseSupplier.forUnaryBoolean( suppliers, "ToStringFromBooleanEvaluator[bool=" + read + "]", From d128d7d6cc67ddd5f7e348f74c172f6fa80570f9 Mon Sep 17 00:00:00 2001 From: cdelgado Date: Tue, 30 Sep 2025 09:34:35 +0200 Subject: [PATCH 16/35] Add ToString support --- .../convert/ToStringFromFloatEvaluator.java | 133 ++++++++++++++++++ .../scalar/convert/ToStringErrorTests.java | 18 ++- 2 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromFloatEvaluator.java diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromFloatEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromFloatEvaluator.java new file mode 100644 index 0000000000000..91b8005e9da4e --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromFloatEvaluator.java @@ -0,0 +1,133 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link ToString}. + * This class is generated. Edit {@code ConvertEvaluatorImplementer} instead. + */ +public final class ToStringFromFloatEvaluator extends AbstractConvertFunction.AbstractEvaluator { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ToStringFromFloatEvaluator.class); + + private final EvalOperator.ExpressionEvaluator flt; + + public ToStringFromFloatEvaluator(Source source, EvalOperator.ExpressionEvaluator flt, + DriverContext driverContext) { + super(driverContext, source); + this.flt = flt; + } + + @Override + public EvalOperator.ExpressionEvaluator next() { + return flt; + } + + @Override + public Block evalVector(Vector v) { + FloatVector vector = (FloatVector) v; + int positionCount = v.getPositionCount(); + if (vector.isConstant()) { + return driverContext.blockFactory().newConstantBytesRefBlockWith(evalValue(vector, 0), positionCount); + } + try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + builder.appendBytesRef(evalValue(vector, p)); + } + return builder.build(); + } + } + + private BytesRef evalValue(FloatVector container, int index) { + float value = container.getFloat(index); + return ToString.fromFloat(value); + } + + @Override + public Block evalBlock(Block b) { + FloatBlock block = (FloatBlock) b; + int positionCount = block.getPositionCount(); + try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + BytesRef value = evalValue(block, i); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendBytesRef(value); + valuesAppended = true; + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private BytesRef evalValue(FloatBlock container, int index) { + float value = container.getFloat(index); + return ToString.fromFloat(value); + } + + @Override + public String toString() { + return "ToStringFromFloatEvaluator[" + "flt=" + flt + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(flt); + } + + @Override + public long baseRamBytesUsed() { + long baseRamBytesUsed = BASE_RAM_BYTES_USED; + baseRamBytesUsed += flt.baseRamBytesUsed(); + return baseRamBytesUsed; + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory flt; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory flt) { + this.source = source; + this.flt = flt; + } + + @Override + public ToStringFromFloatEvaluator get(DriverContext context) { + return new ToStringFromFloatEvaluator(source, flt.get(context), context); + } + + @Override + public String toString() { + return "ToStringFromFloatEvaluator[" + "flt=" + flt + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringErrorTests.java index b46cf4a5fe2ef..31088139b8911 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringErrorTests.java @@ -32,15 +32,21 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> TYPE_ERROR)); + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> { + /* + * In general ToString should support all signatures. While building a + * new type you may we to temporarily remove this. + */ + throw new UnsupportedOperationException("all signatures should be supported"); + })); } - private static final String TYPE_ERROR = - "aggregate_metric_double or boolean or cartesian_point or cartesian_shape or date_nanos or datetime or geo_point or geo_shape " - + "or geohash or geohex or geotile or ip or numeric or string or version"; - @Override protected void assertNumberOfCheckedSignatures(int checked) { - assertThat(checked, equalTo(1)); + /* + * In general ToString should support all signatures. While building a + * new type you may we to temporarily relax this. + */ + assertThat("all signatures should be supported", checked, equalTo(0)); } } From 6f481dc5c00836b9e5f93eed2576a7e3f166d130 Mon Sep 17 00:00:00 2001 From: cdelgado Date: Tue, 30 Sep 2025 09:34:49 +0200 Subject: [PATCH 17/35] Fix analyzer tests --- .../org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index daa0ebd5741f5..6de798562548d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -1643,7 +1643,7 @@ public void testRegexOnInt() { public void testUnsupportedTypesWithToString() { // DATE_PERIOD and TIME_DURATION types have been added, but not really patched through the engine; i.e. supported. final String supportedTypes = "aggregate_metric_double or boolean or cartesian_point or cartesian_shape or date_nanos or datetime " - + "or geo_point or geo_shape or geohash or geohex or geotile or ip or numeric or string or version"; + + "or dense_vector or geo_point or geo_shape or geohash or geohex or geotile or ip or numeric or string or version"; verifyUnsupported( "row period = 1 year | eval to_string(period)", "line 1:28: argument of [to_string(period)] must be [" + supportedTypes + "], found value [period] type [date_period]" From 416ffb8d021a6c446fcb6753b7fe3f541b785a3e Mon Sep 17 00:00:00 2001 From: Carlos Delgado <6339205+carlosdelest@users.noreply.github.com> Date: Tue, 30 Sep 2025 10:20:08 +0200 Subject: [PATCH 18/35] Update docs/changelog/135604.yaml --- docs/changelog/135604.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/135604.yaml diff --git a/docs/changelog/135604.yaml b/docs/changelog/135604.yaml new file mode 100644 index 0000000000000..c1c5ed04cb1be --- /dev/null +++ b/docs/changelog/135604.yaml @@ -0,0 +1,6 @@ +pr: 135604 +summary: ES|QL - add `dense_vector` field type and `to_dense_vector` function to release + builds +area: ES|QL +type: feature +issues: [] From 822cd60d6ccc639db43a7dbb3bfe9e0b4fe27445 Mon Sep 17 00:00:00 2001 From: cdelgado Date: Tue, 30 Sep 2025 11:01:49 +0200 Subject: [PATCH 19/35] Add changelog --- docs/changelog/135604.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/changelog/135604.yaml b/docs/changelog/135604.yaml index c1c5ed04cb1be..41b47a79aa2b4 100644 --- a/docs/changelog/135604.yaml +++ b/docs/changelog/135604.yaml @@ -1,6 +1,5 @@ pr: 135604 -summary: ES|QL - add `dense_vector` field type and `to_dense_vector` function to release - builds +summary: ES|QL - add `dense_vector` field type area: ES|QL type: feature -issues: [] +issues: [125783] From 7565493cc460c45e03360ff138ef3e73de21f0a1 Mon Sep 17 00:00:00 2001 From: cdelgado Date: Tue, 30 Sep 2025 11:02:03 +0200 Subject: [PATCH 20/35] Change BlockUtils CBE guard --- .../compute/data/BlockUtils.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java index ed5519c11a79e..fa5f139c5c6a1 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java @@ -36,29 +36,29 @@ public record BuilderWrapper(Block.Builder builder, Consumer append) imp public BuilderWrapper(Block.Builder builder, Consumer append) { this.builder = builder; this.append = o -> { - if (o == null) { - builder.appendNull(); - return; - } - if (o instanceof List l) { - builder.beginPositionEntry(); - for (Object v : l) { - append.accept(v); + try { + if (o == null) { + builder.appendNull(); + return; + } + if (o instanceof List l) { + builder.beginPositionEntry(); + for (Object v : l) { + append.accept(v); + } + builder.endPositionEntry(); + return; } - builder.endPositionEntry(); - return; + append.accept(o); + } catch (CircuitBreakingException e) { + close(); + throw e; } - append.accept(o); }; } public void accept(Object object) { - try { - append.accept(object); - } catch (CircuitBreakingException e) { - close(); - throw e; - } + append.accept(object); } @Override From 179ee30ad6799d8472b21cb802e878340c65f582 Mon Sep 17 00:00:00 2001 From: cdelgado Date: Tue, 30 Sep 2025 11:11:36 +0200 Subject: [PATCH 21/35] Fix capability check --- .../qa/testFixtures/src/main/resources/dense_vector.csv-spec | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec index b68fd9ba416b6..e23c56f0ecfd6 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec @@ -1,6 +1,7 @@ retrieveDenseVectorData required_capability: dense_vector_field_type required_capability: dense_vector_agg_metric_double_if_fns +required_capability: l2_norm_vector_similarity_function FROM dense_vector | EVAL k = v_l2_norm(float_vector, [1]) // workaround to enable fetching dense_vector From ec5e229d5ff4c10029e81a27a24a69dea6ff24ba Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 30 Sep 2025 12:00:07 +0200 Subject: [PATCH 22/35] Changed method names --- .../xpack/esql/core/expression/TypeResolutions.java | 4 ++-- .../xpack/esql/expression/function/aggregate/Sample.java | 7 ++++--- .../xpack/esql/expression/function/aggregate/Values.java | 2 +- .../expression/function/scalar/multivalue/MvAppend.java | 6 +++--- .../expression/function/scalar/multivalue/MvContains.java | 6 +++--- .../expression/function/scalar/multivalue/MvCount.java | 4 ++-- .../expression/function/scalar/multivalue/MvDedupe.java | 4 ++-- .../expression/function/scalar/multivalue/MvFirst.java | 4 ++-- .../esql/expression/function/scalar/multivalue/MvLast.java | 4 ++-- .../esql/expression/function/scalar/multivalue/MvMax.java | 4 ++-- .../esql/expression/function/scalar/multivalue/MvMin.java | 4 ++-- .../expression/function/scalar/multivalue/MvSlice.java | 4 ++-- .../esql/expression/function/scalar/multivalue/MvSort.java | 4 ++-- 13 files changed, 29 insertions(+), 28 deletions(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java index 07c9bf32db22e..327d9933c59bc 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java @@ -77,7 +77,7 @@ public static TypeResolution isDate(Expression e, String operationName, ParamOrd /** * @see DataType#isRepresentable(DataType) */ - public static TypeResolution isRepresentableExceptCounters(Expression e, String operationName, ParamOrdinal paramOrd) { + public static TypeResolution isRepresentableExceptCountersAndDenseVector(Expression e, String operationName, ParamOrdinal paramOrd) { return isType( e, dt -> isRepresentable(dt) && dt != DENSE_VECTOR, @@ -87,7 +87,7 @@ public static TypeResolution isRepresentableExceptCounters(Expression e, String ); } - public static TypeResolution isRepresentableExceptCountersAndSpatial(Expression e, String operationName, ParamOrdinal paramOrd) { + public static TypeResolution isRepresentableExceptCountersSpatialAndDenseVector(Expression e, String operationName, ParamOrdinal paramOrd) { return isType( e, (t) -> isSpatialOrGrid(t) == false && DataType.isRepresentable(t) && t != DENSE_VECTOR, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sample.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sample.java index 1ed7e439a9f06..66454c7dc6572 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sample.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sample.java @@ -40,7 +40,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNotNull; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndDenseVector; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; import static org.elasticsearch.xpack.esql.expression.Foldables.TypeResolutionValidator.forPostOptimizationValidation; import static org.elasticsearch.xpack.esql.expression.Foldables.TypeResolutionValidator.forPreOptimizationValidation; @@ -126,8 +126,9 @@ protected TypeResolution resolveType() { if (childrenResolved() == false) { return new TypeResolution("Unresolved children"); } - var typeResolution = isRepresentableExceptCounters(field(), sourceText(), FIRST).and(isNotNull(limitField(), sourceText(), SECOND)) - .and(isType(limitField(), dt -> dt == DataType.INTEGER, sourceText(), SECOND, "integer")); + var typeResolution = isRepresentableExceptCountersAndDenseVector(field(), sourceText(), FIRST).and( + isNotNull(limitField(), sourceText(), SECOND) + ).and(isType(limitField(), dt -> dt == DataType.INTEGER, sourceText(), SECOND, "integer")); if (typeResolution.unresolved()) { return typeResolution; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java index 863a9d2d2d58d..4958bbca017f3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java @@ -165,7 +165,7 @@ public DataType dataType() { @Override protected TypeResolution resolveType() { - return TypeResolutions.isRepresentableExceptCounters(field(), sourceText(), DEFAULT); + return TypeResolutions.isRepresentableExceptCountersAndDenseVector(field(), sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java index a950fca954101..6989d9651fd55 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java @@ -40,7 +40,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndDenseVector; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; /** @@ -148,14 +148,14 @@ protected TypeResolution resolveType() { return new TypeResolution("Unresolved children"); } - TypeResolution resolution = isRepresentableExceptCounters(field1, sourceText(), FIRST); + TypeResolution resolution = isRepresentableExceptCountersAndDenseVector(field1, sourceText(), FIRST); if (resolution.unresolved()) { return resolution; } dataType = field1.dataType().noText(); if (dataType == DataType.NULL) { dataType = field2.dataType().noText(); - return isRepresentableExceptCounters(field2, sourceText(), SECOND); + return isRepresentableExceptCountersAndDenseVector(field2, sourceText(), SECOND); } return isType(field2, t -> t.noText() == dataType, sourceText(), SECOND, dataType.typeName()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java index 052dac0d68b1c..fa4cf12505f71 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java @@ -44,7 +44,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndDenseVector; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; /** @@ -147,12 +147,12 @@ protected TypeResolution resolveType() { return new TypeResolution("Unresolved children"); } - TypeResolution resolution = isRepresentableExceptCounters(left(), sourceText(), FIRST); + TypeResolution resolution = isRepresentableExceptCountersAndDenseVector(left(), sourceText(), FIRST); if (resolution.unresolved()) { return resolution; } if (left().dataType() == DataType.NULL) { - return isRepresentableExceptCounters(right(), sourceText(), SECOND); + return isRepresentableExceptCountersAndDenseVector(right(), sourceText(), SECOND); } return isType(right(), t -> t.noText() == left().dataType().noText(), sourceText(), SECOND, left().dataType().noText().typeName()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java index aa6049207fa05..186149817e6db 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java @@ -26,7 +26,7 @@ import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndDenseVector; /** * Reduce a multivalued field to a single valued field containing the count of values. @@ -79,7 +79,7 @@ public String getWriteableName() { @Override protected TypeResolution resolveFieldType() { - return isRepresentableExceptCounters(field(), sourceText(), DEFAULT); + return isRepresentableExceptCountersAndDenseVector(field(), sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java index c7b7ea0523a45..75e2554b7a106 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java @@ -23,7 +23,7 @@ import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndDenseVector; /** * Removes duplicate values from a multivalued field. @@ -95,7 +95,7 @@ public String getWriteableName() { @Override protected TypeResolution resolveFieldType() { - return isRepresentableExceptCounters(field(), sourceText(), DEFAULT); + return isRepresentableExceptCountersAndDenseVector(field(), sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java index 7afcf0a1641cc..a4b62b5e1dbff 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java @@ -31,7 +31,7 @@ import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndDenseVector; /** * Reduce a multivalued field to a single valued field containing the minimum value. @@ -110,7 +110,7 @@ public String getWriteableName() { @Override protected TypeResolution resolveFieldType() { - return isRepresentableExceptCounters(field(), sourceText(), DEFAULT); + return isRepresentableExceptCountersAndDenseVector(field(), sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java index bbcc6367784d4..a9605434a7950 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java @@ -31,7 +31,7 @@ import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndDenseVector; /** * Reduce a multivalued field to a single valued field containing the minimum value. @@ -110,7 +110,7 @@ public String getWriteableName() { @Override protected TypeResolution resolveFieldType() { - return isRepresentableExceptCounters(field(), sourceText(), DEFAULT); + return isRepresentableExceptCountersAndDenseVector(field(), sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java index fc8281188c067..aecbe84c6d009 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java @@ -26,7 +26,7 @@ import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndSpatial; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersSpatialAndDenseVector; /** * Reduce a multivalued field to a single valued field containing the maximum value. @@ -68,7 +68,7 @@ public String getWriteableName() { @Override protected TypeResolution resolveFieldType() { - return isRepresentableExceptCountersAndSpatial(field(), sourceText(), DEFAULT); + return isRepresentableExceptCountersSpatialAndDenseVector(field(), sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java index 36dac6658aa5a..e862090f3ba2e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java @@ -26,7 +26,7 @@ import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndSpatial; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersSpatialAndDenseVector; /** * Reduce a multivalued field to a single valued field containing the minimum value. @@ -68,7 +68,7 @@ public String getWriteableName() { @Override protected TypeResolution resolveFieldType() { - return isRepresentableExceptCountersAndSpatial(field(), sourceText(), DEFAULT); + return isRepresentableExceptCountersSpatialAndDenseVector(field(), sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java index 4ad3426b94719..f025a16c85403 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java @@ -42,7 +42,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.THIRD; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndDenseVector; import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToInt; @@ -168,7 +168,7 @@ protected TypeResolution resolveType() { return new TypeResolution("Unresolved children"); } - TypeResolution resolution = isRepresentableExceptCounters(field, sourceText(), FIRST); + TypeResolution resolution = isRepresentableExceptCountersAndDenseVector(field, sourceText(), FIRST); if (resolution.unresolved()) { return resolution; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java index 8a433bee0a4a1..9731a547dc7d2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java @@ -52,7 +52,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndDenseVector; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; import static org.elasticsearch.xpack.esql.expression.Validations.isFoldable; @@ -127,7 +127,7 @@ protected TypeResolution resolveType() { return new TypeResolution("Unresolved children"); } - TypeResolution resolution = isRepresentableExceptCounters(field, sourceText(), FIRST); + TypeResolution resolution = isRepresentableExceptCountersAndDenseVector(field, sourceText(), FIRST); if (resolution.unresolved()) { return resolution; From 735b15d965572a16aeeb8e36effb37c37f25e44f Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 30 Sep 2025 10:12:25 +0000 Subject: [PATCH 23/35] [CI] Auto commit changes from spotless --- .../xpack/esql/core/expression/TypeResolutions.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java index 327d9933c59bc..75a6c36a1f052 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java @@ -87,7 +87,11 @@ public static TypeResolution isRepresentableExceptCountersAndDenseVector(Express ); } - public static TypeResolution isRepresentableExceptCountersSpatialAndDenseVector(Expression e, String operationName, ParamOrdinal paramOrd) { + public static TypeResolution isRepresentableExceptCountersSpatialAndDenseVector( + Expression e, + String operationName, + ParamOrdinal paramOrd + ) { return isType( e, (t) -> isSpatialOrGrid(t) == false && DataType.isRepresentable(t) && t != DENSE_VECTOR, From 2c31d589ff3b9a7f1134bc63c4260cfff445c5e0 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 30 Sep 2025 12:42:02 +0200 Subject: [PATCH 24/35] Added transport version for dense_vector data type --- .../referable/esql_dense_vector_created_version.csv | 1 + server/src/main/resources/transport/upper_bounds/9.2.csv | 2 +- .../org/elasticsearch/xpack/esql/core/type/DataType.java | 8 +++++--- 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 server/src/main/resources/transport/definitions/referable/esql_dense_vector_created_version.csv diff --git a/server/src/main/resources/transport/definitions/referable/esql_dense_vector_created_version.csv b/server/src/main/resources/transport/definitions/referable/esql_dense_vector_created_version.csv new file mode 100644 index 0000000000000..2752f57900f3e --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/esql_dense_vector_created_version.csv @@ -0,0 +1 @@ +9183000 diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index d501e4094e60d..f575dcaf4efa8 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -index_reshard_shardcount_small,9181000 +esql_dense_vector_created_version,9183000 diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 67502a5b8ca51..c10f83597a552 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -35,7 +35,6 @@ import static java.util.stream.Collectors.toMap; import static org.elasticsearch.TransportVersions.INFERENCE_REQUEST_ADAPTIVE_RATE_LIMITING; -import static org.elasticsearch.TransportVersions.ML_INFERENCE_SAGEMAKER_CHAT_COMPLETION; /** * This enum represents data types the ES|QL query processing layer is able to @@ -326,8 +325,7 @@ public enum DataType implements Writeable { builder().esType("dense_vector") .estimatedSize(4096) .createdVersion( - // Version created just *after* we committed support for dense_vector - ML_INFERENCE_SAGEMAKER_CHAT_COMPLETION + DataTypesTransportVersions.ESQL_DENSE_VECTOR_CREATED_VERSION ) ); @@ -954,4 +952,8 @@ Builder createdVersion(TransportVersion createdVersion) { return this; } } + + private static class DataTypesTransportVersions { + public static final TransportVersion ESQL_DENSE_VECTOR_CREATED_VERSION = TransportVersion.fromName("esql_dense_vector_created_version"); + } } From 2ed9e8b310bf65c743ea883d4830e1c5da6aebc1 Mon Sep 17 00:00:00 2001 From: Carlos Delgado <6339205+carlosdelest@users.noreply.github.com> Date: Tue, 30 Sep 2025 12:46:05 +0200 Subject: [PATCH 25/35] Update docs/changelog/135604.yaml --- docs/changelog/135604.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/135604.yaml b/docs/changelog/135604.yaml index 41b47a79aa2b4..d91daf7325c96 100644 --- a/docs/changelog/135604.yaml +++ b/docs/changelog/135604.yaml @@ -2,4 +2,4 @@ pr: 135604 summary: ES|QL - add `dense_vector` field type area: ES|QL type: feature -issues: [125783] +issues: [] From 3452979fa3fdea197724c474266ab4267a323e28 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 30 Sep 2025 11:00:37 +0000 Subject: [PATCH 26/35] [CI] Auto commit changes from spotless --- .../elasticsearch/xpack/esql/core/type/DataType.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index c10f83597a552..618f941a0214a 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -322,11 +322,7 @@ public enum DataType implements Writeable { * Fields with this type are dense vectors, represented as an array of double values. */ DENSE_VECTOR( - builder().esType("dense_vector") - .estimatedSize(4096) - .createdVersion( - DataTypesTransportVersions.ESQL_DENSE_VECTOR_CREATED_VERSION - ) + builder().esType("dense_vector").estimatedSize(4096).createdVersion(DataTypesTransportVersions.ESQL_DENSE_VECTOR_CREATED_VERSION) ); /** @@ -954,6 +950,8 @@ Builder createdVersion(TransportVersion createdVersion) { } private static class DataTypesTransportVersions { - public static final TransportVersion ESQL_DENSE_VECTOR_CREATED_VERSION = TransportVersion.fromName("esql_dense_vector_created_version"); + public static final TransportVersion ESQL_DENSE_VECTOR_CREATED_VERSION = TransportVersion.fromName( + "esql_dense_vector_created_version" + ); } } From 4d16a4455d6814f4701f7b13a9cdd2bad6f64a23 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 30 Sep 2025 14:12:43 +0200 Subject: [PATCH 27/35] Fix mixed cluster tests --- .../elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java index bb819eaa6ed5e..93f75f8395ab3 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java @@ -34,6 +34,7 @@ import static org.elasticsearch.index.IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING; import static org.elasticsearch.index.mapper.SourceFieldMapper.Mode.SYNTHETIC; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.L2_NORM_VECTOR_SIMILARITY_FUNCTION; public class DenseVectorFieldTypeIT extends AbstractEsqlIntegTestCase { @@ -90,6 +91,8 @@ public DenseVectorFieldTypeIT( private final Map> indexedVectors = new HashMap<>(); public void testRetrieveFieldType() { + assumeTrue("Need L2_NORM available for dense_vector retrieval", L2_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()); + var query = """ FROM test | EVAL k = v_l2_norm(vector, [1]) // workaround to enable fetching dense_vector @@ -104,6 +107,8 @@ public void testRetrieveFieldType() { @SuppressWarnings("unchecked") public void testRetrieveTopNDenseVectorFieldData() { + assumeTrue("Need L2_NORM available for dense_vector retrieval", L2_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()); + var query = """ FROM test | EVAL k = v_l2_norm(vector, [1]) // workaround to enable fetching dense_vector @@ -132,6 +137,8 @@ public void testRetrieveTopNDenseVectorFieldData() { @SuppressWarnings("unchecked") public void testRetrieveDenseVectorFieldData() { + assumeTrue("Need L2_NORM available for dense_vector retrieval", L2_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()); + var query = """ FROM test | EVAL k = v_l2_norm(vector, [1]) // workaround to enable fetching dense_vector From 7af9e82957015f44a4fbbef258cd3b26fb22bddb Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 30 Sep 2025 17:25:31 +0200 Subject: [PATCH 28/35] Renamed capability to avoid failures on mixed clusters --- .../src/main/resources/dense_vector-bit.csv-spec | 6 +++--- .../src/main/resources/dense_vector-byte.csv-spec | 6 +++--- .../src/main/resources/dense_vector.csv-spec | 14 +++++++------- .../xpack/esql/action/EsqlCapabilities.java | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec index 2db3e96de11c0..5cc4c9bc60882 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec @@ -1,5 +1,5 @@ retrieveBitVectorData -required_capability: dense_vector_field_type_bit_elements +required_capability: dense_vector_field_type_released required_capability: l2_norm_vector_similarity_function FROM dense_vector @@ -16,7 +16,7 @@ id:l | bit_vector:dense_vector ; denseBitVectorWithEval -required_capability: dense_vector_field_type_bit_elements +required_capability: dense_vector_field_type_released required_capability: l2_norm_vector_similarity_function FROM dense_vector @@ -33,7 +33,7 @@ id:l | v:dense_vector ; denseBitVectorWithRenameAndDrop -required_capability: dense_vector_field_type_bit_elements +required_capability: dense_vector_field_type_released required_capability: l2_norm_vector_similarity_function FROM dense_vector diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec index 0ecf23332ad58..e298cc9eda963 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec @@ -1,5 +1,5 @@ retrieveByteVectorData -required_capability: dense_vector_field_type_byte_elements +required_capability: dense_vector_field_type_released required_capability: l2_norm_vector_similarity_function FROM dense_vector @@ -16,7 +16,7 @@ id:l | byte_vector:dense_vector ; denseByteVectorWithEval -required_capability: dense_vector_field_type_byte_elements +required_capability: dense_vector_field_type_released required_capability: l2_norm_vector_similarity_function FROM dense_vector @@ -34,7 +34,7 @@ id:l | v:dense_vector ; denseByteVectorWithRenameAndDrop -required_capability: dense_vector_field_type_byte_elements +required_capability: dense_vector_field_type_released required_capability: l2_norm_vector_similarity_function FROM dense_vector diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec index e23c56f0ecfd6..d0133d409e83e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec @@ -1,5 +1,5 @@ retrieveDenseVectorData -required_capability: dense_vector_field_type +required_capability: dense_vector_field_type_released required_capability: dense_vector_agg_metric_double_if_fns required_capability: l2_norm_vector_similarity_function @@ -17,7 +17,7 @@ id:l | float_vector:dense_vector ; denseVectorWithEval -required_capability: dense_vector_field_type +required_capability: dense_vector_field_type_released required_capability: l2_norm_vector_similarity_function FROM dense_vector @@ -35,7 +35,7 @@ id:l | v:dense_vector ; denseVectorWithRenameAndDrop -required_capability: dense_vector_field_type +required_capability: dense_vector_field_type_released required_capability: l2_norm_vector_similarity_function FROM dense_vector @@ -54,7 +54,7 @@ id:l | new_vector:dense_vector ; convertIntsToDenseVector -required_capability: dense_vector_field_type +required_capability: dense_vector_field_type_released required_capability: to_dense_vector_function // tag::to_dense_vector-ints[] @@ -71,7 +71,7 @@ vector:dense_vector ; convertLongsToDenseVector -required_capability: dense_vector_field_type +required_capability: dense_vector_field_type_released required_capability: to_dense_vector_function row longs = [5013792, 2147483647, 501379200000] @@ -84,7 +84,7 @@ vector:dense_vector ; convertDoublesToDenseVector -required_capability: dense_vector_field_type +required_capability: dense_vector_field_type_released required_capability: to_dense_vector_function row doubles = [123.4, 567.8, 901.2] @@ -97,7 +97,7 @@ vector:dense_vector ; convertHexStringToDenseVector -required_capability: dense_vector_field_type +required_capability: dense_vector_field_type_released required_capability: to_dense_vector_function row hex_str = "0102030405060708090a0b0c0d0e0f" diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index f931702c5b2af..a2e1966d79221 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1285,7 +1285,7 @@ public enum Cap { /** * Dense vector field type support */ - DENSE_VECTOR_FIELD_TYPE, + DENSE_VECTOR_FIELD_TYPE_RELEASED, /** * Enable support for index aliases in lookup joins From f371c65ff2cd554ae0439bed4a6b4eaa99a4941d Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 30 Sep 2025 18:05:35 +0200 Subject: [PATCH 29/35] KNN out of snapshot, removed capability check --- .../esql/qa/rest/KnnSemanticTextTestCase.java | 4 --- .../xpack/esql/plugin/KnnFunctionIT.java | 3 -- .../xpack/esql/action/EsqlCapabilities.java | 2 +- .../function/vector/VectorWritables.java | 4 +-- .../xpack/esql/analysis/AnalyzerTests.java | 29 +++++---------- .../xpack/esql/analysis/VerifierTests.java | 36 +++++-------------- .../function/fulltext/KnnTests.java | 7 ---- .../LocalPhysicalPlanOptimizerTests.java | 24 ------------- .../optimizer/LogicalPlanOptimizerTests.java | 30 ---------------- 9 files changed, 20 insertions(+), 119 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/KnnSemanticTextTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/KnnSemanticTextTestCase.java index 7b2909c1d2104..545dce156c74f 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/KnnSemanticTextTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/KnnSemanticTextTestCase.java @@ -12,7 +12,6 @@ import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.esql.AssertWarnings; import org.elasticsearch.xpack.esql.CsvTestsDataLoader; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -43,7 +42,6 @@ public void checkCapability() { @SuppressWarnings("unchecked") public void testKnnQueryWithSemanticText() throws IOException { - assumeTrue("knn with semantic text not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String knnQuery = """ FROM semantic-test METADATA _score | WHERE knn(dense_semantic, [0, 1, 2]) @@ -65,7 +63,6 @@ public void testKnnQueryWithSemanticText() throws IOException { } public void testKnnQueryOnTextField() throws IOException { - assumeTrue("knn with semantic text not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String knnQuery = """ FROM semantic-test METADATA _score | WHERE knn(text, [0, 1, 2]) @@ -80,7 +77,6 @@ public void testKnnQueryOnTextField() throws IOException { } public void testKnnQueryOnSparseSemanticTextField() throws IOException { - assumeTrue("knn with semantic text not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String knnQuery = """ FROM semantic-test METADATA _score | WHERE knn(sparse_semantic, [0, 1, 2]) diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java index c8a426113eaf6..f6635defd1857 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java @@ -21,7 +21,6 @@ import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.parser.ParserUtils; @@ -231,8 +230,6 @@ public void testKnnWithLookupJoin() { @Before public void setup() throws IOException { - assumeTrue("Needs KNN support", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - var indexName = "test"; var client = client().admin().indices(); XContentBuilder mapping = XContentFactory.jsonBuilder() diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 483da630bb4c3..06d9166174852 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1321,7 +1321,7 @@ public enum Cap { /** * Support knn function */ - KNN_FUNCTION_V5(Build.current().isSnapshot()), + KNN_FUNCTION_V5, /** * Support for the {@code TEXT_EMBEDDING} function for generating dense vector embeddings. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorWritables.java index d43ec3e92981d..9c57db4509aee 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorWritables.java @@ -27,9 +27,7 @@ private VectorWritables() { public static List getNamedWritables() { List entries = new ArrayList<>(); - if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { - entries.add(Knn.ENTRY); - } + entries.add(Knn.ENTRY); if (EsqlCapabilities.Cap.COSINE_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { entries.add(CosineSimilarity.ENTRY); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 6de798562548d..cff2ffdd061e2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -2356,22 +2356,14 @@ public void testImplicitCasting() { } public void testDenseVectorImplicitCastingKnn() { - assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - - if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { - checkDenseVectorCastingHexKnn("float_vector"); - checkDenseVectorCastingKnn("float_vector"); - } - if (EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE_BYTE_ELEMENTS.isEnabled()) { - checkDenseVectorCastingKnn("byte_vector"); - checkDenseVectorCastingHexKnn("byte_vector"); - checkDenseVectorEvalCastingKnn("byte_vector"); - } - if (EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE_BIT_ELEMENTS.isEnabled()) { - checkDenseVectorCastingKnn("bit_vector"); - checkDenseVectorCastingHexKnn("bit_vector"); - checkDenseVectorEvalCastingKnn("bit_vector"); - } + checkDenseVectorCastingHexKnn("float_vector"); + checkDenseVectorCastingKnn("float_vector"); + checkDenseVectorCastingKnn("byte_vector"); + checkDenseVectorCastingHexKnn("byte_vector"); + checkDenseVectorEvalCastingKnn("byte_vector"); + checkDenseVectorCastingKnn("bit_vector"); + checkDenseVectorCastingHexKnn("bit_vector"); + checkDenseVectorEvalCastingKnn("bit_vector"); } private static void checkDenseVectorCastingKnn(String fieldName) { @@ -2535,9 +2527,7 @@ private void checkDenseVectorEvalCastingSimilarityFunction(String similarityFunc } public void testVectorFunctionHexImplicitCastingError() { - if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { - checkVectorFunctionHexImplicitCastingError("where knn(float_vector, \"notcorrect\")"); - } + checkVectorFunctionHexImplicitCastingError("where knn(float_vector, \"notcorrect\")"); if (EsqlCapabilities.Cap.DOT_PRODUCT_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { checkVectorFunctionHexImplicitCastingError("eval s = v_dot_product(\"notcorrect\", 0.342)"); } @@ -3920,7 +3910,6 @@ public void testTextEmbeddingFunctionWithoutModel() { } public void testKnnFunctionWithTextEmbedding() { - assumeTrue("KNN function capability required", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); assumeTrue("TEXT_EMBEDDING function required", EsqlCapabilities.Cap.TEXT_EMBEDDING_FUNCTION.isEnabled()); LogicalPlan plan = analyze( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 305d495b21e62..3c9e5aa36f373 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1306,9 +1306,7 @@ public void testFieldBasedFullTextFunctions() throws Exception { checkFieldBasedWithNonIndexedColumn("Term", "term(text, \"cat\")", "function"); checkFieldBasedFunctionNotAllowedAfterCommands("Term", "function", "term(title, \"Meditation\")"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { - checkFieldBasedFunctionNotAllowedAfterCommands("KNN", "function", "knn(vector, [1, 2, 3])"); - } + checkFieldBasedFunctionNotAllowedAfterCommands("KNN", "function", "knn(vector, [1, 2, 3])"); } private void checkFieldBasedFunctionNotAllowedAfterCommands(String functionName, String functionType, String functionInvocation) { @@ -1433,16 +1431,13 @@ public void testFullTextFunctionsOnlyAllowedInWhere() throws Exception { checkFullTextFunctionsOnlyAllowedInWhere("QSTR", "qstr(\"Meditation\")", "function"); checkFullTextFunctionsOnlyAllowedInWhere("KQL", "kql(\"Meditation\")", "function"); checkFullTextFunctionsOnlyAllowedInWhere("MatchPhrase", "match_phrase(title, \"Meditation\")", "function"); + checkFullTextFunctionsOnlyAllowedInWhere("KNN", "knn(vector, [0, 1, 2])", "function"); if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { checkFullTextFunctionsOnlyAllowedInWhere("Term", "term(title, \"Meditation\")", "function"); } if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { checkFullTextFunctionsOnlyAllowedInWhere("MultiMatch", "multi_match(\"Meditation\", title, body)", "function"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { - checkFullTextFunctionsOnlyAllowedInWhere("KNN", "knn(vector, [0, 1, 2])", "function"); - } - } private void checkFullTextFunctionsOnlyAllowedInWhere(String functionName, String functionInvocation, String functionType) @@ -1488,15 +1483,13 @@ public void testFullTextFunctionsDisjunctions() { checkWithFullTextFunctionsDisjunctions("qstr(\"title: Meditation\")"); checkWithFullTextFunctionsDisjunctions("kql(\"title: Meditation\")"); checkWithFullTextFunctionsDisjunctions("match_phrase(title, \"Meditation\")"); + checkWithFullTextFunctionsDisjunctions("knn(vector, [1, 2, 3])"); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { checkWithFullTextFunctionsDisjunctions("multi_match(\"Meditation\", title, body)"); } if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { checkWithFullTextFunctionsDisjunctions("term(title, \"Meditation\")"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { - checkWithFullTextFunctionsDisjunctions("knn(vector, [1, 2, 3])"); - } } private void checkWithFullTextFunctionsDisjunctions(String functionInvocation) { @@ -1553,15 +1546,13 @@ public void testFullTextFunctionsWithNonBooleanFunctions() { checkFullTextFunctionsWithNonBooleanFunctions("QSTR", "qstr(\"title: Meditation\")", "function"); checkFullTextFunctionsWithNonBooleanFunctions("KQL", "kql(\"title: Meditation\")", "function"); checkFullTextFunctionsWithNonBooleanFunctions("MatchPhrase", "match_phrase(title, \"Meditation\")", "function"); + checkFullTextFunctionsWithNonBooleanFunctions("KNN", "knn(vector, [1, 2, 3])", "function"); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { checkFullTextFunctionsWithNonBooleanFunctions("MultiMatch", "multi_match(\"Meditation\", title, body)", "function"); } if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { checkFullTextFunctionsWithNonBooleanFunctions("Term", "term(title, \"Meditation\")", "function"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { - checkFullTextFunctionsWithNonBooleanFunctions("KNN", "knn(vector, [1, 2, 3])", "function"); - } } private void checkFullTextFunctionsWithNonBooleanFunctions(String functionName, String functionInvocation, String functionType) { @@ -1624,15 +1615,13 @@ public void testFullTextFunctionsTargetsExistingField() throws Exception { testFullTextFunctionTargetsExistingField("match(title, \"Meditation\")"); testFullTextFunctionTargetsExistingField("title : \"Meditation\""); testFullTextFunctionTargetsExistingField("match_phrase(title, \"Meditation\")"); + testFullTextFunctionTargetsExistingField("knn(vector, [0, 1, 2], 10)"); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { testFullTextFunctionTargetsExistingField("multi_match(\"Meditation\", title)"); } if (EsqlCapabilities.Cap.TERM_FUNCTION.isEnabled()) { testFullTextFunctionTargetsExistingField("term(fist_name, \"Meditation\")"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { - testFullTextFunctionTargetsExistingField("knn(vector, [0, 1, 2], 10)"); - } } private void testFullTextFunctionTargetsExistingField(String functionInvocation) throws Exception { @@ -2309,12 +2298,10 @@ public void testFullTextFunctionOptions() { checkOptionDataTypes(Match.ALLOWED_OPTIONS, "FROM test | WHERE match(title, \"Jean\", {\"%s\": %s})"); checkOptionDataTypes(QueryString.ALLOWED_OPTIONS, "FROM test | WHERE QSTR(\"title: Jean\", {\"%s\": %s})"); checkOptionDataTypes(MatchPhrase.ALLOWED_OPTIONS, "FROM test | WHERE MATCH_PHRASE(title, \"Jean\", {\"%s\": %s})"); + checkOptionDataTypes(Knn.ALLOWED_OPTIONS, "FROM test | WHERE KNN(vector, [0.1, 0.2, 0.3], {\"%s\": %s})"); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { checkOptionDataTypes(MultiMatch.OPTIONS, "FROM test | WHERE MULTI_MATCH(\"Jean\", title, body, {\"%s\": %s})"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { - checkOptionDataTypes(Knn.ALLOWED_OPTIONS, "FROM test | WHERE KNN(vector, [0.1, 0.2, 0.3], {\"%s\": %s})"); - } } /** @@ -2397,6 +2384,8 @@ public void testFullTextFunctionsNullArgs() throws Exception { checkFullTextFunctionNullArgs("kql(null)", ""); checkFullTextFunctionNullArgs("match_phrase(null, \"query\")", "first"); checkFullTextFunctionNullArgs("match_phrase(title, null)", "second"); + checkFullTextFunctionNullArgs("knn(null, [0, 1, 2])", "first"); + checkFullTextFunctionNullArgs("knn(vector, null)", "second"); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { checkFullTextFunctionNullArgs("multi_match(null, title)", "first"); checkFullTextFunctionNullArgs("multi_match(\"query\", null)", "second"); @@ -2405,10 +2394,6 @@ public void testFullTextFunctionsNullArgs() throws Exception { checkFullTextFunctionNullArgs("term(null, \"query\")", "first"); checkFullTextFunctionNullArgs("term(title, null)", "second"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { - checkFullTextFunctionNullArgs("knn(null, [0, 1, 2])", "first"); - checkFullTextFunctionNullArgs("knn(vector, null)", "second"); - } } private void checkFullTextFunctionNullArgs(String functionInvocation, String argOrdinal) throws Exception { @@ -2433,12 +2418,10 @@ public void testFullTextFunctionsInStats() { checkFullTextFunctionsInStats("qstr(\"title: Meditation\")"); checkFullTextFunctionsInStats("kql(\"title: Meditation\")"); checkFullTextFunctionsInStats("match_phrase(title, \"Meditation\")"); + checkFullTextFunctionsInStats("knn(vector, [0, 1, 2])"); if (EsqlCapabilities.Cap.MULTI_MATCH_FUNCTION.isEnabled()) { checkFullTextFunctionsInStats("multi_match(\"Meditation\", title, body)"); } - if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { - checkFullTextFunctionsInStats("knn(vector, [0, 1, 2])"); - } } public void testDecayFunctionNullArgs() { @@ -2517,7 +2500,6 @@ public void testVectorSimilarityFunctionsNullArgs() throws Exception { } public void testFullTextFunctionsWithSemanticText() { - assumeTrue("requires knn function", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); checkFullTextFunctionsWithSemanticText("knn(semantic, [0, 1, 2])"); checkFullTextFunctionsWithSemanticText("match(semantic, \"hello world\")"); checkFullTextFunctionsWithSemanticText("semantic:\"hello world\""); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KnnTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KnnTests.java index 9871b9ce2e25e..17147ce60f82a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KnnTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KnnTests.java @@ -11,7 +11,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -24,7 +23,6 @@ import org.elasticsearch.xpack.esql.expression.function.vector.Knn; import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; -import org.junit.Before; import java.util.ArrayList; import java.util.List; @@ -51,11 +49,6 @@ public static Iterable parameters() { return parameterSuppliersFromTypedData(addFunctionNamedParams(testCaseSuppliers())); } - @Before - public void checkCapability() { - assumeTrue("KNN is not enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - } - private static List testCaseSuppliers() { List suppliers = new ArrayList<>(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index d53c370a572d5..5980ab3fa6a32 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -1384,8 +1384,6 @@ public void testMultiMatchOptionsPushDown() { } public void testKnnOptionsPushDown() { - assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - String query = """ from test | where KNN(dense_vector, [0.1, 0.2, 0.3], @@ -1410,8 +1408,6 @@ public void testKnnOptionsPushDown() { } public void testKnnUsesLimitForK() { - assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - String query = """ from test | where KNN(dense_vector, [0.1, 0.2, 0.3]) @@ -1428,8 +1424,6 @@ public void testKnnUsesLimitForK() { } public void testKnnKAndMinCandidatesLowerK() { - assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - String query = """ from test | where KNN(dense_vector, [0.1, 0.2, 0.3], {"min_candidates": 50}) @@ -1446,8 +1440,6 @@ public void testKnnKAndMinCandidatesLowerK() { } public void testKnnKAndMinCandidatesHigherK() { - assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - String query = """ from test | where KNN(dense_vector, [0.1, 0.2, 0.3], {"min_candidates": 10}) @@ -1905,8 +1897,6 @@ public void testFullTextFunctionWithStatsBy(FullTextFunctionTestCase testCase) { } public void testKnnPrefilters() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - String query = """ from test | where knn(dense_vector, [0, 1, 2]) and integer > 10 @@ -1938,8 +1928,6 @@ public void testKnnPrefilters() { } public void testKnnPrefiltersWithMultipleFilters() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - String query = """ from test | where knn(dense_vector, [0, 1, 2]) @@ -1975,8 +1963,6 @@ public void testKnnPrefiltersWithMultipleFilters() { } public void testPushDownConjunctionsToKnnPrefilter() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - String query = """ from test | where knn(dense_vector, [0, 1, 2]) and integer > 10 @@ -2013,8 +1999,6 @@ public void testPushDownConjunctionsToKnnPrefilter() { } public void testPushDownNegatedConjunctionsToKnnPrefilter() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - String query = """ from test | where knn(dense_vector, [0, 1, 2]) and NOT integer > 10 @@ -2051,8 +2035,6 @@ public void testPushDownNegatedConjunctionsToKnnPrefilter() { } public void testNotPushDownDisjunctionsToKnnPrefilter() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - String query = """ from test | where knn(dense_vector, [0, 1, 2]) or integer > 10 @@ -2088,8 +2070,6 @@ public void testNotPushDownDisjunctionsToKnnPrefilter() { } public void testNotPushDownKnnWithNonPushablePrefilters() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - String query = """ from test | where ((knn(dense_vector, [0, 1, 2]) AND integer > 10) and ((keyword == "test") or length(text) > 10)) @@ -2122,8 +2102,6 @@ public void testNotPushDownKnnWithNonPushablePrefilters() { } public void testPushDownComplexNegationsToKnnPrefilter() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - String query = """ from test | where ((knn(dense_vector, [0, 1, 2]) or NOT integer > 10) @@ -2172,8 +2150,6 @@ and NOT ((keyword == "test") or knn(dense_vector, [4, 5, 6]))) } public void testMultipleKnnQueriesInPrefilters() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - String query = """ from test | where ((knn(dense_vector, [0, 1, 2]) or integer > 10) and ((keyword == "test") or knn(dense_vector, [4, 5, 6]))) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 5453f3bcd7679..b98c825159215 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -8615,8 +8615,6 @@ public void testSampleNoPushDownChangePoint() { } public void testPushDownConjunctionsToKnnPrefilter() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - var query = """ from test | where knn(dense_vector, [0, 1, 2]) and integer > 10 @@ -8635,8 +8633,6 @@ public void testPushDownConjunctionsToKnnPrefilter() { } public void testPushDownMultipleFiltersToKnnPrefilter() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - var query = """ from test | where knn(dense_vector, [0, 1, 2]) @@ -8658,8 +8654,6 @@ public void testPushDownMultipleFiltersToKnnPrefilter() { } public void testNotPushDownDisjunctionsToKnnPrefilter() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - var query = """ from test | where knn(dense_vector, [0, 1, 2]) or integer > 10 @@ -8675,8 +8669,6 @@ public void testNotPushDownDisjunctionsToKnnPrefilter() { } public void testPushDownConjunctionsAndNotDisjunctionsToKnnPrefilter() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - /* and and @@ -8710,8 +8702,6 @@ public void testPushDownConjunctionsAndNotDisjunctionsToKnnPrefilter() { } public void testMorePushDownConjunctionsAndNotDisjunctionsToKnnPrefilter() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - /* or or @@ -8742,8 +8732,6 @@ public void testMorePushDownConjunctionsAndNotDisjunctionsToKnnPrefilter() { } public void testMultipleKnnQueriesInPrefilters() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - /* and or @@ -8785,8 +8773,6 @@ public void testMultipleKnnQueriesInPrefilters() { } public void testKnnImplicitLimit() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - var query = """ from test | where knn(dense_vector, [0, 1, 2]) @@ -8800,8 +8786,6 @@ public void testKnnImplicitLimit() { } public void testKnnWithLimit() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - var query = """ from test | where knn(dense_vector, [0, 1, 2]) @@ -8816,8 +8800,6 @@ public void testKnnWithLimit() { } public void testKnnWithTopN() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - var query = """ from test metadata _score | where knn(dense_vector, [0, 1, 2]) @@ -8833,8 +8815,6 @@ public void testKnnWithTopN() { } public void testKnnWithMultipleLimitsAfterTopN() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - var query = """ from test metadata _score | where knn(dense_vector, [0, 1, 2]) @@ -8853,8 +8833,6 @@ public void testKnnWithMultipleLimitsAfterTopN() { } public void testKnnWithMultipleLimitsCombined() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - var query = """ from test metadata _score | where knn(dense_vector, [0, 1, 2]) @@ -8871,8 +8849,6 @@ public void testKnnWithMultipleLimitsCombined() { } public void testKnnWithMultipleClauses() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - var query = """ from test metadata _score | where knn(dense_vector, [0, 1, 2]) and match(keyword, "test") @@ -8894,8 +8870,6 @@ public void testKnnWithMultipleClauses() { } public void testKnnWithStats() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - assertThat( typesError("from test | where knn(dense_vector, [0, 1, 2]) | stats c = count(*)"), containsString("Knn function must be used with a LIMIT clause") @@ -8903,8 +8877,6 @@ public void testKnnWithStats() { } public void testKnnWithRerankAmdTopN() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - assertThat(typesError(""" from test metadata _score | where knn(dense_vector, [0, 1, 2]) @@ -8915,8 +8887,6 @@ public void testKnnWithRerankAmdTopN() { } public void testKnnWithRerankAmdLimit() { - assumeTrue("knn must be enabled", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - var query = """ from test metadata _score | where knn(dense_vector, [0, 1, 2]) From 2412e00954cdf79c928376238d09a4913435cca7 Mon Sep 17 00:00:00 2001 From: Carlos Delgado <6339205+carlosdelest@users.noreply.github.com> Date: Tue, 30 Sep 2025 18:08:47 +0200 Subject: [PATCH 30/35] Update docs/changelog/135709.yaml --- docs/changelog/135709.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/135709.yaml diff --git a/docs/changelog/135709.yaml b/docs/changelog/135709.yaml new file mode 100644 index 0000000000000..db9b995a8d779 --- /dev/null +++ b/docs/changelog/135709.yaml @@ -0,0 +1,5 @@ +pr: 135709 +summary: ES|QL - KNN function in release builds +area: "ES|QL, ES|QL" +type: feature +issues: [] From 2879010d1438cad4cb56312a4bc89f4f28ee7099 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 30 Sep 2025 18:13:57 +0200 Subject: [PATCH 31/35] Add changelog --- docs/changelog/135709.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog/135709.yaml b/docs/changelog/135709.yaml index db9b995a8d779..45a6e29a125e8 100644 --- a/docs/changelog/135709.yaml +++ b/docs/changelog/135709.yaml @@ -1,5 +1,5 @@ pr: 135709 -summary: ES|QL - KNN function in release builds -area: "ES|QL, ES|QL" +summary: ES|QL - KNN function +area: "ES|QL" type: feature -issues: [] +issues: [126710] From 74b48bbcf1ce77cb2ec2de222e4fe97ec491a66f Mon Sep 17 00:00:00 2001 From: Carlos Delgado <6339205+carlosdelest@users.noreply.github.com> Date: Tue, 30 Sep 2025 18:14:55 +0200 Subject: [PATCH 32/35] Update docs/changelog/135709.yaml --- docs/changelog/135709.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog/135709.yaml b/docs/changelog/135709.yaml index 45a6e29a125e8..11ddfb9e91cb1 100644 --- a/docs/changelog/135709.yaml +++ b/docs/changelog/135709.yaml @@ -1,5 +1,5 @@ pr: 135709 summary: ES|QL - KNN function -area: "ES|QL" +area: ES|QL type: feature -issues: [126710] +issues: [] From 23bfc538b6ab007cada4e1e9c254065aa7b6d0cb Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 30 Sep 2025 20:16:56 +0200 Subject: [PATCH 33/35] Change snapshot registry --- .../xpack/esql/expression/function/EsqlFunctionRegistry.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index a648c2f77d9c6..2d2c644bbcace 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -506,6 +506,7 @@ private static FunctionDefinition[][] functions() { new FunctionDefinition[] { def(Decay.class, quad(Decay::new), "decay"), def(Kql.class, uni(Kql::new), "kql"), + def(Knn.class, tri(Knn::new), "knn"), def(Match.class, tri(Match::new), "match"), def(MultiMatch.class, MultiMatch::new, "multi_match"), def(QueryString.class, bi(QueryString::new), "qstr"), @@ -540,7 +541,6 @@ private static FunctionDefinition[][] snapshotFunctions() { def(Last.class, bi(Last::new), "last"), def(Score.class, uni(Score::new), Score.NAME), def(Term.class, bi(Term::new), "term"), - def(Knn.class, tri(Knn::new), "knn"), def(CosineSimilarity.class, CosineSimilarity::new, "v_cosine"), def(DotProduct.class, DotProduct::new, "v_dot_product"), def(L1Norm.class, L1Norm::new, "v_l1_norm"), From 9d7542a8f7d65a5aaa4dcd31eafe56db8bc72cf9 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 30 Sep 2025 21:12:20 +0200 Subject: [PATCH 34/35] Fix knn docs --- .../query-languages/esql/kibana/definition/functions/knn.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/knn.json b/docs/reference/query-languages/esql/kibana/definition/functions/knn.json index 33845ebd490ce..5aa44b7917eb6 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/knn.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/knn.json @@ -59,5 +59,5 @@ "from colors metadata _score\n| where knn(rgb_vector, [0, 120, 0])\n| sort _score desc, color asc" ], "preview" : true, - "snapshot_only" : true + "snapshot_only" : false } From 5266f09856956fb56cfa2b3f3f68b02940dd904d Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 30 Sep 2025 21:23:09 +0200 Subject: [PATCH 35/35] Fix applies to in Knn --- .../query-languages/esql/_snippets/functions/layout/knn.md | 2 +- .../xpack/esql/expression/function/vector/Knn.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/knn.md b/docs/reference/query-languages/esql/_snippets/functions/layout/knn.md index dae08f95757b9..c834b317bcd3f 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/layout/knn.md +++ b/docs/reference/query-languages/esql/_snippets/functions/layout/knn.md @@ -2,7 +2,7 @@ ## `KNN` [esql-knn] ```{applies_to} -stack: development +stack: preview serverless: preview ``` diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java index 79930ef057837..8588bbffaf0f0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Knn.java @@ -103,7 +103,7 @@ public class Knn extends FullTextFunction description = "Finds the k nearest vectors to a query vector, as measured by a similarity metric. " + "knn function finds nearest vectors through approximate search on indexed dense_vectors or semantic_text fields.", examples = { @Example(file = "knn-function", tag = "knn-function") }, - appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.DEVELOPMENT) } + appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW) } ) public Knn( Source source,