diff --git a/docs/reference/query-languages/esql/_snippets/functions/description/v_l2_norm.md b/docs/reference/query-languages/esql/_snippets/functions/description/v_l2_norm.md new file mode 100644 index 0000000000000..5f02d973970ad --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/description/v_l2_norm.md @@ -0,0 +1,6 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Description** + +Calculates the l2 norm between two dense_vectors. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/v_l2_norm.md b/docs/reference/query-languages/esql/_snippets/functions/examples/v_l2_norm.md new file mode 100644 index 0000000000000..9d111420067a0 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/v_l2_norm.md @@ -0,0 +1,24 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Example** + +```esql + from colors + | eval similarity = v_l2_norm(rgb_vector, [0, 255, 255]) + | sort similarity desc, color asc +``` + +| color:text | similarity:double | +| --- | --- | +| red | 441.6729431152344 | +| maroon | 382.6669616699219 | +| crimson | 376.36419677734375 | +| orange | 371.68536376953125 | +| gold | 362.8360595703125 | +| black | 360.62445068359375 | +| magenta | 360.62445068359375 | +| yellow | 360.62445068359375 | +| firebrick | 359.67486572265625 | +| tomato | 351.0227966308594 | + + diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/v_l2_norm.md b/docs/reference/query-languages/esql/_snippets/functions/layout/v_l2_norm.md new file mode 100644 index 0000000000000..8cca7903f1a14 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/layout/v_l2_norm.md @@ -0,0 +1,27 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +## `V_L2_NORM` [esql-v_l2_norm] +```{applies_to} +stack: development +serverless: preview +``` + +**Syntax** + +:::{image} ../../../images/functions/v_l2_norm.svg +:alt: Embedded +:class: text-center +::: + + +:::{include} ../parameters/v_l2_norm.md +::: + +:::{include} ../description/v_l2_norm.md +::: + +:::{include} ../types/v_l2_norm.md +::: + +:::{include} ../examples/v_l2_norm.md +::: diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/v_l2_norm.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/v_l2_norm.md new file mode 100644 index 0000000000000..dc40de91316a3 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/parameters/v_l2_norm.md @@ -0,0 +1,10 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Parameters** + +`left` +: first dense_vector to calculate l2 norm similarity + +`right` +: second dense_vector to calculate l2 norm similarity + diff --git a/docs/reference/query-languages/esql/images/functions/v_l2_norm.svg b/docs/reference/query-languages/esql/images/functions/v_l2_norm.svg new file mode 100644 index 0000000000000..ccae0c9ff1872 --- /dev/null +++ b/docs/reference/query-languages/esql/images/functions/v_l2_norm.svg @@ -0,0 +1 @@ +V_L2_NORM(left,right) \ No newline at end of file diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/v_l2_norm.json b/docs/reference/query-languages/esql/kibana/definition/functions/v_l2_norm.json new file mode 100644 index 0000000000000..573de7891d893 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/functions/v_l2_norm.json @@ -0,0 +1,12 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "scalar", + "name" : "v_l2_norm", + "description" : "Calculates the l2 norm between two dense_vectors.", + "signatures" : [ ], + "examples" : [ + " from colors\n | eval similarity = v_l2_norm(rgb_vector, [0, 255, 255])\n | sort similarity desc, color asc" + ], + "preview" : true, + "snapshot_only" : true +} diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/v_l2_norm.md b/docs/reference/query-languages/esql/kibana/docs/functions/v_l2_norm.md new file mode 100644 index 0000000000000..519113f7351fb --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/docs/functions/v_l2_norm.md @@ -0,0 +1,10 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +### V L2 NORM +Calculates the l2 norm between two dense_vectors. + +```esql + from colors + | eval similarity = v_l2_norm(rgb_vector, [0, 255, 255]) + | sort similarity desc, color asc +``` diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-l2-norm.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-l2-norm.csv-spec new file mode 100644 index 0000000000000..c623a21ca6885 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-l2-norm.csv-spec @@ -0,0 +1,90 @@ + # Tests for l2_norm similarity function + + similarityWithVectorField + required_capability: l2_norm_vector_similarity_function + +// tag::vector-l2-norm[] + from colors + | eval similarity = v_l2_norm(rgb_vector, [0, 255, 255]) + | sort similarity desc, color asc +// end::vector-l2-norm[] + | limit 10 + | keep color, similarity + ; + +// tag::vector-l2-norm-result[] +color:text | similarity:double +red | 441.6729431152344 +maroon | 382.6669616699219 +crimson | 376.36419677734375 +orange | 371.68536376953125 +gold | 362.8360595703125 +black | 360.62445068359375 +magenta | 360.62445068359375 +yellow | 360.62445068359375 +firebrick | 359.67486572265625 +tomato | 351.0227966308594 +// end::vector-l2-norm-result[] +; + + similarityAsPartOfExpression + required_capability: l2_norm_vector_similarity_function + + from colors + | eval score = round((1 + v_l2_norm(rgb_vector, [0, 255, 255]) / 2), 3) + | sort score desc, color asc + | limit 10 + | keep color, score + ; + +color:text | score:double +red | 221.836 +maroon | 192.333 +crimson | 189.182 +orange | 186.843 +gold | 182.418 +black | 181.312 +magenta | 181.312 +yellow | 181.312 +firebrick | 180.837 +tomato | 176.511 +; + +similarityWithLiteralVectors +required_capability: l2_norm_vector_similarity_function + +row a = 1 +| eval similarity = round(v_l2_norm([1, 2, 3], [0, 1, 2]), 3) +| keep similarity +; + +similarity:double +1.732 +; + + similarityWithStats + required_capability: l2_norm_vector_similarity_function + + from colors + | eval similarity = round(v_l2_norm(rgb_vector, [0, 255, 255]), 3) + | stats avg = round(avg(similarity), 3), min = min(similarity), max = max(similarity) + ; + +avg:double | min:double | max:double +274.974 | 0.0 | 441.673 +; + +# TODO Need to implement a conversion function to convert a non-foldable row to a dense_vector +similarityWithRow-Ignore +required_capability: l2_norm_vector_similarity_function + +row vector = [1, 2, 3] +| eval similarity = round(v_l2_norm(vector, [0, 1, 2]), 3) +| sort similarity desc, color asc +| limit 10 +| keep color, similarity +; + +similarity:double +0.978 +; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/vector/VectorSimilarityFunctionsIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/vector/VectorSimilarityFunctionsIT.java index debe448834f8d..024763ba690db 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/vector/VectorSimilarityFunctionsIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/vector/VectorSimilarityFunctionsIT.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.expression.function.vector.L1Norm; +import org.elasticsearch.xpack.esql.expression.function.vector.L2Norm; import org.elasticsearch.xpack.esql.expression.function.vector.VectorSimilarityFunction.SimilarityEvaluatorFunction; import org.junit.Before; @@ -47,6 +48,9 @@ public static Iterable parameters() throws Exception { if (EsqlCapabilities.Cap.L1_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { params.add(new Object[] { "v_l1_norm", (SimilarityEvaluatorFunction) L1Norm::calculateSimilarity }); } + if (EsqlCapabilities.Cap.L2_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { + params.add(new Object[] { "v_l2_norm", (SimilarityEvaluatorFunction) L2Norm::calculateSimilarity }); + } return params; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 8aa224f6fc875..f09a5209d6a18 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1317,6 +1317,11 @@ public enum Cap { */ L1_NORM_VECTOR_SIMILARITY_FUNCTION(Build.current().isSnapshot()), + /** + * l2 norm vector similarity function + */ + L2_NORM_VECTOR_SIMILARITY_FUNCTION(Build.current().isSnapshot()), + /** * Support for the options field of CATEGORIZE. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 29475f7a8dce1..649503b1443d2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -184,6 +184,7 @@ import org.elasticsearch.xpack.esql.expression.function.vector.DotProduct; import org.elasticsearch.xpack.esql.expression.function.vector.Knn; import org.elasticsearch.xpack.esql.expression.function.vector.L1Norm; +import org.elasticsearch.xpack.esql.expression.function.vector.L2Norm; import org.elasticsearch.xpack.esql.parser.ParsingException; import org.elasticsearch.xpack.esql.session.Configuration; @@ -495,7 +496,8 @@ private static FunctionDefinition[][] snapshotFunctions() { def(StGeohexToString.class, StGeohexToString::new, "st_geohex_to_string"), def(CosineSimilarity.class, CosineSimilarity::new, "v_cosine"), def(DotProduct.class, DotProduct::new, "v_dot_product"), - def(L1Norm.class, L1Norm::new, "v_l1_norm") } }; + def(L1Norm.class, L1Norm::new, "v_l1_norm"), + def(L2Norm.class, L2Norm::new, "v_l2_norm") } }; } public EsqlFunctionRegistry snapshotRegistry() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/L2Norm.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/L2Norm.java new file mode 100644 index 0000000000000..cbaa50704c8c6 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/L2Norm.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.vector; + +import org.apache.lucene.util.VectorUtil; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesTo; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesToLifecycle; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; + +import java.io.IOException; + +public class L2Norm extends VectorSimilarityFunction { + + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "L2Norm", L2Norm::new); + static final SimilarityEvaluatorFunction SIMILARITY_FUNCTION = L2Norm::calculateSimilarity; + + @FunctionInfo( + returnType = "double", + preview = true, + description = "Calculates the l2 norm between two dense_vectors.", + examples = { @Example(file = "vector-l2-norm", tag = "vector-l2-norm") }, + appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.DEVELOPMENT) } + ) + public L2Norm( + Source source, + @Param( + name = "left", + type = { "dense_vector" }, + description = "first dense_vector to calculate l2 norm similarity" + ) Expression left, + @Param( + name = "right", + type = { "dense_vector" }, + description = "second dense_vector to calculate l2 norm similarity" + ) Expression right + ) { + super(source, left, right); + } + + private L2Norm(StreamInput in) throws IOException { + super(in); + } + + @Override + protected BinaryScalarFunction replaceChildren(Expression newLeft, Expression newRight) { + return new L2Norm(source(), newLeft, newRight); + } + + @Override + protected SimilarityEvaluatorFunction getSimilarityFunction() { + return SIMILARITY_FUNCTION; + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, L2Norm::new, left(), right()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + public static float calculateSimilarity(float[] leftScratch, float[] rightScratch) { + return (float) Math.sqrt(VectorUtil.squareDistance(leftScratch, rightScratch)); + } + +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorWritables.java index 1f657a31e26e2..4a1a2ec9386ae 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorWritables.java @@ -39,6 +39,9 @@ public static List getNamedWritables() { if (EsqlCapabilities.Cap.L1_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { entries.add(L1Norm.ENTRY); } + if (EsqlCapabilities.Cap.L2_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { + entries.add(L2Norm.ENTRY); + } return Collections.unmodifiableList(entries); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 2adae459cd735..2a13bdfe60c76 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -2369,6 +2369,10 @@ public void testDenseVectorImplicitCastingSimilarityFunctions() { checkDenseVectorImplicitCastingSimilarityFunction("v_l1_norm(vector, [0.342, 0.164, 0.234])", List.of(0.342f, 0.164f, 0.234f)); checkDenseVectorImplicitCastingSimilarityFunction("v_l1_norm(vector, [1, 2, 3])", List.of(1f, 2f, 3f)); } + if (EsqlCapabilities.Cap.L2_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { + checkDenseVectorImplicitCastingSimilarityFunction("v_l2_norm(vector, [0.342, 0.164, 0.234])", List.of(0.342f, 0.164f, 0.234f)); + checkDenseVectorImplicitCastingSimilarityFunction("v_l2_norm(vector, [1, 2, 3])", List.of(1f, 2f, 3f)); + } } private void checkDenseVectorImplicitCastingSimilarityFunction(String similarityFunction, List expectedElems) { @@ -2398,6 +2402,9 @@ public void testNoDenseVectorFailsSimilarityFunction() { if (EsqlCapabilities.Cap.L1_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { checkNoDenseVectorFailsSimilarityFunction("v_l1_norm([0, 1, 2], 0.342)"); } + if (EsqlCapabilities.Cap.L2_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { + checkNoDenseVectorFailsSimilarityFunction("v_l2_norm([0, 1, 2], 0.342)"); + } } private void checkNoDenseVectorFailsSimilarityFunction(String similarityFunction) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 5f3123bd9574a..3c87818d76852 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -2487,6 +2487,10 @@ public void testVectorSimilarityFunctionsNullArgs() throws Exception { checkVectorSimilarityFunctionsNullArgs("v_l1_norm(null, vector)", "first"); checkVectorSimilarityFunctionsNullArgs("v_l1_norm(vector, null)", "second"); } + if (EsqlCapabilities.Cap.L2_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { + checkVectorSimilarityFunctionsNullArgs("v_l2_norm(null, vector)", "first"); + checkVectorSimilarityFunctionsNullArgs("v_l2_norm(vector, null)", "second"); + } } private void checkVectorSimilarityFunctionsNullArgs(String functionInvocation, String argOrdinal) throws Exception { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/vector/L2NormSimilarityTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/vector/L2NormSimilarityTests.java new file mode 100644 index 0000000000000..2dc7929ffa05e --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/vector/L2NormSimilarityTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.vector; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.List; +import java.util.function.Supplier; + +@FunctionName("v_l2_norm") +public class L2NormSimilarityTests extends AbstractVectorSimilarityFunctionTestCase { + + public L2NormSimilarityTests(@Name("TestCase") Supplier testCaseSupplier) { + super(testCaseSupplier); + } + + @ParametersFactory + public static Iterable parameters() { + return similarityParameters(L2Norm.class.getSimpleName(), L2Norm.SIMILARITY_FUNCTION); + } + + protected EsqlCapabilities.Cap capability() { + return EsqlCapabilities.Cap.L2_NORM_VECTOR_SIMILARITY_FUNCTION; + } + + @Override + protected Expression build(Source source, List args) { + return new L2Norm(source, args.get(0), args.get(1)); + } +}