diff --git a/docs/reference/query-languages/esql/_snippets/functions/description/v_l1_norm.md b/docs/reference/query-languages/esql/_snippets/functions/description/v_l1_norm.md new file mode 100644 index 0000000000000..23697416312a7 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/description/v_l1_norm.md @@ -0,0 +1,6 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Description** + +Calculates the l1 norm between two dense_vectors. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/v_l1_norm.md b/docs/reference/query-languages/esql/_snippets/functions/examples/v_l1_norm.md new file mode 100644 index 0000000000000..b3efac50b7ae7 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/v_l1_norm.md @@ -0,0 +1,9 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Example** + +```esql +null +``` + + diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/v_l1_norm.md b/docs/reference/query-languages/esql/_snippets/functions/layout/v_l1_norm.md new file mode 100644 index 0000000000000..b23e25ce17016 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/layout/v_l1_norm.md @@ -0,0 +1,27 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +## `V_L1_NORM` [esql-v_l1_norm] +```{applies_to} +stack: development +serverless: preview +``` + +**Syntax** + +:::{image} ../../../images/functions/v_l1_norm.svg +:alt: Embedded +:class: text-center +::: + + +:::{include} ../parameters/v_l1_norm.md +::: + +:::{include} ../description/v_l1_norm.md +::: + +:::{include} ../types/v_l1_norm.md +::: + +:::{include} ../examples/v_l1_norm.md +::: diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/v_l1_norm.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/v_l1_norm.md new file mode 100644 index 0000000000000..2919df7816d41 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/parameters/v_l1_norm.md @@ -0,0 +1,10 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Parameters** + +`left` +: first dense_vector to calculate l1 norm similarity + +`right` +: second dense_vector to calculate l1 norm similarity + diff --git a/docs/reference/query-languages/esql/images/functions/v_l1_norm.svg b/docs/reference/query-languages/esql/images/functions/v_l1_norm.svg new file mode 100644 index 0000000000000..58d5e03874662 --- /dev/null +++ b/docs/reference/query-languages/esql/images/functions/v_l1_norm.svg @@ -0,0 +1 @@ +V_L1_NORM(left,right) \ No newline at end of file diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/v_l1_norm.json b/docs/reference/query-languages/esql/kibana/definition/functions/v_l1_norm.json new file mode 100644 index 0000000000000..a46c2c8b0b62f --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/functions/v_l1_norm.json @@ -0,0 +1,12 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "scalar", + "name" : "v_l1_norm", + "description" : "Calculates the l1 norm between two dense_vectors.", + "signatures" : [ ], + "examples" : [ + null + ], + "preview" : true, + "snapshot_only" : true +} diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/v_l1_norm.md b/docs/reference/query-languages/esql/kibana/docs/functions/v_l1_norm.md new file mode 100644 index 0000000000000..40a1b5efbd625 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/docs/functions/v_l1_norm.md @@ -0,0 +1,8 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +### V L1 NORM +Calculates the l1 norm between two dense_vectors. + +```esql +null +``` diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-l1-norm.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-l1-norm.csv-spec new file mode 100644 index 0000000000000..5c85a7e711982 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-l1-norm.csv-spec @@ -0,0 +1,90 @@ + # Tests for l1_norm similarity function + + similarityWithVectorField + required_capability: l1_norm_vector_similarity_function + +// tag::vector-l1-norm-similarity[] + from colors + | eval similarity = v_l1_norm(rgb_vector, [0, 255, 255]) + | sort similarity desc, color asc +// end::vector-l1-norm-similarity[] + | limit 10 + | keep color, similarity + ; + +// tag::vector-l1-norm-similarity-result[] +color:text | similarity:double +red | 765.0 +crimson | 650.0 +maroon | 638.0 +firebrick | 620.0 +orange | 600.0 +tomato | 595.0 +brown | 591.0 +chocolate | 585.0 +coral | 558.0 +gold | 550.0 +// end::vector-l1-norm-similarity-result[] +; + + similarityAsPartOfExpression + required_capability: l1_norm_vector_similarity_function + + from colors + | eval score = round((1 + v_l1_norm(rgb_vector, [0, 255, 255]) / 2), 3) + | sort score desc, color asc + | limit 10 + | keep color, score + ; + +color:text | score:double +red | 383.5 +crimson | 326.0 +maroon | 320.0 +firebrick | 311.0 +orange | 301.0 +tomato | 298.5 +brown | 296.5 +chocolate | 293.5 +coral | 280.0 +gold | 276.0 +; + +similarityWithLiteralVectors +required_capability: l1_norm_vector_similarity_function + +row a = 1 +| eval similarity = round(v_l1_norm([1, 2, 3], [0, 1, 2]), 3) +| keep similarity +; + +similarity:double +3.0 +; + + similarityWithStats + required_capability: l1_norm_vector_similarity_function + + from colors + | eval similarity = round(v_l1_norm(rgb_vector, [0, 255, 255]), 3) + | stats avg = round(avg(similarity), 3), min = min(similarity), max = max(similarity) + ; + +avg:double | min:double | max:double +391.254 | 0.0 | 765.0 +; + +# TODO Need to implement a conversion function to convert a non-foldable row to a dense_vector +similarityWithRow-Ignore +required_capability: l1_norm_vector_similarity_function + +row vector = [1, 2, 3] +| eval similarity = round(v_l1_norm(vector, [0, 1, 2]), 3) +| sort similarity desc, color asc +| limit 10 +| keep color, similarity +; + +similarity:double +0.978 +; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/vector/VectorSimilarityFunctionsIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/vector/VectorSimilarityFunctionsIT.java index 17a1d5e945af3..debe448834f8d 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/vector/VectorSimilarityFunctionsIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/vector/VectorSimilarityFunctionsIT.java @@ -20,6 +20,8 @@ import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.elasticsearch.xpack.esql.expression.function.vector.L1Norm; +import org.elasticsearch.xpack.esql.expression.function.vector.VectorSimilarityFunction.SimilarityEvaluatorFunction; import org.junit.Before; import java.io.IOException; @@ -37,22 +39,25 @@ public static Iterable parameters() throws Exception { List params = new ArrayList<>(); if (EsqlCapabilities.Cap.COSINE_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { - params.add(new Object[] { "v_cosine", VectorSimilarityFunction.COSINE }); + params.add(new Object[] { "v_cosine", (SimilarityEvaluatorFunction) VectorSimilarityFunction.COSINE::compare }); } if (EsqlCapabilities.Cap.DOT_PRODUCT_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { - params.add(new Object[] { "v_dot_product", VectorSimilarityFunction.DOT_PRODUCT }); + params.add(new Object[] { "v_dot_product", (SimilarityEvaluatorFunction) VectorSimilarityFunction.DOT_PRODUCT::compare }); + } + if (EsqlCapabilities.Cap.L1_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { + params.add(new Object[] { "v_l1_norm", (SimilarityEvaluatorFunction) L1Norm::calculateSimilarity }); } return params; } private final String functionName; - private final VectorSimilarityFunction similarityFunction; + private final SimilarityEvaluatorFunction similarityFunction; private int numDims; public VectorSimilarityFunctionsIT( @Name("functionName") String functionName, - @Name("similarityFunction") VectorSimilarityFunction similarityFunction + @Name("similarityFunction") SimilarityEvaluatorFunction similarityFunction ) { this.functionName = functionName; this.similarityFunction = similarityFunction; @@ -74,7 +79,7 @@ public void testSimilarityBetweenVectors() { Double similarity = (Double) values.get(2); assertNotNull(similarity); - float expectedSimilarity = similarityFunction.compare(left, right); + float expectedSimilarity = similarityFunction.calculateSimilarity(left, right); assertEquals(expectedSimilarity, similarity, 0.0001); }); } @@ -96,7 +101,7 @@ public void testSimilarityBetweenConstantVectorAndField() { Double similarity = (Double) values.get(1); assertNotNull(similarity); - float expectedSimilarity = similarityFunction.compare(left, randomVector); + float expectedSimilarity = similarityFunction.calculateSimilarity(left, randomVector); assertEquals(expectedSimilarity, similarity, 0.0001); }); } @@ -130,7 +135,7 @@ public void testSimilarityBetweenConstantVectors() { Double similarity = (Double) valuesList.get(0).get(0); assertNotNull(similarity); - float expectedSimilarity = similarityFunction.compare(vectorLeft, vectorRight); + float expectedSimilarity = similarityFunction.calculateSimilarity(vectorLeft, vectorRight); assertEquals(expectedSimilarity, similarity, 0.0001); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index def103def27bd..5491ba58887f7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1296,6 +1296,11 @@ public enum Cap { */ DOT_PRODUCT_VECTOR_SIMILARITY_FUNCTION(Build.current().isSnapshot()), + /** + * l1 norm vector similarity function + */ + L1_NORM_VECTOR_SIMILARITY_FUNCTION(Build.current().isSnapshot()), + /** * Support for the options field of CATEGORIZE. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index b2fa2a13f6710..707de5b31ffb7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -183,6 +183,7 @@ import org.elasticsearch.xpack.esql.expression.function.vector.CosineSimilarity; import org.elasticsearch.xpack.esql.expression.function.vector.DotProduct; import org.elasticsearch.xpack.esql.expression.function.vector.Knn; +import org.elasticsearch.xpack.esql.expression.function.vector.L1Norm; import org.elasticsearch.xpack.esql.parser.ParsingException; import org.elasticsearch.xpack.esql.session.Configuration; @@ -493,7 +494,8 @@ private static FunctionDefinition[][] snapshotFunctions() { def(StGeohexToLong.class, StGeohexToLong::new, "st_geohex_to_long"), def(StGeohexToString.class, StGeohexToString::new, "st_geohex_to_string"), def(CosineSimilarity.class, CosineSimilarity::new, "v_cosine"), - def(DotProduct.class, DotProduct::new, "v_dot_product") } }; + def(DotProduct.class, DotProduct::new, "v_dot_product"), + def(L1Norm.class, L1Norm::new, "v_l1_norm") } }; } public EsqlFunctionRegistry snapshotRegistry() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/L1Norm.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/L1Norm.java new file mode 100644 index 0000000000000..ee8eabe0c8a1a --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/L1Norm.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.vector; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesTo; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesToLifecycle; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; + +import java.io.IOException; + +public class L1Norm extends VectorSimilarityFunction { + + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "L1Norm", L1Norm::new); + static final SimilarityEvaluatorFunction SIMILARITY_FUNCTION = L1Norm::calculateSimilarity; + + @FunctionInfo( + returnType = "double", + preview = true, + description = "Calculates the l1 norm between two dense_vectors.", + examples = { @Example(file = "vector-l1-norm", tag = "vector-l1-norm") }, + appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.DEVELOPMENT) } + ) + public L1Norm( + Source source, + @Param( + name = "left", + type = { "dense_vector" }, + description = "first dense_vector to calculate l1 norm similarity" + ) Expression left, + @Param( + name = "right", + type = { "dense_vector" }, + description = "second dense_vector to calculate l1 norm similarity" + ) Expression right + ) { + super(source, left, right); + } + + private L1Norm(StreamInput in) throws IOException { + super(in); + } + + @Override + protected BinaryScalarFunction replaceChildren(Expression newLeft, Expression newRight) { + return new L1Norm(source(), newLeft, newRight); + } + + @Override + protected SimilarityEvaluatorFunction getSimilarityFunction() { + return SIMILARITY_FUNCTION; + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, L1Norm::new, left(), right()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + public static float calculateSimilarity(float[] leftScratch, float[] rightScratch) { + float result = 0f; + for (int i = 0; i < leftScratch.length; i++) { + result += Math.abs(leftScratch[i] - rightScratch[i]); + } + return result; + } + +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorWritables.java index d44c5681438b0..1f657a31e26e2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorWritables.java @@ -36,6 +36,9 @@ public static List getNamedWritables() { if (EsqlCapabilities.Cap.DOT_PRODUCT_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { entries.add(DotProduct.ENTRY); } + if (EsqlCapabilities.Cap.L1_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { + entries.add(L1Norm.ENTRY); + } return Collections.unmodifiableList(entries); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index e5f8b34298295..595b7699bdbd7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -2365,6 +2365,10 @@ public void testDenseVectorImplicitCastingSimilarityFunctions() { ); checkDenseVectorImplicitCastingSimilarityFunction("v_dot_product(vector, [1, 2, 3])", List.of(1f, 2f, 3f)); } + if (EsqlCapabilities.Cap.L1_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { + checkDenseVectorImplicitCastingSimilarityFunction("v_l1_norm(vector, [0.342, 0.164, 0.234])", List.of(0.342f, 0.164f, 0.234f)); + checkDenseVectorImplicitCastingSimilarityFunction("v_l1_norm(vector, [1, 2, 3])", List.of(1f, 2f, 3f)); + } } private void checkDenseVectorImplicitCastingSimilarityFunction(String similarityFunction, List expectedElems) { @@ -2391,6 +2395,9 @@ public void testNoDenseVectorFailsSimilarityFunction() { if (EsqlCapabilities.Cap.DOT_PRODUCT_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { checkNoDenseVectorFailsSimilarityFunction("v_dot_product([0, 1, 2], 0.342)"); } + if (EsqlCapabilities.Cap.L1_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { + checkNoDenseVectorFailsSimilarityFunction("v_l1_norm([0, 1, 2], 0.342)"); + } } private void checkNoDenseVectorFailsSimilarityFunction(String similarityFunction) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 9b3ffcb242e93..52271c7ee263a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -2508,6 +2508,10 @@ public void testVectorSimilarityFunctionsNullArgs() throws Exception { checkVectorSimilarityFunctionsNullArgs("v_dot_product(null, vector)", "first"); checkVectorSimilarityFunctionsNullArgs("v_dot_product(vector, null)", "second"); } + if (EsqlCapabilities.Cap.L1_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { + checkVectorSimilarityFunctionsNullArgs("v_l1_norm(null, vector)", "first"); + checkVectorSimilarityFunctionsNullArgs("v_l1_norm(vector, null)", "second"); + } } private void checkVectorSimilarityFunctionsNullArgs(String functionInvocation, String argOrdinal) throws Exception { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/vector/L1NormSimilarityTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/vector/L1NormSimilarityTests.java new file mode 100644 index 0000000000000..149f5a1cd650a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/vector/L1NormSimilarityTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.vector; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.List; +import java.util.function.Supplier; + +@FunctionName("v_l1_norm") +public class L1NormSimilarityTests extends AbstractVectorSimilarityFunctionTestCase { + + public L1NormSimilarityTests(@Name("TestCase") Supplier testCaseSupplier) { + super(testCaseSupplier); + } + + @ParametersFactory + public static Iterable parameters() { + return similarityParameters(L1Norm.class.getSimpleName(), L1Norm.SIMILARITY_FUNCTION); + } + + protected EsqlCapabilities.Cap capability() { + return EsqlCapabilities.Cap.L1_NORM_VECTOR_SIMILARITY_FUNCTION; + } + + @Override + protected Expression build(Source source, List args) { + return new L1Norm(source, args.get(0), args.get(1)); + } +}