Implement v_hamming

docs/changelog/132959.yaml

@@ -0,0 +1,5 @@
+pr: 132959
+summary: Implement `v_hamming`
+area: ES|QL
+type: feature
+issues: [132056]

x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-hamming.csv-spec

@@ -0,0 +1,103 @@
+ # Tests for hamming similarity function
+
+ similarityWithVectorField
+ required_capability: hamming_vector_similarity_function
+
+// tag::vector-hamming[]
+ from colors
+ | eval similarity = v_hamming(rgb_vector, [0, 255, 255]) 
+ | sort similarity desc, color asc 
+// end::vector-hamming[]
+ | limit 10
+ | keep color, similarity
+ ;
+
+// tag::vector-hamming-result[]
+color:text  | similarity:double
+cyan        | 1.0
+azure       | 0.8333333134651184
+blue        | 0.6666666865348816
+honeydew    | 0.6666666865348816
+lime        | 0.6666666865348816
+mint cream  | 0.6666666865348816
+white       | 0.6666666865348816
+thistle     | 0.625
+lavender    | 0.5833333134651184
+aqua marine | 0.5416666865348816
+// end::vector-hamming-result[] 
+;
+
+similarityAsPartOfExpression
+required_capability: hamming_vector_similarity_function
+
+from colors
+| eval score = round((1 + v_hamming(rgb_vector, [0, 255, 255]) / 2), 3) 
+| sort score desc, color asc 
+| limit 10
+| keep color, score
+;
+
+color:text  | score:double
+cyan        | 1.5
+azure       | 1.417
+blue        | 1.333
+honeydew    | 1.333
+lime        | 1.333
+mint cream  | 1.333
+white       | 1.333
+thistle     | 1.313
+lavender    | 1.292
+aqua marine | 1.271
+;
+
+similarityWithLiteralVectors
+required_capability: hamming_vector_similarity_function
+
+row a = 1
+| eval similarity = round(v_hamming([1, 2, 3], [0, 1, 2]), 3) 
+| keep similarity
+;
+
+similarity:double
+0.833
+;
+
+similarityWithStats
+required_capability: hamming_vector_similarity_function
+
+from colors
+| eval similarity = round(v_hamming(rgb_vector, [0, 255, 255]), 3) 
+| stats avg = round(avg(similarity), 3), min = min(similarity), max = max(similarity)
+;
+
+avg:double | min:double | max:double
+0.445      | 0.0        | 1.0
+;
+
+similarityWithNull
+required_capability: hamming_vector_similarity_function
+required_capability: vector_similarity_functions_support_null
+
+from colors
+| eval similarity = v_hamming(rgb_vector, null) 
+| stats total_null = count(*) where similarity is null
+;
+
+total_null:long
+59
+;
+
+# TODO Need to implement a conversion function to convert a non-foldable row to a dense_vector
+similarityWithRow-Ignore
+required_capability: hamming_vector_similarity_function
+
+row vector = [1, 2, 3] 
+| eval similarity = round(v_hamming(vector, [0, 1, 2]), 3) 
+| sort similarity desc, color asc 
+| limit 10
+| keep color, similarity
+;
+
+similarity:double
+0.978  
+;

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/vector/VectorSimilarityFunctionsIT.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
 import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase;
 import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
+import org.elasticsearch.xpack.esql.expression.function.vector.Hamming;
 import org.elasticsearch.xpack.esql.expression.function.vector.L1Norm;
 import org.elasticsearch.xpack.esql.expression.function.vector.L2Norm;
 import org.elasticsearch.xpack.esql.expression.function.vector.VectorSimilarityFunction.SimilarityEvaluatorFunction;
@@ -56,6 +57,9 @@ public static Iterable<Object[]> parameters() throws Exception {
         if (EsqlCapabilities.Cap.L2_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()) {
             params.add(new Object[] { "v_l2_norm", (SimilarityEvaluatorFunction) L2Norm::calculateSimilarity });
         }
+        if (EsqlCapabilities.Cap.HAMMING_VECTOR_SIMILARITY_FUNCTION.isEnabled()) {
+            params.add(new Object[] { "v_hamming", (SimilarityEvaluatorFunction) Hamming::calculateSimilarity });
+        }
 
         return params;
     }

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -1374,7 +1374,12 @@ public enum Cap {
         /**
          * Support null elements on vector similarity functions
          */
-        VECTOR_SIMILARITY_FUNCTIONS_SUPPORT_NULL;
+        VECTOR_SIMILARITY_FUNCTIONS_SUPPORT_NULL,
+
+        /**
+         * Support for vector Hamming distance.
+         */
+        HAMMING_VECTOR_SIMILARITY_FUNCTION(Build.current().isSnapshot());
 
         private final boolean enabled;
 

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java

@@ -186,6 +186,7 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.util.Delay;
 import org.elasticsearch.xpack.esql.expression.function.vector.CosineSimilarity;
 import org.elasticsearch.xpack.esql.expression.function.vector.DotProduct;
+import org.elasticsearch.xpack.esql.expression.function.vector.Hamming;
 import org.elasticsearch.xpack.esql.expression.function.vector.Knn;
 import org.elasticsearch.xpack.esql.expression.function.vector.L1Norm;
 import org.elasticsearch.xpack.esql.expression.function.vector.L2Norm;
@@ -507,7 +508,8 @@ private static FunctionDefinition[][] snapshotFunctions() {
                 def(DotProduct.class, DotProduct::new, "v_dot_product"),
                 def(L1Norm.class, L1Norm::new, "v_l1_norm"),
                 def(L2Norm.class, L2Norm::new, "v_l2_norm"),
-                def(Magnitude.class, Magnitude::new, "v_magnitude") } };
+                def(Magnitude.class, Magnitude::new, "v_magnitude"),
+                def(Hamming.class, Hamming::new, "v_hamming") } };
     }
 
     public EsqlFunctionRegistry snapshotRegistry() {

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Hamming.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.vector;
+
+import org.apache.lucene.util.VectorUtil;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.Example;
+import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesTo;
+import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesToLifecycle;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
+
+import java.io.IOException;
+
+public class Hamming extends VectorSimilarityFunction {
+
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Hamming", Hamming::new);
+    static final SimilarityEvaluatorFunction SIMILARITY_FUNCTION = Hamming::calculateSimilarity;
+
+    @FunctionInfo(
+        returnType = "double",
+        preview = true,
+        description = "Calculates the hamming distance between two dense_vectors.",
+        examples = { @Example(file = "vector-hamming", tag = "vector-hamming") },
+        appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.DEVELOPMENT) }
+    )
+    public Hamming(
+        Source source,
+        @Param(
+            name = "left",
+            type = { "dense_vector" },
+            description = "first dense_vector to calculate hamming distance between"
+        ) Expression left,
+        @Param(
+            name = "right",
+            type = { "dense_vector" },
+            description = "second dense_vector to calculate hamming distance between"
+        ) Expression right
+    ) {
+        super(source, left, right);
+    }
+
+    private Hamming(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    protected SimilarityEvaluatorFunction getSimilarityFunction() {
+        return SIMILARITY_FUNCTION;
+    }
+
+    @Override
+    protected BinaryScalarFunction replaceChildren(Expression newLeft, Expression newRight) {
+        return new Hamming(source(), newLeft, newRight);
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Hamming::new, left(), right());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
+    public static float calculateSimilarity(float[] leftScratch, float[] rightScratch) {
+        byte[] a = new byte[leftScratch.length];
+        byte[] b = new byte[rightScratch.length];
+        for (int i = 0; i < leftScratch.length; i++) {
+            a[i] = (byte) leftScratch[i];
+        }
+        for (int i = 0; i < leftScratch.length; i++) {
+            b[i] = (byte) rightScratch[i];
+        }
+        return ((a.length * Byte.SIZE) - VectorUtil.xorBitCount(a, b)) / (float) (a.length * Byte.SIZE);
+    }
+}

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorWritables.java

@@ -45,6 +45,9 @@ public static List<NamedWriteableRegistry.Entry> getNamedWritables() {
         if (EsqlCapabilities.Cap.MAGNITUDE_SCALAR_VECTOR_FUNCTION.isEnabled()) {
             entries.add(Magnitude.ENTRY);
         }
+        if (EsqlCapabilities.Cap.HAMMING_VECTOR_SIMILARITY_FUNCTION.isEnabled()) {
+            entries.add(Hamming.ENTRY);
+        }
 
         return Collections.unmodifiableList(entries);
     }

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java

@@ -2390,6 +2390,13 @@ public void testDenseVectorImplicitCastingSimilarityFunctions() {
             );
             checkDenseVectorImplicitCastingSimilarityFunction("v_l2_norm(float_vector, [1, 2, 3])", List.of(1f, 2f, 3f));
         }
+        if (EsqlCapabilities.Cap.HAMMING_VECTOR_SIMILARITY_FUNCTION.isEnabled()) {
+            checkDenseVectorImplicitCastingSimilarityFunction(
+                "v_hamming(byte_vector, [0.342, 0.164, 0.234])",
+                List.of(0.342f, 0.164f, 0.234f)
+            );
+            checkDenseVectorImplicitCastingSimilarityFunction("v_hamming(byte_vector, [1, 2, 3])", List.of(1f, 2f, 3f));
+        }
     }
 
     private void checkDenseVectorImplicitCastingSimilarityFunction(String similarityFunction, List<Number> expectedElems) {
@@ -2422,6 +2429,9 @@ public void testNoDenseVectorFailsSimilarityFunction() {
         if (EsqlCapabilities.Cap.L2_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()) {
             checkNoDenseVectorFailsSimilarityFunction("v_l2_norm([0, 1, 2], 0.342)");
         }
+        if (EsqlCapabilities.Cap.HAMMING_VECTOR_SIMILARITY_FUNCTION.isEnabled()) {
+            checkNoDenseVectorFailsSimilarityFunction("v_hamming([0, 1, 2], 0.342)");
+        }
     }
 
     private void checkNoDenseVectorFailsSimilarityFunction(String similarityFunction) {

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java

@@ -2318,6 +2318,10 @@ public void testVectorSimilarityFunctionsNullArgs() throws Exception {
         if (EsqlCapabilities.Cap.MAGNITUDE_SCALAR_VECTOR_FUNCTION.isEnabled()) {
             checkVectorFunctionsNullArgs("v_magnitude(null)");
         }
+        if (EsqlCapabilities.Cap.HAMMING_VECTOR_SIMILARITY_FUNCTION.isEnabled()) {
+            checkVectorFunctionsNullArgs("v_hamming(null, vector)");
+            checkVectorFunctionsNullArgs("v_hamming(vector, null)");
+        }
     }
 
     private void checkVectorFunctionsNullArgs(String functionInvocation) throws Exception {