Hash functions

docs/changelog/118938.yaml

@@ -0,0 +1,5 @@
+pr: 118938
+summary: Hash functions
+area: ES|QL
+type: enhancement
+issues: []

docs/reference/esql/functions/string-functions.asciidoc

@@ -43,6 +43,7 @@ include::layout/left.asciidoc[]
 include::layout/length.asciidoc[]
 include::layout/locate.asciidoc[]
 include::layout/ltrim.asciidoc[]
+include::layout/md5.asciidoc[]
 include::layout/repeat.asciidoc[]
 include::layout/replace.asciidoc[]
 include::layout/reverse.asciidoc[]

x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec

@@ -103,3 +103,12 @@ FROM sample_data
 count:long | hash(md5, message):keyword
 3          | 2e92ae79ff32b37fee4368a594792183
 ;
+
+md5Hash
+required_capability: short_hash_functions
+
+ROW input="input" | EVAL md5 = hash("md5", input);
+
+input:keyword | md5:keyword
+input         | a43c1b0aa53a0c908810c06ab1ff3967
+;

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -458,6 +458,10 @@ public enum Cap {
          * Hash function
          */
         HASH_FUNCTION,
+        /**
+         * Hash functions such as MD5
+         */
+        SHORT_HASH_FUNCTIONS,
 
         /**
          * Don't optimize CASE IS NOT NULL function by not requiring the fields to be not null as well.

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java

@@ -134,6 +134,7 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Left;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Locate;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.Md5;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.RTrim;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Repeat;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Replace;
@@ -333,6 +334,7 @@ private static FunctionDefinition[][] functions() {
                 def(Left.class, Left::new, "left"),
                 def(Length.class, Length::new, "length"),
                 def(Locate.class, Locate::new, "locate"),
+                def(Md5.class, Md5::new, "md5"),
                 def(RTrim.class, RTrim::new, "rtrim"),
                 def(Repeat.class, Repeat::new, "repeat"),
                 def(Replace.class, Replace::new, "replace"),

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java

@@ -37,6 +37,7 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Hash;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Left;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Locate;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.Md5;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Repeat;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Replace;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.Reverse;
@@ -79,6 +80,7 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         entries.add(Left.ENTRY);
         entries.add(Locate.ENTRY);
         entries.add(Log.ENTRY);
+        entries.add(Md5.ENTRY);
         entries.add(Now.ENTRY);
         entries.add(Or.ENTRY);
         entries.add(Pi.ENTRY);

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractHashFunction.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.Hash.HashFunction;
+
+import java.io.IOException;
+import java.util.function.Function;
+
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
+
+public abstract class AbstractHashFunction extends UnaryScalarFunction {
+
+    protected AbstractHashFunction(Source source, Expression field) {
+        super(source, field);
+    }
+
+    protected AbstractHashFunction(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    protected abstract HashFunction getHashFunction();
+
+    @Override
+    public DataType dataType() {
+        return DataType.KEYWORD;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (childrenResolved() == false) {
+            return new TypeResolution("Unresolved children");
+        }
+        return isString(field, sourceText(), DEFAULT);
+    }
+
+    @Override
+    public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
+        return new HashConstantEvaluator.Factory(
+            source(),
+            context -> new BreakingBytesRefBuilder(context.breaker(), "hash"),
+            new Function<>() {
+                @Override
+                public HashFunction apply(DriverContext context) {
+                    return getHashFunction().copy();
+                }
+
+                @Override
+                public String toString() {
+                    return getHashFunction().toString();
+                }
+            },
+            toEvaluator.apply(field)
+        );
+    }
+}

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java

@@ -186,10 +186,18 @@ protected NodeInfo<? extends Expression> info() {
 
     public record HashFunction(String algorithm, MessageDigest digest) {
 
+        public static HashFunction create(String algorithm) {
+            try {
+                return new HashFunction(algorithm, MessageDigest.getInstance(algorithm));
+            } catch (NoSuchAlgorithmException e) {
+                assert false : "Expected to create a valid hashing algorithm";
+                throw new IllegalStateException(e);
+            }
+        }
+
         public static HashFunction create(BytesRef literal) throws NoSuchAlgorithmException {
             var algorithm = literal.utf8ToString();
-            var digest = MessageDigest.getInstance(algorithm);
-            return new HashFunction(algorithm, digest);
+            return new HashFunction(algorithm, MessageDigest.getInstance(algorithm));
         }
 
         public HashFunction copy() {

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.Param;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.Hash.HashFunction;
+
+import java.io.IOException;
+import java.util.List;
+
+public class Md5 extends AbstractHashFunction {
+
+    public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MD5", Md5::new);
+
+    private static final HashFunction MD5 = HashFunction.create("MD5");
+
+    @FunctionInfo(returnType = "keyword", description = "Computes MD5 hash of the input.")
+    public Md5(Source source, @Param(name = "input", type = { "keyword", "text" }, description = "Input to hash.") Expression input) {
+        super(source, input);
+    }
+
+    private Md5(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    protected HashFunction getHashFunction() {
+        return MD5;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ENTRY.name;
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        return new Md5(source(), field);
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Md5::new, field);
+    }
+}

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java

@@ -87,12 +87,28 @@ private static TestCaseSupplier createTestCase(String algorithm, boolean forceLi
         });
     }
 
+    static List<TestCaseSupplier> createHashFunctionTestCases(String algorithm) {
+        return List.of(createHashFunctionTestCase(algorithm, DataType.KEYWORD), createHashFunctionTestCase(algorithm, DataType.TEXT));
+    }
+
+    private static TestCaseSupplier createHashFunctionTestCase(String algorithm, DataType inputType) {
+        return new TestCaseSupplier(algorithm + " with " + inputType, List.of(inputType), () -> {
+            var input = randomFrom(TestCaseSupplier.stringCases(inputType)).get();
+            return new TestCaseSupplier.TestCase(
+                List.of(input),
+                "HashConstantEvaluator[algorithm=" + algorithm + ", input=Attribute[channel=0]]",
+                DataType.KEYWORD,
+                equalTo(new BytesRef(HashTests.hash(algorithm, BytesRefs.toString(input.data()))))
+            );
+        });
+    }
+
     private static TestCaseSupplier.TypedData createTypedData(String value, boolean forceLiteral, DataType type, String name) {
         var data = new TestCaseSupplier.TypedData(new BytesRef(value), type, name);
         return forceLiteral ? data.forceLiteral() : data;
     }
 
-    private static String hash(String algorithm, String input) {
+    static String hash(String algorithm, String input) {
         try {
             return HexFormat.of().formatHex(MessageDigest.getInstance(algorithm).digest(input.getBytes(StandardCharsets.UTF_8)));
         } catch (NoSuchAlgorithmException e) {

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5SerializationTests.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests;
+
+import java.io.IOException;
+
+public class Md5SerializationTests extends AbstractExpressionSerializationTests<Md5> {
+
+    @Override
+    protected Md5 createTestInstance() {
+        return new Md5(randomSource(), randomChild());
+    }
+
+    @Override
+    protected Md5 mutateInstance(Md5 instance) throws IOException {
+        return new Md5(instance.source(), mutateExpression(instance.field()));
+    }
+}

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.string;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
+import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+
+public class Md5Tests extends AbstractScalarFunctionTestCase {
+
+    public Md5Tests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
+        this.testCase = testCaseSupplier.get();
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        List<TestCaseSupplier> cases = new ArrayList<>();
+        cases.addAll(HashTests.createHashFunctionTestCases("MD5"));
+        return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "string");
+    }
+
+    @Override
+    protected Expression build(Source source, List<Expression> args) {
+        return new Md5(source, args.get(0));
+    }
+}