From c3e18d5fa6fb35d06a027abc5ddc5ce0eb46ee8e Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Wed, 18 Dec 2024 11:58:11 +0100 Subject: [PATCH 01/14] Hash functions --- .../esql/functions/description/md5.asciidoc | 5 ++ .../esql/functions/kibana/definition/md5.json | 34 +++++++++ .../esql/functions/kibana/docs/md5.md | 7 ++ .../esql/functions/layout/md5.asciidoc | 14 ++++ .../esql/functions/parameters/md5.asciidoc | 6 ++ .../esql/functions/signature/md5.svg | 1 + .../esql/functions/string-functions.asciidoc | 1 + .../esql/functions/types/md5.asciidoc | 10 +++ .../src/main/resources/hash.csv-spec | 9 +++ .../xpack/esql/action/EsqlCapabilities.java | 4 ++ .../function/EsqlFunctionRegistry.java | 2 + .../scalar/ScalarFunctionWritables.java | 2 + .../scalar/string/AbstractHashFunction.java | 70 +++++++++++++++++++ .../function/scalar/string/Hash.java | 12 +++- .../function/scalar/string/Md5.java | 56 +++++++++++++++ .../function/scalar/string/HashTests.java | 18 ++++- .../scalar/string/Md5SerializationTests.java | 25 +++++++ .../function/scalar/string/Md5Tests.java | 39 +++++++++++ 18 files changed, 312 insertions(+), 3 deletions(-) create mode 100644 docs/reference/esql/functions/description/md5.asciidoc create mode 100644 docs/reference/esql/functions/kibana/definition/md5.json create mode 100644 docs/reference/esql/functions/kibana/docs/md5.md create mode 100644 docs/reference/esql/functions/layout/md5.asciidoc create mode 100644 docs/reference/esql/functions/parameters/md5.asciidoc create mode 100644 docs/reference/esql/functions/signature/md5.svg create mode 100644 docs/reference/esql/functions/types/md5.asciidoc create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractHashFunction.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5SerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java diff --git a/docs/reference/esql/functions/description/md5.asciidoc b/docs/reference/esql/functions/description/md5.asciidoc new file mode 100644 index 0000000000000..2d6d8515b6caf --- /dev/null +++ b/docs/reference/esql/functions/description/md5.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Computes MD5 hash of the input. diff --git a/docs/reference/esql/functions/kibana/definition/md5.json b/docs/reference/esql/functions/kibana/definition/md5.json new file mode 100644 index 0000000000000..f39610b62a039 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/md5.json @@ -0,0 +1,34 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "md5", + "description" : "Computes MD5 hash of the input.", + "signatures" : [ + { + "params" : [ + { + "name" : "input", + "type" : "keyword", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "input", + "type" : "text", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + } + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/docs/md5.md b/docs/reference/esql/functions/kibana/docs/md5.md new file mode 100644 index 0000000000000..e040d17897f1a --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/md5.md @@ -0,0 +1,7 @@ + + +### MD5 +Computes MD5 hash of the input. + diff --git a/docs/reference/esql/functions/layout/md5.asciidoc b/docs/reference/esql/functions/layout/md5.asciidoc new file mode 100644 index 0000000000000..23f443c1c38b0 --- /dev/null +++ b/docs/reference/esql/functions/layout/md5.asciidoc @@ -0,0 +1,14 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-md5]] +=== `MD5` + +*Syntax* + +[.text-center] +image::esql/functions/signature/md5.svg[Embedded,opts=inline] + +include::../parameters/md5.asciidoc[] +include::../description/md5.asciidoc[] +include::../types/md5.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/md5.asciidoc b/docs/reference/esql/functions/parameters/md5.asciidoc new file mode 100644 index 0000000000000..99eba4dc2cb3d --- /dev/null +++ b/docs/reference/esql/functions/parameters/md5.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`input`:: +Input to hash. diff --git a/docs/reference/esql/functions/signature/md5.svg b/docs/reference/esql/functions/signature/md5.svg new file mode 100644 index 0000000000000..419af764a212e --- /dev/null +++ b/docs/reference/esql/functions/signature/md5.svg @@ -0,0 +1 @@ +MD5(input) \ No newline at end of file diff --git a/docs/reference/esql/functions/string-functions.asciidoc b/docs/reference/esql/functions/string-functions.asciidoc index da9580a55151a..4b81f247b7276 100644 --- a/docs/reference/esql/functions/string-functions.asciidoc +++ b/docs/reference/esql/functions/string-functions.asciidoc @@ -43,6 +43,7 @@ include::layout/left.asciidoc[] include::layout/length.asciidoc[] include::layout/locate.asciidoc[] include::layout/ltrim.asciidoc[] +include::layout/md5.asciidoc[] include::layout/repeat.asciidoc[] include::layout/replace.asciidoc[] include::layout/reverse.asciidoc[] diff --git a/docs/reference/esql/functions/types/md5.asciidoc b/docs/reference/esql/functions/types/md5.asciidoc new file mode 100644 index 0000000000000..049a553397bbd --- /dev/null +++ b/docs/reference/esql/functions/types/md5.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +input | result +keyword | keyword +text | keyword +|=== diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec index fcac1e1859c6d..d1983cf12c25d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec @@ -103,3 +103,12 @@ FROM sample_data count:long | hash(md5, message):keyword 3 | 2e92ae79ff32b37fee4368a594792183 ; + +md5Hash +required_capability: short_hash_functions + +ROW input="input" | EVAL md5 = hash("md5", input); + +input:keyword | md5:keyword +input | a43c1b0aa53a0c908810c06ab1ff3967 +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index d6c1539088d47..7dadd02b369b6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -453,6 +453,10 @@ public enum Cap { * Hash function */ HASH_FUNCTION, + /** + * Hash functions such as MD5 + */ + SHORT_HASH_FUNCTIONS, /** * Don't optimize CASE IS NOT NULL function by not requiring the fields to be not null as well. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 908c9c5f197a8..5ba3f58033bc5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -134,6 +134,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.Left; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Locate; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Md5; import org.elasticsearch.xpack.esql.expression.function.scalar.string.RTrim; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Repeat; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Replace; @@ -333,6 +334,7 @@ private static FunctionDefinition[][] functions() { def(Left.class, Left::new, "left"), def(Length.class, Length::new, "length"), def(Locate.class, Locate::new, "locate"), + def(Md5.class, uni(Md5::new), "md5"), def(RTrim.class, RTrim::new, "rtrim"), def(Repeat.class, Repeat::new, "repeat"), def(Replace.class, Replace::new, "replace"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java index 3cf0eef9074ad..1e1728d03fcc5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java @@ -37,6 +37,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.Hash; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Left; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Locate; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Md5; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Repeat; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Replace; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Reverse; @@ -79,6 +80,7 @@ public static List getNamedWriteables() { entries.add(Left.ENTRY); entries.add(Locate.ENTRY); entries.add(Log.ENTRY); + entries.add(Md5.ENTRY); entries.add(Now.ENTRY); entries.add(Or.ENTRY); entries.add(Pi.ENTRY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractHashFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractHashFunction.java new file mode 100644 index 0000000000000..a39f0a4f32db2 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AbstractHashFunction.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Hash.HashFunction; + +import java.io.IOException; +import java.util.function.Function; + +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; + +public abstract class AbstractHashFunction extends UnaryScalarFunction { + + protected AbstractHashFunction(Source source, Expression field) { + super(source, field); + } + + protected AbstractHashFunction(StreamInput in) throws IOException { + super(in); + } + + protected abstract HashFunction getHashFunction(); + + @Override + public DataType dataType() { + return DataType.KEYWORD; + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + return isString(field, sourceText(), DEFAULT); + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + return new HashConstantEvaluator.Factory( + source(), + context -> new BreakingBytesRefBuilder(context.breaker(), "hash"), + new Function<>() { + @Override + public HashFunction apply(DriverContext context) { + return getHashFunction().copy(); + } + + @Override + public String toString() { + return getHashFunction().toString(); + } + }, + toEvaluator.apply(field) + ); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java index 99c5908699ec2..dfe007c708060 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java @@ -186,10 +186,18 @@ protected NodeInfo info() { public record HashFunction(String algorithm, MessageDigest digest) { + public static HashFunction create(String algorithm) { + try { + return new HashFunction(algorithm, MessageDigest.getInstance(algorithm)); + } catch (NoSuchAlgorithmException e) { + assert false : "Expected to create a valid hashing algorithm"; + throw new IllegalStateException(e); + } + } + public static HashFunction create(BytesRef literal) throws NoSuchAlgorithmException { var algorithm = literal.utf8ToString(); - var digest = MessageDigest.getInstance(algorithm); - return new HashFunction(algorithm, digest); + return new HashFunction(algorithm, MessageDigest.getInstance(algorithm)); } public HashFunction copy() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java new file mode 100644 index 0000000000000..eae559c4fffdc --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Hash.HashFunction; + +import java.io.IOException; +import java.util.List; + +public class Md5 extends AbstractHashFunction { + + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MD5", Md5::new); + + private static final HashFunction MD5 = HashFunction.create("MD5"); + + @FunctionInfo(returnType = "keyword", description = "Computes MD5 hash of the input.") + public Md5(Source source, @Param(name = "input", type = { "keyword", "text" }, description = "Input to hash.") Expression input) { + super(source, input); + } + + public Md5(StreamInput in) throws IOException { + super(in); + } + + @Override + protected HashFunction getHashFunction() { + return MD5; + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new Md5(source(), field); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Md5::new, field); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java index c5cdf97eccd17..b16742292e24e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java @@ -87,12 +87,28 @@ private static TestCaseSupplier createTestCase(String algorithm, boolean forceLi }); } + static List createHashFunctionTestCases(String algorithm) { + return List.of(createHashFunctionTestCase(algorithm, DataType.KEYWORD), createHashFunctionTestCase(algorithm, DataType.TEXT)); + } + + private static TestCaseSupplier createHashFunctionTestCase(String algorithm, DataType inputType) { + return new TestCaseSupplier(algorithm + " with " + inputType, List.of(inputType), () -> { + var input = randomFrom(TestCaseSupplier.stringCases(inputType)).get(); + return new TestCaseSupplier.TestCase( + List.of(input), + "HashConstantEvaluator[algorithm=" + algorithm + ", input=Attribute[channel=0]]", + DataType.KEYWORD, + equalTo(new BytesRef(HashTests.hash(algorithm, BytesRefs.toString(input.data())))) + ); + }); + } + private static TestCaseSupplier.TypedData createTypedData(String value, boolean forceLiteral, DataType type, String name) { var data = new TestCaseSupplier.TypedData(new BytesRef(value), type, name); return forceLiteral ? data.forceLiteral() : data; } - private static String hash(String algorithm, String input) { + static String hash(String algorithm, String input) { try { return HexFormat.of().formatHex(MessageDigest.getInstance(algorithm).digest(input.getBytes(StandardCharsets.UTF_8))); } catch (NoSuchAlgorithmException e) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5SerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5SerializationTests.java new file mode 100644 index 0000000000000..666e5a3ea5d58 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5SerializationTests.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class Md5SerializationTests extends AbstractExpressionSerializationTests { + + @Override + protected Md5 createTestInstance() { + return new Md5(randomSource(), randomChild()); + } + + @Override + protected Md5 mutateInstance(Md5 instance) throws IOException { + return new Md5(instance.source(), mutateExpression(instance.field())); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java new file mode 100644 index 0000000000000..125e412ee5067 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +public class Md5Tests extends AbstractScalarFunctionTestCase { + + public Md5Tests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List cases = new ArrayList<>(); + cases.addAll(HashTests.createHashFunctionTestCases("MD5")); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "string"); + } + + @Override + protected Expression build(Source source, List args) { + return new Md5(source, args.get(0)); + } +} From 3eb01a9ad57063a0a58b79a96e8dc4d3dec749d8 Mon Sep 17 00:00:00 2001 From: Ievgen Degtiarenko Date: Wed, 18 Dec 2024 12:01:23 +0100 Subject: [PATCH 02/14] Update docs/changelog/118938.yaml --- docs/changelog/118938.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/118938.yaml diff --git a/docs/changelog/118938.yaml b/docs/changelog/118938.yaml new file mode 100644 index 0000000000000..395da7912fd4b --- /dev/null +++ b/docs/changelog/118938.yaml @@ -0,0 +1,5 @@ +pr: 118938 +summary: Hash functions +area: ES|QL +type: enhancement +issues: [] From 8d52bc79cde464737262e10401166d588eea2c4f Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Wed, 18 Dec 2024 12:03:18 +0100 Subject: [PATCH 03/14] simplify constructor selection --- .../xpack/esql/expression/function/EsqlFunctionRegistry.java | 2 +- .../xpack/esql/expression/function/scalar/string/Md5.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 5ba3f58033bc5..fd9cddf93c518 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -334,7 +334,7 @@ private static FunctionDefinition[][] functions() { def(Left.class, Left::new, "left"), def(Length.class, Length::new, "length"), def(Locate.class, Locate::new, "locate"), - def(Md5.class, uni(Md5::new), "md5"), + def(Md5.class, Md5::new, "md5"), def(RTrim.class, RTrim::new, "rtrim"), def(Repeat.class, Repeat::new, "repeat"), def(Replace.class, Replace::new, "replace"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java index eae559c4fffdc..1522c4c08669f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java @@ -30,7 +30,7 @@ public Md5(Source source, @Param(name = "input", type = { "keyword", "text" }, d super(source, input); } - public Md5(StreamInput in) throws IOException { + private Md5(StreamInput in) throws IOException { super(in); } From 961eda1af2d428983f9ed7964b870c02cd2e03a1 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Thu, 2 Jan 2025 13:42:25 +0100 Subject: [PATCH 04/14] fix functions count --- .../resources/rest-api-spec/test/esql/60_usage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index b6d75048591e5..8a45049e0872b 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -92,7 +92,7 @@ setup: - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} # Testing for the entire function set isn't feasbile, so we just check that we return the correct count as an approximation. - - length: {esql.functions: 130} # check the "sister" test below for a likely update to the same esql.functions length check + - length: {esql.functions: 131} # check the "sister" test below for a likely update to the same esql.functions length check --- "Basic ESQL usage output (telemetry) non-snapshot version": @@ -163,4 +163,4 @@ setup: - match: {esql.functions.cos: $functions_cos} - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} - - length: {esql.functions: 126} # check the "sister" test above for a likely update to the same esql.functions length check + - length: {esql.functions: 127} # check the "sister" test above for a likely update to the same esql.functions length check From e2877ec471c696f2de187460f6ec8f3f3b7ab02d Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Thu, 2 Jan 2025 14:13:43 +0100 Subject: [PATCH 05/14] fix replace children --- docs/reference/esql/functions/description/md5.asciidoc | 2 +- docs/reference/esql/functions/kibana/definition/md5.json | 2 +- docs/reference/esql/functions/kibana/docs/md5.md | 2 +- .../esql/qa/testFixtures/src/main/resources/hash.csv-spec | 6 +++--- .../xpack/esql/expression/function/scalar/string/Md5.java | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/reference/esql/functions/description/md5.asciidoc b/docs/reference/esql/functions/description/md5.asciidoc index 2d6d8515b6caf..2ad847c0ce0e3 100644 --- a/docs/reference/esql/functions/description/md5.asciidoc +++ b/docs/reference/esql/functions/description/md5.asciidoc @@ -2,4 +2,4 @@ *Description* -Computes MD5 hash of the input. +Computes the MD5 hash of the input. diff --git a/docs/reference/esql/functions/kibana/definition/md5.json b/docs/reference/esql/functions/kibana/definition/md5.json index f39610b62a039..8c3b8ea8defbb 100644 --- a/docs/reference/esql/functions/kibana/definition/md5.json +++ b/docs/reference/esql/functions/kibana/definition/md5.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "md5", - "description" : "Computes MD5 hash of the input.", + "description" : "Computes the MD5 hash of the input.", "signatures" : [ { "params" : [ diff --git a/docs/reference/esql/functions/kibana/docs/md5.md b/docs/reference/esql/functions/kibana/docs/md5.md index e040d17897f1a..a45c9dc6cd2d8 100644 --- a/docs/reference/esql/functions/kibana/docs/md5.md +++ b/docs/reference/esql/functions/kibana/docs/md5.md @@ -3,5 +3,5 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### MD5 -Computes MD5 hash of the input. +Computes the MD5 hash of the input. diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec index d1983cf12c25d..7607169a06117 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec @@ -97,11 +97,11 @@ required_capability: hash_function FROM sample_data | EVAL md5="md5" -| STATS count = count(*) by hash(md5, message) +| STATS count = count(*) by hash(md5, message), md5(message) | WHERE count > 1; -count:long | hash(md5, message):keyword -3 | 2e92ae79ff32b37fee4368a594792183 +count:long | hash(md5, message):keyword | md5(message):keyword +3 | 2e92ae79ff32b37fee4368a594792183 | 2e92ae79ff32b37fee4368a594792183 ; md5Hash diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java index 1522c4c08669f..179ec3a7d56e2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java @@ -25,7 +25,7 @@ public class Md5 extends AbstractHashFunction { private static final HashFunction MD5 = HashFunction.create("MD5"); - @FunctionInfo(returnType = "keyword", description = "Computes MD5 hash of the input.") + @FunctionInfo(returnType = "keyword", description = "Computes the MD5 hash of the input.") public Md5(Source source, @Param(name = "input", type = { "keyword", "text" }, description = "Input to hash.") Expression input) { super(source, input); } @@ -46,7 +46,7 @@ public String getWriteableName() { @Override public Expression replaceChildren(List newChildren) { - return new Md5(source(), field); + return new Md5(source(), newChildren.get(0)); } @Override From 9a9bc757588f08062d0c12b5f80093467f771172 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Thu, 2 Jan 2025 15:05:17 +0100 Subject: [PATCH 06/14] add examples --- .../esql/functions/examples/hash.asciidoc | 13 +++++++++++ .../esql/functions/examples/md5.asciidoc | 13 +++++++++++ .../functions/kibana/definition/hash.json | 3 +++ .../esql/functions/kibana/definition/md5.json | 3 +++ .../esql/functions/kibana/docs/hash.md | 6 +++++ .../esql/functions/kibana/docs/md5.md | 6 +++++ .../esql/functions/layout/hash.asciidoc | 1 + .../esql/functions/layout/md5.asciidoc | 1 + .../src/main/resources/hash.csv-spec | 22 ++++++++++++++++--- .../function/scalar/string/Hash.java | 4 +++- .../function/scalar/string/Md5.java | 7 +++++- 11 files changed, 74 insertions(+), 5 deletions(-) create mode 100644 docs/reference/esql/functions/examples/hash.asciidoc create mode 100644 docs/reference/esql/functions/examples/md5.asciidoc diff --git a/docs/reference/esql/functions/examples/hash.asciidoc b/docs/reference/esql/functions/examples/hash.asciidoc new file mode 100644 index 0000000000000..492e466eb395e --- /dev/null +++ b/docs/reference/esql/functions/examples/hash.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/hash.csv-spec[tag=hash] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/hash.csv-spec[tag=hash-result] +|=== + diff --git a/docs/reference/esql/functions/examples/md5.asciidoc b/docs/reference/esql/functions/examples/md5.asciidoc new file mode 100644 index 0000000000000..0b43bc5b791c9 --- /dev/null +++ b/docs/reference/esql/functions/examples/md5.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/hash.csv-spec[tag=md5] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/hash.csv-spec[tag=md5-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/hash.json b/docs/reference/esql/functions/kibana/definition/hash.json index 17a60cf45acfe..dbf4a2542afc5 100644 --- a/docs/reference/esql/functions/kibana/definition/hash.json +++ b/docs/reference/esql/functions/kibana/definition/hash.json @@ -77,6 +77,9 @@ "returnType" : "keyword" } ], + "examples" : [ + "FROM sample_data \n| WHERE message != \"Connection error\"\n| EVAL md5 = hash(\"md5\", message), sha256 = hash(\"sha256\", message) \n| KEEP message, md5, sha256;" + ], "preview" : false, "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/definition/md5.json b/docs/reference/esql/functions/kibana/definition/md5.json index 8c3b8ea8defbb..4d3a88e123ff4 100644 --- a/docs/reference/esql/functions/kibana/definition/md5.json +++ b/docs/reference/esql/functions/kibana/definition/md5.json @@ -29,6 +29,9 @@ "returnType" : "keyword" } ], + "examples" : [ + "FROM sample_data \n| WHERE message != \"Connection error\"\n| EVAL md5 = md5(message)\n| KEEP message, md5;" + ], "preview" : false, "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/docs/hash.md b/docs/reference/esql/functions/kibana/docs/hash.md index 9826e80ec5bec..4e937778ba67a 100644 --- a/docs/reference/esql/functions/kibana/docs/hash.md +++ b/docs/reference/esql/functions/kibana/docs/hash.md @@ -5,3 +5,9 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ ### HASH Computes the hash of the input using various algorithms such as MD5, SHA, SHA-224, SHA-256, SHA-384, SHA-512. +``` +FROM sample_data +| WHERE message != "Connection error" +| EVAL md5 = hash("md5", message), sha256 = hash("sha256", message) +| KEEP message, md5, sha256; +``` diff --git a/docs/reference/esql/functions/kibana/docs/md5.md b/docs/reference/esql/functions/kibana/docs/md5.md index a45c9dc6cd2d8..aacb8a3960165 100644 --- a/docs/reference/esql/functions/kibana/docs/md5.md +++ b/docs/reference/esql/functions/kibana/docs/md5.md @@ -5,3 +5,9 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ ### MD5 Computes the MD5 hash of the input. +``` +FROM sample_data +| WHERE message != "Connection error" +| EVAL md5 = md5(message) +| KEEP message, md5; +``` diff --git a/docs/reference/esql/functions/layout/hash.asciidoc b/docs/reference/esql/functions/layout/hash.asciidoc index 27c55ada6319b..daf7fbf1170b2 100644 --- a/docs/reference/esql/functions/layout/hash.asciidoc +++ b/docs/reference/esql/functions/layout/hash.asciidoc @@ -12,3 +12,4 @@ image::esql/functions/signature/hash.svg[Embedded,opts=inline] include::../parameters/hash.asciidoc[] include::../description/hash.asciidoc[] include::../types/hash.asciidoc[] +include::../examples/hash.asciidoc[] diff --git a/docs/reference/esql/functions/layout/md5.asciidoc b/docs/reference/esql/functions/layout/md5.asciidoc index 23f443c1c38b0..82d3031d6bdfd 100644 --- a/docs/reference/esql/functions/layout/md5.asciidoc +++ b/docs/reference/esql/functions/layout/md5.asciidoc @@ -12,3 +12,4 @@ image::esql/functions/signature/md5.svg[Embedded,opts=inline] include::../parameters/md5.asciidoc[] include::../description/md5.asciidoc[] include::../types/md5.asciidoc[] +include::../examples/md5.asciidoc[] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec index 7607169a06117..b792d6c1cdd9a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec @@ -1,18 +1,22 @@ hash required_capability: hash_function +// tag::hash[] FROM sample_data | WHERE message != "Connection error" | EVAL md5 = hash("md5", message), sha256 = hash("sha256", message) | KEEP message, md5, sha256; +// end::hash[] ignoreOrder:true +// tag::hash-result[] message:keyword | md5:keyword | sha256:keyword Connected to 10.1.0.1 | abd7d1ce2bb636842a29246b3512dcae | 6d8372129ad78770f7185554dd39864749a62690216460752d6c075fa38ad85c Connected to 10.1.0.2 | 8f8f1cb60832d153f5b9ec6dc828b93f | b0db24720f15857091b3c99f4c4833586d0ea3229911b8777efb8d917cf27e9a Connected to 10.1.0.3 | 912b6dc13503165a15de43304bb77c78 | 75b0480188db8acc4d5cc666a51227eb2bc5b989cd8ca912609f33e0846eff57 Disconnected | ef70e46fd3bbc21e3e1f0b6815e750c0 | 04dfac3671b494ad53fcd152f7a14511bfb35747278aad8ce254a0d6e4ba4718 ; +// end::hash-result[] hashOfConvertedType @@ -104,11 +108,23 @@ count:long | hash(md5, message):keyword | md5(message):keyword 3 | 2e92ae79ff32b37fee4368a594792183 | 2e92ae79ff32b37fee4368a594792183 ; + md5Hash required_capability: short_hash_functions -ROW input="input" | EVAL md5 = hash("md5", input); +// tag::md5[] +FROM sample_data +| WHERE message != "Connection error" +| EVAL md5 = md5(message) +| KEEP message, md5; +// end::md5[] +ignoreOrder:true -input:keyword | md5:keyword -input | a43c1b0aa53a0c908810c06ab1ff3967 +// tag::md5-result[] +message:keyword | md5:keyword +Connected to 10.1.0.1 | abd7d1ce2bb636842a29246b3512dcae +Connected to 10.1.0.2 | 8f8f1cb60832d153f5b9ec6dc828b93f +Connected to 10.1.0.3 | 912b6dc13503165a15de43304bb77c78 +Disconnected | ef70e46fd3bbc21e3e1f0b6815e750c0 ; +// end::md5-result[] diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java index dfe007c708060..52d33c0fc9d3d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; @@ -46,7 +47,8 @@ public class Hash extends EsqlScalarFunction { @FunctionInfo( returnType = "keyword", - description = "Computes the hash of the input using various algorithms such as MD5, SHA, SHA-224, SHA-256, SHA-384, SHA-512." + description = "Computes the hash of the input using various algorithms such as MD5, SHA, SHA-224, SHA-256, SHA-384, SHA-512.", + examples = { @Example(file = "hash", tag = "hash") } ) public Hash( Source source, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java index 179ec3a7d56e2..b42ec1036cb5b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Hash.HashFunction; @@ -25,7 +26,11 @@ public class Md5 extends AbstractHashFunction { private static final HashFunction MD5 = HashFunction.create("MD5"); - @FunctionInfo(returnType = "keyword", description = "Computes the MD5 hash of the input.") + @FunctionInfo( + returnType = "keyword", + description = "Computes the MD5 hash of the input.", + examples = { @Example(file = "hash", tag = "md5") } + ) public Md5(Source source, @Param(name = "input", type = { "keyword", "text" }, description = "Input to hash.") Expression input) { super(source, input); } From 00fce7fe09aad7f94c579e42b5df4a0fe38a08b5 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Thu, 2 Jan 2025 15:49:51 +0100 Subject: [PATCH 07/14] simplify test case creation --- .../function/scalar/string/HashTests.java | 22 +++++++------------ .../function/scalar/string/Md5Tests.java | 2 +- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java index b16742292e24e..6e5a8ed37c25e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/HashTests.java @@ -87,20 +87,14 @@ private static TestCaseSupplier createTestCase(String algorithm, boolean forceLi }); } - static List createHashFunctionTestCases(String algorithm) { - return List.of(createHashFunctionTestCase(algorithm, DataType.KEYWORD), createHashFunctionTestCase(algorithm, DataType.TEXT)); - } - - private static TestCaseSupplier createHashFunctionTestCase(String algorithm, DataType inputType) { - return new TestCaseSupplier(algorithm + " with " + inputType, List.of(inputType), () -> { - var input = randomFrom(TestCaseSupplier.stringCases(inputType)).get(); - return new TestCaseSupplier.TestCase( - List.of(input), - "HashConstantEvaluator[algorithm=" + algorithm + ", input=Attribute[channel=0]]", - DataType.KEYWORD, - equalTo(new BytesRef(HashTests.hash(algorithm, BytesRefs.toString(input.data())))) - ); - }); + static void addHashFunctionTestCases(List cases, String algorithm) { + TestCaseSupplier.forUnaryStrings( + cases, + "HashConstantEvaluator[algorithm=" + algorithm + ", input=Attribute[channel=0]]", + DataType.KEYWORD, + input -> new BytesRef(HashTests.hash(algorithm, BytesRefs.toString(input))), + List.of() + ); } private static TestCaseSupplier.TypedData createTypedData(String value, boolean forceLiteral, DataType type, String name) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java index 125e412ee5067..316553d2985d1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java @@ -28,7 +28,7 @@ public Md5Tests(@Name("TestCase") Supplier testCaseSu @ParametersFactory public static Iterable parameters() { List cases = new ArrayList<>(); - cases.addAll(HashTests.createHashFunctionTestCases("MD5")); + HashTests.addHashFunctionTestCases(cases, "MD5"); return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "string"); } From 60834e89f73aa45f2b4f890447874ddc54aa0cbd Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Fri, 3 Jan 2025 08:09:54 +0100 Subject: [PATCH 08/14] add missing required capability --- .../plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec index b792d6c1cdd9a..87e8d297fc658 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec @@ -98,6 +98,7 @@ input:integer | md5:keyword | sha256:keyword hashWithStats required_capability: hash_function +required_capability: short_hash_functions FROM sample_data | EVAL md5="md5" From 89be7d0a1b0ba386ad04a83407aa801b974c64f2 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Wed, 8 Jan 2025 11:15:34 +0100 Subject: [PATCH 09/14] rename the capability --- .../esql/qa/testFixtures/src/main/resources/hash.csv-spec | 4 ++-- .../org/elasticsearch/xpack/esql/action/EsqlCapabilities.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec index 87e8d297fc658..37316f661ca6a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec @@ -98,7 +98,7 @@ input:integer | md5:keyword | sha256:keyword hashWithStats required_capability: hash_function -required_capability: short_hash_functions +required_capability: hash_function_aliases_v1 FROM sample_data | EVAL md5="md5" @@ -111,7 +111,7 @@ count:long | hash(md5, message):keyword | md5(message):keyword md5Hash -required_capability: short_hash_functions +required_capability: hash_function_aliases_v1 // tag::md5[] FROM sample_data diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index ade63b99cc65e..387e48702e708 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -459,9 +459,9 @@ public enum Cap { */ HASH_FUNCTION, /** - * Hash functions such as MD5 + * Hash function aliases such as MD5 */ - SHORT_HASH_FUNCTIONS, + HASH_FUNCTION_ALIASES_V1, /** * Don't optimize CASE IS NOT NULL function by not requiring the fields to be not null as well. From 8caebb5632f31cf2c05aa2b6e6d5f4e42370d3c2 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Wed, 8 Jan 2025 12:57:25 +0100 Subject: [PATCH 10/14] sha1 and sha256 --- .../esql/functions/description/sha1.asciidoc | 5 ++ .../functions/description/sha256.asciidoc | 5 ++ .../esql/functions/examples/sha1.asciidoc | 13 ++++ .../esql/functions/examples/sha256.asciidoc | 13 ++++ .../functions/kibana/definition/sha1.json | 37 ++++++++++++ .../functions/kibana/definition/sha256.json | 37 ++++++++++++ .../esql/functions/kibana/docs/sha1.md | 13 ++++ .../esql/functions/kibana/docs/sha256.md | 13 ++++ .../esql/functions/layout/sha1.asciidoc | 15 +++++ .../esql/functions/layout/sha256.asciidoc | 15 +++++ .../esql/functions/parameters/sha1.asciidoc | 6 ++ .../esql/functions/parameters/sha256.asciidoc | 6 ++ .../esql/functions/signature/sha1.svg | 1 + .../esql/functions/signature/sha256.svg | 1 + .../esql/functions/types/sha1.asciidoc | 10 ++++ .../esql/functions/types/sha256.asciidoc | 10 ++++ .../src/main/resources/hash.csv-spec | 47 ++++++++++++++- .../function/EsqlFunctionRegistry.java | 4 ++ .../scalar/ScalarFunctionWritables.java | 4 ++ .../function/scalar/string/Sha1.java | 60 +++++++++++++++++++ .../function/scalar/string/Sha256.java | 60 +++++++++++++++++++ .../scalar/string/Sha1SerializationTests.java | 25 ++++++++ .../function/scalar/string/Sha1Tests.java | 39 ++++++++++++ .../string/Sha256SerializationTests.java | 25 ++++++++ .../function/scalar/string/Sha256Tests.java | 39 ++++++++++++ 25 files changed, 500 insertions(+), 3 deletions(-) create mode 100644 docs/reference/esql/functions/description/sha1.asciidoc create mode 100644 docs/reference/esql/functions/description/sha256.asciidoc create mode 100644 docs/reference/esql/functions/examples/sha1.asciidoc create mode 100644 docs/reference/esql/functions/examples/sha256.asciidoc create mode 100644 docs/reference/esql/functions/kibana/definition/sha1.json create mode 100644 docs/reference/esql/functions/kibana/definition/sha256.json create mode 100644 docs/reference/esql/functions/kibana/docs/sha1.md create mode 100644 docs/reference/esql/functions/kibana/docs/sha256.md create mode 100644 docs/reference/esql/functions/layout/sha1.asciidoc create mode 100644 docs/reference/esql/functions/layout/sha256.asciidoc create mode 100644 docs/reference/esql/functions/parameters/sha1.asciidoc create mode 100644 docs/reference/esql/functions/parameters/sha256.asciidoc create mode 100644 docs/reference/esql/functions/signature/sha1.svg create mode 100644 docs/reference/esql/functions/signature/sha256.svg create mode 100644 docs/reference/esql/functions/types/sha1.asciidoc create mode 100644 docs/reference/esql/functions/types/sha256.asciidoc create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1SerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1Tests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256SerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256Tests.java diff --git a/docs/reference/esql/functions/description/sha1.asciidoc b/docs/reference/esql/functions/description/sha1.asciidoc new file mode 100644 index 0000000000000..5bc29f86cc591 --- /dev/null +++ b/docs/reference/esql/functions/description/sha1.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Computes the SHA1 hash of the input. diff --git a/docs/reference/esql/functions/description/sha256.asciidoc b/docs/reference/esql/functions/description/sha256.asciidoc new file mode 100644 index 0000000000000..b2a7ef01e1069 --- /dev/null +++ b/docs/reference/esql/functions/description/sha256.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Computes the SHA256 hash of the input. diff --git a/docs/reference/esql/functions/examples/sha1.asciidoc b/docs/reference/esql/functions/examples/sha1.asciidoc new file mode 100644 index 0000000000000..77786431a738a --- /dev/null +++ b/docs/reference/esql/functions/examples/sha1.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/hash.csv-spec[tag=sha1] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/hash.csv-spec[tag=sha1-result] +|=== + diff --git a/docs/reference/esql/functions/examples/sha256.asciidoc b/docs/reference/esql/functions/examples/sha256.asciidoc new file mode 100644 index 0000000000000..801c36d8effc8 --- /dev/null +++ b/docs/reference/esql/functions/examples/sha256.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/hash.csv-spec[tag=sha256] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/hash.csv-spec[tag=sha256-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/sha1.json b/docs/reference/esql/functions/kibana/definition/sha1.json new file mode 100644 index 0000000000000..a6abb31368bb3 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/sha1.json @@ -0,0 +1,37 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "sha1", + "description" : "Computes the SHA1 hash of the input.", + "signatures" : [ + { + "params" : [ + { + "name" : "input", + "type" : "keyword", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "input", + "type" : "text", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + } + ], + "examples" : [ + "FROM sample_data \n| WHERE message != \"Connection error\"\n| EVAL sha1 = sha1(message)\n| KEEP message, sha1;" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/definition/sha256.json b/docs/reference/esql/functions/kibana/definition/sha256.json new file mode 100644 index 0000000000000..700425d485b61 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/sha256.json @@ -0,0 +1,37 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "sha256", + "description" : "Computes the SHA256 hash of the input.", + "signatures" : [ + { + "params" : [ + { + "name" : "input", + "type" : "keyword", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "input", + "type" : "text", + "optional" : false, + "description" : "Input to hash." + } + ], + "variadic" : false, + "returnType" : "keyword" + } + ], + "examples" : [ + "FROM sample_data \n| WHERE message != \"Connection error\"\n| EVAL sha256 = sha256(message)\n| KEEP message, sha256;" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/docs/sha1.md b/docs/reference/esql/functions/kibana/docs/sha1.md new file mode 100644 index 0000000000000..a940aa133f06e --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/sha1.md @@ -0,0 +1,13 @@ + + +### SHA1 +Computes the SHA1 hash of the input. + +``` +FROM sample_data +| WHERE message != "Connection error" +| EVAL sha1 = sha1(message) +| KEEP message, sha1; +``` diff --git a/docs/reference/esql/functions/kibana/docs/sha256.md b/docs/reference/esql/functions/kibana/docs/sha256.md new file mode 100644 index 0000000000000..fbe576c7c20d6 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/sha256.md @@ -0,0 +1,13 @@ + + +### SHA256 +Computes the SHA256 hash of the input. + +``` +FROM sample_data +| WHERE message != "Connection error" +| EVAL sha256 = sha256(message) +| KEEP message, sha256; +``` diff --git a/docs/reference/esql/functions/layout/sha1.asciidoc b/docs/reference/esql/functions/layout/sha1.asciidoc new file mode 100644 index 0000000000000..23e1e0e9ac2ab --- /dev/null +++ b/docs/reference/esql/functions/layout/sha1.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-sha1]] +=== `SHA1` + +*Syntax* + +[.text-center] +image::esql/functions/signature/sha1.svg[Embedded,opts=inline] + +include::../parameters/sha1.asciidoc[] +include::../description/sha1.asciidoc[] +include::../types/sha1.asciidoc[] +include::../examples/sha1.asciidoc[] diff --git a/docs/reference/esql/functions/layout/sha256.asciidoc b/docs/reference/esql/functions/layout/sha256.asciidoc new file mode 100644 index 0000000000000..d36a1345271f5 --- /dev/null +++ b/docs/reference/esql/functions/layout/sha256.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-sha256]] +=== `SHA256` + +*Syntax* + +[.text-center] +image::esql/functions/signature/sha256.svg[Embedded,opts=inline] + +include::../parameters/sha256.asciidoc[] +include::../description/sha256.asciidoc[] +include::../types/sha256.asciidoc[] +include::../examples/sha256.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/sha1.asciidoc b/docs/reference/esql/functions/parameters/sha1.asciidoc new file mode 100644 index 0000000000000..99eba4dc2cb3d --- /dev/null +++ b/docs/reference/esql/functions/parameters/sha1.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`input`:: +Input to hash. diff --git a/docs/reference/esql/functions/parameters/sha256.asciidoc b/docs/reference/esql/functions/parameters/sha256.asciidoc new file mode 100644 index 0000000000000..99eba4dc2cb3d --- /dev/null +++ b/docs/reference/esql/functions/parameters/sha256.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`input`:: +Input to hash. diff --git a/docs/reference/esql/functions/signature/sha1.svg b/docs/reference/esql/functions/signature/sha1.svg new file mode 100644 index 0000000000000..bab03a7eb88c8 --- /dev/null +++ b/docs/reference/esql/functions/signature/sha1.svg @@ -0,0 +1 @@ +SHA1(input) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/sha256.svg b/docs/reference/esql/functions/signature/sha256.svg new file mode 100644 index 0000000000000..b77126bbefbd8 --- /dev/null +++ b/docs/reference/esql/functions/signature/sha256.svg @@ -0,0 +1 @@ +SHA256(input) \ No newline at end of file diff --git a/docs/reference/esql/functions/types/sha1.asciidoc b/docs/reference/esql/functions/types/sha1.asciidoc new file mode 100644 index 0000000000000..049a553397bbd --- /dev/null +++ b/docs/reference/esql/functions/types/sha1.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +input | result +keyword | keyword +text | keyword +|=== diff --git a/docs/reference/esql/functions/types/sha256.asciidoc b/docs/reference/esql/functions/types/sha256.asciidoc new file mode 100644 index 0000000000000..049a553397bbd --- /dev/null +++ b/docs/reference/esql/functions/types/sha256.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +input | result +keyword | keyword +text | keyword +|=== diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec index 37316f661ca6a..2614ff09fed06 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/hash.csv-spec @@ -102,11 +102,11 @@ required_capability: hash_function_aliases_v1 FROM sample_data | EVAL md5="md5" -| STATS count = count(*) by hash(md5, message), md5(message) +| STATS count = count(*) by hash(md5, message), md5(message), sha1(message), sha256(message) | WHERE count > 1; -count:long | hash(md5, message):keyword | md5(message):keyword -3 | 2e92ae79ff32b37fee4368a594792183 | 2e92ae79ff32b37fee4368a594792183 +count:long | hash(md5, message):keyword | md5(message):keyword | sha1(message):keyword | sha256(message):keyword +3 | 2e92ae79ff32b37fee4368a594792183 | 2e92ae79ff32b37fee4368a594792183 | 1dbb3521876a899f82d6b0ff10eb32a01e03aba8 | 8d137af9c64fba09bbb003aba93f0b029898fe19e7927cd696f4c3e2b69f538d ; @@ -129,3 +129,44 @@ Connected to 10.1.0.3 | 912b6dc13503165a15de43304bb77c78 Disconnected | ef70e46fd3bbc21e3e1f0b6815e750c0 ; // end::md5-result[] + + +sha1Hash +required_capability: hash_function_aliases_v1 + +// tag::sha1[] +FROM sample_data +| WHERE message != "Connection error" +| EVAL sha1 = sha1(message) +| KEEP message, sha1; +// end::sha1[] +ignoreOrder:true + +// tag::sha1-result[] +message:keyword | sha1:keyword +Connected to 10.1.0.1 | 42b85531a79088036a17759db7d2de292b92f57f +Connected to 10.1.0.2 | d30db445da2e9237c9718d0c7e4fb7cbbe9c2cb4 +Connected to 10.1.0.3 | 2733848d943809f0b10cad3e980763e88afb9853 +Disconnected | 771e05f27b99fd59f638f41a7a4e977b1d4691fe +; +// end::sha1-result[] + +sha256Hash +required_capability: hash_function_aliases_v1 + +// tag::sha256[] +FROM sample_data +| WHERE message != "Connection error" +| EVAL sha256 = sha256(message) +| KEEP message, sha256; +// end::sha256[] +ignoreOrder:true + +// tag::sha256-result[] +message:keyword | sha256:keyword +Connected to 10.1.0.1 | 6d8372129ad78770f7185554dd39864749a62690216460752d6c075fa38ad85c +Connected to 10.1.0.2 | b0db24720f15857091b3c99f4c4833586d0ea3229911b8777efb8d917cf27e9a +Connected to 10.1.0.3 | 75b0480188db8acc4d5cc666a51227eb2bc5b989cd8ca912609f33e0846eff57 +Disconnected | 04dfac3671b494ad53fcd152f7a14511bfb35747278aad8ce254a0d6e4ba4718 +; +// end::sha256-result[] diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index fd9cddf93c518..d310973d6dbe7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -140,6 +140,8 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.Replace; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Reverse; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Right; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Sha1; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Sha256; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Space; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; @@ -340,6 +342,8 @@ private static FunctionDefinition[][] functions() { def(Replace.class, Replace::new, "replace"), def(Reverse.class, Reverse::new, "reverse"), def(Right.class, Right::new, "right"), + def(Sha1.class, Sha1::new, "sha1"), + def(Sha256.class, Sha256::new, "sha256"), def(Space.class, Space::new, "space"), def(StartsWith.class, StartsWith::new, "starts_with"), def(Substring.class, Substring::new, "substring"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java index 1e1728d03fcc5..c4b9f6885e617 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java @@ -42,6 +42,8 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.Replace; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Reverse; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Right; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Sha1; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Sha256; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring; @@ -90,6 +92,8 @@ public static List getNamedWriteables() { entries.add(Replace.ENTRY); entries.add(Reverse.ENTRY); entries.add(Round.ENTRY); + entries.add(Sha1.ENTRY); + entries.add(Sha256.ENTRY); entries.add(Split.ENTRY); entries.add(Substring.ENTRY); entries.add(StartsWith.ENTRY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1.java new file mode 100644 index 0000000000000..ba1f62562c03a --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; + +import java.io.IOException; +import java.util.List; + +public class Sha1 extends AbstractHashFunction { + + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "SHA1", Sha1::new); + + private static final Hash.HashFunction SHA1 = Hash.HashFunction.create("SHA1"); + + @FunctionInfo( + returnType = "keyword", + description = "Computes the SHA1 hash of the input.", + examples = { @Example(file = "hash", tag = "sha1") } + ) + public Sha1(Source source, @Param(name = "input", type = { "keyword", "text" }, description = "Input to hash.") Expression input) { + super(source, input); + } + + private Sha1(StreamInput in) throws IOException { + super(in); + } + + @Override + protected Hash.HashFunction getHashFunction() { + return SHA1; + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new Sha1(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Sha1::new, field); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256.java new file mode 100644 index 0000000000000..b16767a3f7948 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; + +import java.io.IOException; +import java.util.List; + +public class Sha256 extends AbstractHashFunction { + + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "SHA256", Sha256::new); + + private static final Hash.HashFunction SHA256 = Hash.HashFunction.create("SHA256"); + + @FunctionInfo( + returnType = "keyword", + description = "Computes the SHA256 hash of the input.", + examples = { @Example(file = "hash", tag = "sha256") } + ) + public Sha256(Source source, @Param(name = "input", type = { "keyword", "text" }, description = "Input to hash.") Expression input) { + super(source, input); + } + + private Sha256(StreamInput in) throws IOException { + super(in); + } + + @Override + protected Hash.HashFunction getHashFunction() { + return SHA256; + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new Sha256(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Sha256::new, field); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1SerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1SerializationTests.java new file mode 100644 index 0000000000000..12c9cc7580de9 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1SerializationTests.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class Sha1SerializationTests extends AbstractExpressionSerializationTests { + + @Override + protected Sha1 createTestInstance() { + return new Sha1(randomSource(), randomChild()); + } + + @Override + protected Sha1 mutateInstance(Sha1 instance) throws IOException { + return new Sha1(instance.source(), mutateExpression(instance.field())); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1Tests.java new file mode 100644 index 0000000000000..f5cbbbe9530e2 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1Tests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +public class Sha1Tests extends AbstractScalarFunctionTestCase { + + public Sha1Tests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List cases = new ArrayList<>(); + HashTests.addHashFunctionTestCases(cases, "SHA1"); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "string"); + } + + @Override + protected Expression build(Source source, List args) { + return new Sha1(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256SerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256SerializationTests.java new file mode 100644 index 0000000000000..2f209fef25d36 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256SerializationTests.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class Sha256SerializationTests extends AbstractExpressionSerializationTests { + + @Override + protected Sha256 createTestInstance() { + return new Sha256(randomSource(), randomChild()); + } + + @Override + protected Sha256 mutateInstance(Sha256 instance) throws IOException { + return new Sha256(instance.source(), mutateExpression(instance.field())); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256Tests.java new file mode 100644 index 0000000000000..514dedcdf6db9 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256Tests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +public class Sha256Tests extends AbstractScalarFunctionTestCase { + + public Sha256Tests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List cases = new ArrayList<>(); + HashTests.addHashFunctionTestCases(cases, "SHA256"); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "string"); + } + + @Override + protected Expression build(Source source, List args) { + return new Sha256(source, args.get(0)); + } +} From 4cba449bafc98f8b51c71b2fc83a742c4e942a4a Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Wed, 8 Jan 2025 13:45:32 +0100 Subject: [PATCH 11/14] update function count --- .../resources/rest-api-spec/test/esql/60_usage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index 8a45049e0872b..35e5d66e91d97 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -92,7 +92,7 @@ setup: - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} # Testing for the entire function set isn't feasbile, so we just check that we return the correct count as an approximation. - - length: {esql.functions: 131} # check the "sister" test below for a likely update to the same esql.functions length check + - length: {esql.functions: 133} # check the "sister" test below for a likely update to the same esql.functions length check --- "Basic ESQL usage output (telemetry) non-snapshot version": @@ -163,4 +163,4 @@ setup: - match: {esql.functions.cos: $functions_cos} - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} - - length: {esql.functions: 127} # check the "sister" test above for a likely update to the same esql.functions length check + - length: {esql.functions: 129} # check the "sister" test above for a likely update to the same esql.functions length check From 500658f408a94edd67caef3934c57ec2f3ced076 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Wed, 8 Jan 2025 15:24:52 +0100 Subject: [PATCH 12/14] link new hash functions --- docs/reference/esql/functions/string-functions.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/reference/esql/functions/string-functions.asciidoc b/docs/reference/esql/functions/string-functions.asciidoc index 4b81f247b7276..30a3377ccb2a3 100644 --- a/docs/reference/esql/functions/string-functions.asciidoc +++ b/docs/reference/esql/functions/string-functions.asciidoc @@ -49,6 +49,8 @@ include::layout/replace.asciidoc[] include::layout/reverse.asciidoc[] include::layout/right.asciidoc[] include::layout/rtrim.asciidoc[] +include::layout/sha1.asciidoc[] +include::layout/sha256.asciidoc[] include::layout/space.asciidoc[] include::layout/split.asciidoc[] include::layout/starts_with.asciidoc[] From 4d7c381f6c0ffff64be3f7dfc8ac8f2fd3c37ee3 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Wed, 8 Jan 2025 15:28:00 +0100 Subject: [PATCH 13/14] add hash functions to the list --- docs/reference/esql/functions/string-functions.asciidoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/reference/esql/functions/string-functions.asciidoc b/docs/reference/esql/functions/string-functions.asciidoc index 30a3377ccb2a3..dd10e4c77581e 100644 --- a/docs/reference/esql/functions/string-functions.asciidoc +++ b/docs/reference/esql/functions/string-functions.asciidoc @@ -18,11 +18,14 @@ * <> * <> * <> +* <> * <> * <> * <> * <> * <> +* <> +* <> * <> * <> * <> From f0b0deb91f1a00107a183a08e9776a856fc071dc Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Wed, 8 Jan 2025 15:40:26 +0100 Subject: [PATCH 14/14] use new api --- .../function/scalar/string/Md5ErrorTests.java | 38 +++++++++++++++++++ .../function/scalar/string/Md5Tests.java | 2 +- .../scalar/string/Sha1ErrorTests.java | 38 +++++++++++++++++++ .../function/scalar/string/Sha1Tests.java | 2 +- .../scalar/string/Sha256ErrorTests.java | 38 +++++++++++++++++++ .../function/scalar/string/Sha256Tests.java | 2 +- 6 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5ErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1ErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256ErrorTests.java diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5ErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5ErrorTests.java new file mode 100644 index 0000000000000..701cef748c932 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5ErrorTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class Md5ErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + + @Override + protected List cases() { + return paramsToSuppliers(Md5Tests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Md5(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java index 316553d2985d1..3c0a2067a81b0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Md5Tests.java @@ -29,7 +29,7 @@ public Md5Tests(@Name("TestCase") Supplier testCaseSu public static Iterable parameters() { List cases = new ArrayList<>(); HashTests.addHashFunctionTestCases(cases, "MD5"); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1ErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1ErrorTests.java new file mode 100644 index 0000000000000..613aabe658ceb --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1ErrorTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class Sha1ErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + + @Override + protected List cases() { + return paramsToSuppliers(Sha1Tests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Sha1(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1Tests.java index f5cbbbe9530e2..06c23f186d4c8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1Tests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha1Tests.java @@ -29,7 +29,7 @@ public Sha1Tests(@Name("TestCase") Supplier testCaseS public static Iterable parameters() { List cases = new ArrayList<>(); HashTests.addHashFunctionTestCases(cases, "SHA1"); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256ErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256ErrorTests.java new file mode 100644 index 0000000000000..a9678346f8508 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256ErrorTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class Sha256ErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + + @Override + protected List cases() { + return paramsToSuppliers(Sha256Tests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new Sha256(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256Tests.java index 514dedcdf6db9..0a136e3214082 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256Tests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Sha256Tests.java @@ -29,7 +29,7 @@ public Sha256Tests(@Name("TestCase") Supplier testCas public static Iterable parameters() { List cases = new ArrayList<>(); HashTests.addHashFunctionTestCases(cases, "SHA256"); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, cases); } @Override