From 6f592273c4025faeaf74561b533e23180fd2fd7f Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Mon, 28 Oct 2024 17:19:20 +0100 Subject: [PATCH 01/22] Add ES|QL BIT_LENGTH support. --- .../functions/description/bit_length.asciidoc | 5 + .../functions/examples/bit_length.asciidoc | 13 ++ .../esql/functions/kibana/docs/bit_length.md | 12 ++ .../esql/functions/layout/bit_length.asciidoc | 15 +++ .../functions/parameters/bit_length.asciidoc | 6 + .../esql/functions/signature/bit_length.svg | 1 + .../esql/functions/string-functions.asciidoc | 2 + .../esql/functions/types/bit_length.asciidoc | 10 ++ .../src/main/resources/docs.csv-spec | 18 +++ .../src/main/resources/string.csv-spec | 46 +++++++ .../scalar/string/BitLengthEvaluator.java | 127 ++++++++++++++++++ .../xpack/esql/action/EsqlCapabilities.java | 5 + .../function/EsqlFunctionRegistry.java | 2 + .../function/scalar/EsqlScalarFunction.java | 2 + .../function/scalar/string/BitLength.java | 96 +++++++++++++ .../string/BitLengthSerializationTests.java | 19 +++ .../scalar/string/BitLengthTests.java | 82 +++++++++++ 17 files changed, 461 insertions(+) create mode 100644 docs/reference/esql/functions/description/bit_length.asciidoc create mode 100644 docs/reference/esql/functions/examples/bit_length.asciidoc create mode 100644 docs/reference/esql/functions/kibana/docs/bit_length.md create mode 100644 docs/reference/esql/functions/layout/bit_length.asciidoc create mode 100644 docs/reference/esql/functions/parameters/bit_length.asciidoc create mode 100644 docs/reference/esql/functions/signature/bit_length.svg create mode 100644 docs/reference/esql/functions/types/bit_length.asciidoc create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthSerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java diff --git a/docs/reference/esql/functions/description/bit_length.asciidoc b/docs/reference/esql/functions/description/bit_length.asciidoc new file mode 100644 index 0000000000000..1aad47488802d --- /dev/null +++ b/docs/reference/esql/functions/description/bit_length.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Returns the bit length of a string. diff --git a/docs/reference/esql/functions/examples/bit_length.asciidoc b/docs/reference/esql/functions/examples/bit_length.asciidoc new file mode 100644 index 0000000000000..a99f6f664e79e --- /dev/null +++ b/docs/reference/esql/functions/examples/bit_length.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/docs.csv-spec[tag=bitLength] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/docs.csv-spec[tag=bitLength-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/docs/bit_length.md b/docs/reference/esql/functions/kibana/docs/bit_length.md new file mode 100644 index 0000000000000..22280febd7876 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/bit_length.md @@ -0,0 +1,12 @@ + + +### BIT_LENGTH +Returns the bit length of a string. + +``` +FROM employees +| KEEP first_name, last_name +| EVAL fn_bit_length = BIT_LENGTH(first_name) +``` diff --git a/docs/reference/esql/functions/layout/bit_length.asciidoc b/docs/reference/esql/functions/layout/bit_length.asciidoc new file mode 100644 index 0000000000000..00a7206f3ceda --- /dev/null +++ b/docs/reference/esql/functions/layout/bit_length.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-bit_length]] +=== `BIT_LENGTH` + +*Syntax* + +[.text-center] +image::esql/functions/signature/bit_length.svg[Embedded,opts=inline] + +include::../parameters/bit_length.asciidoc[] +include::../description/bit_length.asciidoc[] +include::../types/bit_length.asciidoc[] +include::../examples/bit_length.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/bit_length.asciidoc b/docs/reference/esql/functions/parameters/bit_length.asciidoc new file mode 100644 index 0000000000000..7bb8c080ce4a1 --- /dev/null +++ b/docs/reference/esql/functions/parameters/bit_length.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`string`:: +String expression. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/signature/bit_length.svg b/docs/reference/esql/functions/signature/bit_length.svg new file mode 100644 index 0000000000000..904dbbe25c9c2 --- /dev/null +++ b/docs/reference/esql/functions/signature/bit_length.svg @@ -0,0 +1 @@ +BIT_LENGTH(string) \ No newline at end of file diff --git a/docs/reference/esql/functions/string-functions.asciidoc b/docs/reference/esql/functions/string-functions.asciidoc index f5222330d579d..422860f0a7a1d 100644 --- a/docs/reference/esql/functions/string-functions.asciidoc +++ b/docs/reference/esql/functions/string-functions.asciidoc @@ -8,6 +8,7 @@ {esql} supports these string functions: // tag::string_list[] +* <> * <> * <> * <> @@ -30,6 +31,7 @@ * <> // end::string_list[] +include::layout/bit_length.asciidoc[] include::layout/concat.asciidoc[] include::layout/ends_with.asciidoc[] include::layout/from_base64.asciidoc[] diff --git a/docs/reference/esql/functions/types/bit_length.asciidoc b/docs/reference/esql/functions/types/bit_length.asciidoc new file mode 100644 index 0000000000000..db5a48c7c4390 --- /dev/null +++ b/docs/reference/esql/functions/types/bit_length.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +string | result +keyword | integer +text | integer +|=== diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec index a9c5a5214f159..45c0e66e7b76d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec @@ -656,3 +656,21 @@ FROM sample_data @timestamp:date | client_ip:ip | event_duration:long | message:keyword ; + +docsBitLength +// tag::bitLength[] +FROM employees +| KEEP first_name, last_name +| EVAL fn_bit_length = BIT_LENGTH(first_name) +// end::bitLength[] +| SORT first_name +| LIMIT 3 +; + +// tag::bitLength-result[] +first_name:keyword | last_name:keyword | fn_bit_length:integer +Alejandro |McAlpine |72 +Amabile |Gomatam |56 +Anneke |Preusig |48 +// end::bitLength-result[] +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index 305b8f3d8011e..e2e0c6ef7b35e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -38,6 +38,52 @@ emp_no:integer | l:integer 10003 | 5 ; +bitLength +row a = "hello", b = "" | eval y = bit_length(a) + bit_length(b); + +a:keyword | b:keyword | y:integer +hello | | 40 +; + +bitLengthWithNonAsciiChars +row a = "¡", b = "❗️" | eval y = bit_length(a) | eval z = bit_length(b); + +a:keyword | b:keyword | y:integer | z:integer +¡ | ❗️ | 16 | 48 +; + +foldBitLength +row a = 1 | eval b = bit_length("hello"); + +a:integer | b:integer +1 | 40 +; + +bitLengthAndSourceQuoting +required_capability: double_quotes_source_enclosing +from "employees" | sort emp_no | limit 3 | eval l = bit_length(first_name) | keep emp_no, l; + +emp_no:integer | l:integer +10001 | 48 +10002 | 56 +10003 | 40 +; + +bitLengthInsideOtherFunction +required_capability: fn_greatest +row a = "abc", b = "de" | eval g = greatest(bit_length(a), bit_length(b), bit_length("fghi")); + +a:keyword | b:keyword | g:integer +abc | de | 32 +; + +bitLengthNull +row a = "abc" | eval l = bit_length(null); + +a:string | l:integer +abc | null +; + startsWithConstant from employees | sort emp_no | limit 10 | eval f_S = starts_with(first_name, "S") | keep emp_no, first_name, f_S; diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthEvaluator.java new file mode 100644 index 0000000000000..5aae528ad40ef --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthEvaluator.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link BitLength}. + * This class is generated. Do not edit it. + */ +public final class BitLengthEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator val; + + private final DriverContext driverContext; + + private Warnings warnings; + + public BitLengthEvaluator(Source source, EvalOperator.ExpressionEvaluator val, + DriverContext driverContext) { + this.source = source; + this.val = val; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (BytesRefBlock valBlock = (BytesRefBlock) val.eval(page)) { + BytesRefVector valVector = valBlock.asVector(); + if (valVector == null) { + return eval(page.getPositionCount(), valBlock); + } + return eval(page.getPositionCount(), valVector).asBlock(); + } + } + + public IntBlock eval(int positionCount, BytesRefBlock valBlock) { + try(IntBlock.Builder result = driverContext.blockFactory().newIntBlockBuilder(positionCount)) { + BytesRef valScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (valBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (valBlock.getValueCount(p) != 1) { + if (valBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendInt(BitLength.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch))); + } + return result.build(); + } + } + + public IntVector eval(int positionCount, BytesRefVector valVector) { + try(IntVector.FixedBuilder result = driverContext.blockFactory().newIntVectorFixedBuilder(positionCount)) { + BytesRef valScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + result.appendInt(p, BitLength.process(valVector.getBytesRef(p, valScratch))); + } + return result.build(); + } + } + + @Override + public String toString() { + return "BitLengthEvaluator[" + "val=" + val + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(val); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory val; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory val) { + this.source = source; + this.val = val; + } + + @Override + public BitLengthEvaluator get(DriverContext context) { + return new BitLengthEvaluator(source, val.get(context), context); + } + + @Override + public String toString() { + return "BitLengthEvaluator[" + "val=" + val + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 196a864db2c15..8bf978a8d8757 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -44,6 +44,11 @@ public enum Cap { */ FN_CBRT, + /** + * Support for function {@code GREATEST}. Done in #98630. + */ + FN_GREATEST, + /** * Support for function {@code HYPOT}. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 66151275fc2e8..d7a23a6589d4f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -117,6 +117,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StDistance; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StX; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StY; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.BitLength; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat; import org.elasticsearch.xpack.esql.expression.function.scalar.string.EndsWith; import org.elasticsearch.xpack.esql.expression.function.scalar.string.LTrim; @@ -305,6 +306,7 @@ private FunctionDefinition[][] functions() { def(Tau.class, Tau::new, "tau") }, // string new FunctionDefinition[] { + def(BitLength.class, BitLength::new, "bit_length"), def(Concat.class, Concat::new, "concat"), def(EndsWith.class, EndsWith::new, "ends_with"), def(LTrim.class, LTrim::new, "ltrim"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java index e4e1fbb6e5aac..65985f234ac92 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java @@ -38,6 +38,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tau; import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.BinarySpatialFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.BitLength; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat; import org.elasticsearch.xpack.esql.expression.function.scalar.string.EndsWith; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Left; @@ -74,6 +75,7 @@ public static List getNamedWriteables() { List entries = new ArrayList<>(); entries.add(And.ENTRY); entries.add(Atan2.ENTRY); + entries.add(BitLength.ENTRY); entries.add(Bucket.ENTRY); entries.add(Case.ENTRY); entries.add(Categorize.ENTRY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java new file mode 100644 index 0000000000000..74e8089c71d27 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; + +public class BitLength extends UnaryScalarFunction { + + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "BitLength", + BitLength::new + ); + + @FunctionInfo( + returnType = "integer", + description = "Returns the bit length of a string.", + examples = @Example(file = "docs", tag = "bitLength") + ) + public BitLength( + Source source, + @Param( + name = "string", + type = { "keyword", "text" }, + description = "String expression. If `null`, the function returns `null`." + ) Expression field + ) { + super(source, field); + } + + private BitLength(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public DataType dataType() { + return DataType.INTEGER; + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + + return isString(field(), sourceText(), DEFAULT); + } + + @Evaluator + static int process(BytesRef val) { + return val.length * 8; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new BitLength(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, BitLength::new, field()); + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + return new BitLengthEvaluator.Factory(source(), toEvaluator.apply(field())); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthSerializationTests.java new file mode 100644 index 0000000000000..2564ac0bdb1cf --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthSerializationTests.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractUnaryScalarSerializationTests; + +public class BitLengthSerializationTests extends AbstractUnaryScalarSerializationTests { + @Override + protected BitLength create(Source source, Expression child) { + return new BitLength(source, child); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java new file mode 100644 index 0000000000000..043924da7bee1 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.equalTo; + +public class BitLengthTests extends AbstractScalarFunctionTestCase { + + public BitLengthTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List cases = new ArrayList<>(); + cases.addAll(List.of(new TestCaseSupplier("bitLength basic test", List.of(DataType.KEYWORD), () -> { + BytesRef value = new BytesRef(randomAlphaOfLength(between(0, 10000))); + return new TestCaseSupplier.TestCase( + List.of(new TestCaseSupplier.TypedData(value, DataType.KEYWORD, "str")), + "BitLengthEvaluator[val=Attribute[channel=0]]", + DataType.INTEGER, + equalTo(value.length * 8) + ); + }))); + cases.addAll(makeTestCases("empty string", () -> "", 0)); + cases.addAll(makeTestCases("single ascii character", () -> "a", 8)); + cases.addAll(makeTestCases("ascii string", () -> "clump", 40)); + cases.addAll(makeTestCases("3 bytes", () -> "☕", 24)); + cases.addAll(makeTestCases("6 bytes", () -> "❗️", 48)); + cases.addAll(makeTestCases("100 random alpha", () -> randomAlphaOfLength(100), 800)); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "string"); + } + + private static List makeTestCases(String title, Supplier text, int expectedLength) { + return List.of( + new TestCaseSupplier( + title + " with keyword", + List.of(DataType.KEYWORD), + () -> new TestCaseSupplier.TestCase( + List.of(new TestCaseSupplier.TypedData(new BytesRef(text.get()), DataType.KEYWORD, "f")), + "BitLengthEvaluator[val=Attribute[channel=0]]", + DataType.INTEGER, + equalTo(expectedLength) + ) + ), + new TestCaseSupplier( + title + " with text", + List.of(DataType.TEXT), + () -> new TestCaseSupplier.TestCase( + List.of(new TestCaseSupplier.TypedData(new BytesRef(text.get()), DataType.TEXT, "f")), + "BitLengthEvaluator[val=Attribute[channel=0]]", + DataType.INTEGER, + equalTo(expectedLength) + ) + ) + ); + } + + @Override + protected Expression build(Source source, List args) { + return new BitLength(source, args.get(0)); + } +} From 3de014355625214ce463a79f3317ad49b2d4e4df Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Mon, 28 Oct 2024 17:23:52 +0100 Subject: [PATCH 02/22] Update docs/changelog/115792.yaml --- docs/changelog/115792.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/115792.yaml diff --git a/docs/changelog/115792.yaml b/docs/changelog/115792.yaml new file mode 100644 index 0000000000000..2945a64e3043a --- /dev/null +++ b/docs/changelog/115792.yaml @@ -0,0 +1,5 @@ +pr: 115792 +summary: Add ES|QL `bit_length` function +area: ES|QL +type: enhancement +issues: [] From 7b6955676bd054afbac6bb0502eca449cc46e8f0 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Mon, 28 Oct 2024 17:26:30 +0100 Subject: [PATCH 03/22] Use "str" as typed data name in BitLengthTests --- .../expression/function/scalar/string/BitLengthTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java index 043924da7bee1..b77e6f05be039 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java @@ -56,7 +56,7 @@ private static List makeTestCases(String title, Supplier new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(new BytesRef(text.get()), DataType.KEYWORD, "f")), + List.of(new TestCaseSupplier.TypedData(new BytesRef(text.get()), DataType.KEYWORD, "str")), "BitLengthEvaluator[val=Attribute[channel=0]]", DataType.INTEGER, equalTo(expectedLength) @@ -66,7 +66,7 @@ private static List makeTestCases(String title, Supplier new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(new BytesRef(text.get()), DataType.TEXT, "f")), + List.of(new TestCaseSupplier.TypedData(new BytesRef(text.get()), DataType.TEXT, "str")), "BitLengthEvaluator[val=Attribute[channel=0]]", DataType.INTEGER, equalTo(expectedLength) From 7a2ecf9261ff8f5366db59d3619ec767e072140e Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Mon, 28 Oct 2024 17:57:17 +0100 Subject: [PATCH 04/22] Update number of functions in 60_usage.yml --- .../resources/rest-api-spec/test/esql/60_usage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index b51bbdc4d2f87..f26bd799297f6 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -91,7 +91,7 @@ setup: - match: {esql.functions.cos: $functions_cos} - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} - - length: {esql.functions: 117} # check the "sister" test below for a likely update to the same esql.functions length check + - length: {esql.functions: 118} # check the "sister" test below for a likely update to the same esql.functions length check --- "Basic ESQL usage output (telemetry) non-snapshot version": @@ -162,4 +162,4 @@ setup: - match: {esql.functions.cos: $functions_cos} - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} - - length: {esql.functions: 115} # check the "sister" test above for a likely update to the same esql.functions length check + - length: {esql.functions: 116} # check the "sister" test above for a likely update to the same esql.functions length check From 2583805ea128890b5cbeb2deba4dfb8ca2628294 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Tue, 29 Oct 2024 11:00:46 +0100 Subject: [PATCH 05/22] Add FN_BIT_LENGTH capability and guard csv tests accordingly --- .../esql/qa/testFixtures/src/main/resources/string.csv-spec | 6 ++++++ .../elasticsearch/xpack/esql/action/EsqlCapabilities.java | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index e2e0c6ef7b35e..fe794890c76d3 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -39,6 +39,7 @@ emp_no:integer | l:integer ; bitLength +required_capability: fn_bit_length row a = "hello", b = "" | eval y = bit_length(a) + bit_length(b); a:keyword | b:keyword | y:integer @@ -46,6 +47,7 @@ hello | | 40 ; bitLengthWithNonAsciiChars +required_capability: fn_bit_length row a = "¡", b = "❗️" | eval y = bit_length(a) | eval z = bit_length(b); a:keyword | b:keyword | y:integer | z:integer @@ -53,6 +55,7 @@ a:keyword | b:keyword | y:integer | z:integer ; foldBitLength +required_capability: fn_bit_length row a = 1 | eval b = bit_length("hello"); a:integer | b:integer @@ -60,6 +63,7 @@ a:integer | b:integer ; bitLengthAndSourceQuoting +required_capability: fn_bit_length required_capability: double_quotes_source_enclosing from "employees" | sort emp_no | limit 3 | eval l = bit_length(first_name) | keep emp_no, l; @@ -70,6 +74,7 @@ emp_no:integer | l:integer ; bitLengthInsideOtherFunction +required_capability: fn_bit_length required_capability: fn_greatest row a = "abc", b = "de" | eval g = greatest(bit_length(a), bit_length(b), bit_length("fghi")); @@ -78,6 +83,7 @@ abc | de | 32 ; bitLengthNull +required_capability: fn_bit_length row a = "abc" | eval l = bit_length(null); a:string | l:integer diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 8bf978a8d8757..efdef48de491f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -27,6 +27,12 @@ */ public class EsqlCapabilities { public enum Cap { + + /** + * Support for function {@code BIT_LENGTH}. Done in #115792 + */ + FN_BIT_LENGTH, + /** * Support for function {@code REVERSE}. */ From d6e1c66307e14c22847d0b5bbff85736373baece Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Tue, 29 Oct 2024 11:29:44 +0100 Subject: [PATCH 06/22] Use new stream reading/writing approach --- .../expression/function/scalar/string/BitLength.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java index 74e8089c71d27..17b004a40ff00 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java @@ -10,6 +10,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -20,6 +21,7 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; import java.util.List; @@ -52,7 +54,13 @@ public BitLength( } private BitLength(StreamInput in) throws IOException { - super(in); + this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + out.writeNamedWriteable(field()); } @Override From a27f94a5ad27ed6623ff5d46eab326fedc9b02ca Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Tue, 29 Oct 2024 13:46:49 +0100 Subject: [PATCH 07/22] Use Byte.SIZE instead of magic number --- .../xpack/esql/expression/function/scalar/string/BitLength.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java index 17b004a40ff00..676dfa336d764 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java @@ -84,7 +84,7 @@ protected TypeResolution resolveType() { @Evaluator static int process(BytesRef val) { - return val.length * 8; + return val.length * Byte.SIZE; } @Override From 7a94caabed2922d8c57b685ac6a49c05bd915607 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Wed, 30 Oct 2024 09:11:38 +0100 Subject: [PATCH 08/22] Tryout remote --- .../xpack/esql/expression/function/scalar/string/BitLength.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java index 676dfa336d764..17b004a40ff00 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java @@ -84,7 +84,7 @@ protected TypeResolution resolveType() { @Evaluator static int process(BytesRef val) { - return val.length * Byte.SIZE; + return val.length * 8; } @Override From 0a4130c0ad7e970838d22da782b44e791fb87e0b Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Wed, 30 Oct 2024 09:12:28 +0100 Subject: [PATCH 09/22] Use Byte.SIZE instead of magic number --- .../xpack/esql/expression/function/scalar/string/BitLength.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java index 17b004a40ff00..676dfa336d764 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java @@ -84,7 +84,7 @@ protected TypeResolution resolveType() { @Evaluator static int process(BytesRef val) { - return val.length * 8; + return val.length * Byte.SIZE; } @Override From 666198e7c410df6ea59c9ce8327ada5f0497d375 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Wed, 30 Oct 2024 09:30:22 +0100 Subject: [PATCH 10/22] Remove FN_GREATEST capability and only use the most recent in bitLength tests --- .../esql/qa/testFixtures/src/main/resources/string.csv-spec | 2 -- .../elasticsearch/xpack/esql/action/EsqlCapabilities.java | 5 ----- 2 files changed, 7 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index fe794890c76d3..de5981df999c7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -64,7 +64,6 @@ a:integer | b:integer bitLengthAndSourceQuoting required_capability: fn_bit_length -required_capability: double_quotes_source_enclosing from "employees" | sort emp_no | limit 3 | eval l = bit_length(first_name) | keep emp_no, l; emp_no:integer | l:integer @@ -75,7 +74,6 @@ emp_no:integer | l:integer bitLengthInsideOtherFunction required_capability: fn_bit_length -required_capability: fn_greatest row a = "abc", b = "de" | eval g = greatest(bit_length(a), bit_length(b), bit_length("fghi")); a:keyword | b:keyword | g:integer diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index efdef48de491f..62a3c0fa11422 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -50,11 +50,6 @@ public enum Cap { */ FN_CBRT, - /** - * Support for function {@code GREATEST}. Done in #98630. - */ - FN_GREATEST, - /** * Support for function {@code HYPOT}. */ From db7931a48872dcecf6f059ed57cc37e99a51f68d Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Wed, 30 Oct 2024 09:37:14 +0100 Subject: [PATCH 11/22] Use TestCaseSupplier.stringCases as test cases --- .../scalar/string/BitLengthTests.java | 61 ++++++------------- 1 file changed, 20 insertions(+), 41 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java index b77e6f05be039..8fc44b4eac9aa 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java @@ -11,6 +11,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -31,52 +32,30 @@ public BitLengthTests(@Name("TestCase") Supplier test @ParametersFactory public static Iterable parameters() { - List cases = new ArrayList<>(); - cases.addAll(List.of(new TestCaseSupplier("bitLength basic test", List.of(DataType.KEYWORD), () -> { - BytesRef value = new BytesRef(randomAlphaOfLength(between(0, 10000))); - return new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(value, DataType.KEYWORD, "str")), - "BitLengthEvaluator[val=Attribute[channel=0]]", - DataType.INTEGER, - equalTo(value.length * 8) - ); - }))); - cases.addAll(makeTestCases("empty string", () -> "", 0)); - cases.addAll(makeTestCases("single ascii character", () -> "a", 8)); - cases.addAll(makeTestCases("ascii string", () -> "clump", 40)); - cases.addAll(makeTestCases("3 bytes", () -> "☕", 24)); - cases.addAll(makeTestCases("6 bytes", () -> "❗️", 48)); - cases.addAll(makeTestCases("100 random alpha", () -> randomAlphaOfLength(100), 800)); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> "string"); - } + List suppliers = new ArrayList<>(); + + for (DataType stringType : new DataType[] { DataType.KEYWORD, DataType.TEXT }) { + for (var supplier : TestCaseSupplier.stringCases(stringType)) { + suppliers.add(makeSupplier(supplier)); + } + } - private static List makeTestCases(String title, Supplier text, int expectedLength) { - return List.of( - new TestCaseSupplier( - title + " with keyword", - List.of(DataType.KEYWORD), - () -> new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(new BytesRef(text.get()), DataType.KEYWORD, "str")), - "BitLengthEvaluator[val=Attribute[channel=0]]", - DataType.INTEGER, - equalTo(expectedLength) - ) - ), - new TestCaseSupplier( - title + " with text", - List.of(DataType.TEXT), - () -> new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(new BytesRef(text.get()), DataType.TEXT, "str")), - "BitLengthEvaluator[val=Attribute[channel=0]]", - DataType.INTEGER, - equalTo(expectedLength) - ) - ) - ); + return parameterSuppliersFromTypedData(suppliers); } @Override protected Expression build(Source source, List args) { return new BitLength(source, args.get(0)); } + + private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier) { + return new TestCaseSupplier(fieldSupplier.name(), List.of(fieldSupplier.type()), () -> { + var fieldTypedData = fieldSupplier.get(); + String evaluatorToString = "BitLengthEvaluator[val=Attribute[channel=0]]"; + BytesRef value = BytesRefs.toBytesRef(fieldTypedData.data()); + var expectedValue = value.length * Byte.SIZE; + + return new TestCaseSupplier.TestCase(List.of(fieldTypedData), evaluatorToString, DataType.INTEGER, equalTo(expectedValue)); + }); + } } From b800d4fe9be4f1814e62b55605836a384e7731f9 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Wed, 30 Oct 2024 13:22:29 +0100 Subject: [PATCH 12/22] Update x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Iván Cea Fontenla --- .../esql/expression/function/scalar/string/BitLengthTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java index 8fc44b4eac9aa..b1c953dbcdc11 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java @@ -40,7 +40,7 @@ public static Iterable parameters() { } } - return parameterSuppliersFromTypedData(suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); } @Override From db02eb756e4cf22f1ad6ece7262a6d234c009a51 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Wed, 30 Oct 2024 15:18:54 +0100 Subject: [PATCH 13/22] Add bit length capability to usage integration tests --- .../resources/rest-api-spec/test/esql/60_usage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index f26bd799297f6..66abbffbf5636 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -30,7 +30,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [ snapshot_test_for_telemetry ] + capabilities: [ snapshot_test_for_telemetry, fn_bit_length ] reason: "Test that should only be executed on snapshot versions" - do: {xpack.usage: {}} @@ -101,7 +101,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [ non_snapshot_test_for_telemetry ] + capabilities: [ non_snapshot_test_for_telemetry, fn_bit_length ] reason: "Test that should only be executed on release versions" - do: {xpack.usage: {}} From 3cdde43d62dc8a1f2f8fc4a665499c869c4ca81e Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Wed, 30 Oct 2024 17:07:45 +0100 Subject: [PATCH 14/22] Catch ArithmeticException in Evaluator and turn expression result into a null value --- .../scalar/string/BitLengthEvaluator.java | 22 ++++++++++++++----- .../function/scalar/string/BitLength.java | 4 ++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthEvaluator.java index 5aae528ad40ef..6564a2f3ef167 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthEvaluator.java @@ -4,6 +4,7 @@ // 2.0. package org.elasticsearch.xpack.esql.expression.function.scalar.string; +import java.lang.ArithmeticException; import java.lang.IllegalArgumentException; import java.lang.Override; import java.lang.String; @@ -12,7 +13,6 @@ import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.IntBlock; -import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; @@ -47,7 +47,7 @@ public Block eval(Page page) { if (valVector == null) { return eval(page.getPositionCount(), valBlock); } - return eval(page.getPositionCount(), valVector).asBlock(); + return eval(page.getPositionCount(), valVector); } } @@ -66,17 +66,27 @@ public IntBlock eval(int positionCount, BytesRefBlock valBlock) { result.appendNull(); continue position; } - result.appendInt(BitLength.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch))); + try { + result.appendInt(BitLength.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch))); + } catch (ArithmeticException e) { + warnings().registerException(e); + result.appendNull(); + } } return result.build(); } } - public IntVector eval(int positionCount, BytesRefVector valVector) { - try(IntVector.FixedBuilder result = driverContext.blockFactory().newIntVectorFixedBuilder(positionCount)) { + public IntBlock eval(int positionCount, BytesRefVector valVector) { + try(IntBlock.Builder result = driverContext.blockFactory().newIntBlockBuilder(positionCount)) { BytesRef valScratch = new BytesRef(); position: for (int p = 0; p < positionCount; p++) { - result.appendInt(p, BitLength.process(valVector.getBytesRef(p, valScratch))); + try { + result.appendInt(BitLength.process(valVector.getBytesRef(p, valScratch))); + } catch (ArithmeticException e) { + warnings().registerException(e); + result.appendNull(); + } } return result.build(); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java index 676dfa336d764..2173c90015db4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java @@ -82,9 +82,9 @@ protected TypeResolution resolveType() { return isString(field(), sourceText(), DEFAULT); } - @Evaluator + @Evaluator(warnExceptions = { ArithmeticException.class }) static int process(BytesRef val) { - return val.length * Byte.SIZE; + return Math.multiplyExact(val.length, Byte.SIZE); } @Override From f4c0abeb217c99c9e86cfa71355c835c6c4cb8c4 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Wed, 30 Oct 2024 17:11:04 +0100 Subject: [PATCH 15/22] Use stringTypes() instead of manually constructing the data type array --- .../esql/expression/function/scalar/string/BitLengthTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java index b1c953dbcdc11..bce4328a08abf 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLengthTests.java @@ -34,7 +34,7 @@ public BitLengthTests(@Name("TestCase") Supplier test public static Iterable parameters() { List suppliers = new ArrayList<>(); - for (DataType stringType : new DataType[] { DataType.KEYWORD, DataType.TEXT }) { + for (DataType stringType : DataType.stringTypes()) { for (var supplier : TestCaseSupplier.stringCases(stringType)) { suppliers.add(makeSupplier(supplier)); } From a9cfaf0da73164d654da0635b2554b319d758280 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Wed, 30 Oct 2024 17:15:31 +0100 Subject: [PATCH 16/22] Use ternary operator for type resolution --- .../esql/expression/function/scalar/string/BitLength.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java index 2173c90015db4..5deb6fa7feba6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/BitLength.java @@ -75,11 +75,7 @@ public DataType dataType() { @Override protected TypeResolution resolveType() { - if (childrenResolved() == false) { - return new TypeResolution("Unresolved children"); - } - - return isString(field(), sourceText(), DEFAULT); + return childrenResolved() == false ? new TypeResolution("Unresolved children") : isString(field(), sourceText(), DEFAULT); } @Evaluator(warnExceptions = { ArithmeticException.class }) From 762be473240a74df1ae5c4b55d74e8e9af2e9dfa Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Wed, 30 Oct 2024 18:37:32 +0100 Subject: [PATCH 17/22] Only use most recent capability in usage tests --- .../resources/rest-api-spec/test/esql/60_usage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index 66abbffbf5636..7f884c91dc232 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -30,7 +30,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [ snapshot_test_for_telemetry, fn_bit_length ] + capabilities: [ fn_bit_length ] reason: "Test that should only be executed on snapshot versions" - do: {xpack.usage: {}} @@ -101,7 +101,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [ non_snapshot_test_for_telemetry, fn_bit_length ] + capabilities: [ fn_bit_length ] reason: "Test that should only be executed on release versions" - do: {xpack.usage: {}} From 9c9511bebff4b411c516a606740a8a74dd25e187 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 31 Oct 2024 14:19:46 +0100 Subject: [PATCH 18/22] Ignore ESQL usage output tests on older branches as checking the exact number of functions won't work on older versions as soon as you add at least one new function on main. --- x-pack/plugin/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index cf6a8f51d1b81..8814bea708190 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -83,7 +83,8 @@ tasks.named("precommit").configure { tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("security/10_forbidden/Test bulk response with invalid credentials", "warning does not exist for compatibility") task.skipTest("inference/inference_crud/Test get all", "Assertions on number of inference models break due to default configs") - task.skipTest("esql/60_usage/Basic ESQL usage output (telemetry)", "The telemetry output changed. We dropped a column. That's safe.") + task.skipTest("esql/60_usage/Basic ESQL usage output (telemetry) snapshot version", "The number of functions is constantly increasing") + task.skipTest("esql/60_usage/Basic ESQL usage output (telemetry) non-snapshot version", "The number of functions is constantly increasing") task.skipTest("esql/80_text/reverse text", "The output type changed from TEXT to KEYWORD.") task.skipTest("esql/80_text/values function", "The output type changed from TEXT to KEYWORD.") }) From 6dbadf274af3cd0e8838da324a619c5f7148a941 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 31 Oct 2024 14:21:12 +0100 Subject: [PATCH 19/22] Keep ignore old test for now --- x-pack/plugin/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 8814bea708190..8b920ac11cee7 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -82,6 +82,7 @@ tasks.named("precommit").configure { tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("security/10_forbidden/Test bulk response with invalid credentials", "warning does not exist for compatibility") + task.skipTest("esql/60_usage/Basic ESQL usage output (telemetry)", "The telemetry output changed. We dropped a column. That's safe.") task.skipTest("inference/inference_crud/Test get all", "Assertions on number of inference models break due to default configs") task.skipTest("esql/60_usage/Basic ESQL usage output (telemetry) snapshot version", "The number of functions is constantly increasing") task.skipTest("esql/60_usage/Basic ESQL usage output (telemetry) non-snapshot version", "The number of functions is constantly increasing") From b88922e82d52f85c67a00e4681f7e9cb2a573a43 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 31 Oct 2024 15:36:25 +0100 Subject: [PATCH 20/22] Fix expected function number for current main --- .../yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index 7f884c91dc232..6e39fd1f9f1e3 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -162,4 +162,4 @@ setup: - match: {esql.functions.cos: $functions_cos} - gt: {esql.functions.to_long: $functions_to_long} - match: {esql.functions.coalesce: $functions_coalesce} - - length: {esql.functions: 116} # check the "sister" test above for a likely update to the same esql.functions length check + - length: {esql.functions: 118} # check the "sister" test above for a likely update to the same esql.functions length check From d868e08e67523ce27adfae03025b94725ce530b5 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 31 Oct 2024 16:44:22 +0100 Subject: [PATCH 21/22] Reintroduce snapshot capabilities into usage tests --- .../resources/rest-api-spec/test/esql/60_usage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index 6e39fd1f9f1e3..df88de8da4e01 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -30,7 +30,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [ fn_bit_length ] + capabilities: [ snapshot_test_for_telemetry, fn_bit_length ] reason: "Test that should only be executed on snapshot versions" - do: {xpack.usage: {}} @@ -101,7 +101,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [ fn_bit_length ] + capabilities: [ non_snapshot_test_for_telemetry, fn_bit_length ] reason: "Test that should only be executed on release versions" - do: {xpack.usage: {}} From c708715e0dc5400594563da6eff885572f95f1b5 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Wed, 6 Nov 2024 17:55:28 +0100 Subject: [PATCH 22/22] Add required_capability: fn_bit_length to docs.csv-spec --- .../plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec index 45c0e66e7b76d..14d811535aafd 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec @@ -658,6 +658,7 @@ FROM sample_data ; docsBitLength +required_capability: fn_bit_length // tag::bitLength[] FROM employees | KEEP first_name, last_name