diff --git a/docs/reference/esql/functions/kibana/definition/mv_append.json b/docs/reference/esql/functions/kibana/definition/mv_append.json index 7cbcc678464c7..fb62fb170c953 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_append.json +++ b/docs/reference/esql/functions/kibana/definition/mv_append.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "scalar", "name" : "mv_append", - "description" : "Concatenates values of two multi-value fields.", + "description" : "Concatenates values of two or more multi-value fields.", "signatures" : [ { "params" : [ @@ -19,7 +19,7 @@ "description" : "" } ], - "variadic" : false, + "variadic" : true, "returnType" : "boolean" }, { @@ -37,7 +37,7 @@ "description" : "" } ], - "variadic" : false, + "variadic" : true, "returnType" : "cartesian_point" }, { @@ -55,7 +55,7 @@ "description" : "" } ], - "variadic" : false, + "variadic" : true, "returnType" : "cartesian_shape" }, { @@ -73,7 +73,7 @@ "description" : "" } ], - "variadic" : false, + "variadic" : true, "returnType" : "date" }, { @@ -109,7 +109,7 @@ "description" : "" } ], - "variadic" : false, + "variadic" : true, "returnType" : "double" }, { @@ -127,7 +127,7 @@ "description" : "" } ], - "variadic" : false, + "variadic" : true, "returnType" : "geo_point" }, { @@ -145,7 +145,7 @@ "description" : "" } ], - "variadic" : false, + "variadic" : true, "returnType" : "geo_shape" }, { @@ -163,7 +163,7 @@ "description" : "" } ], - "variadic" : false, + "variadic" : true, "returnType" : "integer" }, { @@ -181,7 +181,7 @@ "description" : "" } ], - "variadic" : false, + "variadic" : true, "returnType" : "ip" }, { @@ -199,7 +199,7 @@ "description" : "" } ], - "variadic" : false, + "variadic" : true, "returnType" : "keyword" }, { @@ -235,7 +235,7 @@ "description" : "" } ], - "variadic" : false, + "variadic" : true, "returnType" : "long" }, { @@ -271,7 +271,7 @@ "description" : "" } ], - "variadic" : false, + "variadic" : true, "returnType" : "keyword" }, { @@ -307,7 +307,7 @@ "description" : "" } ], - "variadic" : false, + "variadic" : true, "returnType" : "version" } ], diff --git a/docs/reference/esql/functions/kibana/docs/mv_append.md b/docs/reference/esql/functions/kibana/docs/mv_append.md index 36b285be1877c..18ace6a206ee1 100644 --- a/docs/reference/esql/functions/kibana/docs/mv_append.md +++ b/docs/reference/esql/functions/kibana/docs/mv_append.md @@ -3,5 +3,5 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### MV_APPEND -Concatenates values of two multi-value fields. +Concatenates values of two or more multi-value fields. diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java index f0044ae4774f8..2393ffd89afdd 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java @@ -210,6 +210,17 @@ private MethodSpec realEval(boolean blockStyle) { builder.addStatement("allBlocksAreNulls = false"); } builder.endControlFlow(); + } else if (a instanceof ArrayProcessFunctionArg aa) { + builder.beginControlFlow("for (int i = 0; i < $L.length; i++)", aa.paramName(true)); + { + builder.beginControlFlow("if (!$L[i].isNull(p))", aa.paramName(true)); + { + builder.addStatement("allBlocksAreNulls = false"); + } + builder.endControlFlow(); + } + builder.endControlFlow(); + } }); @@ -671,6 +682,12 @@ public void resolveVectors(MethodSpec.Builder builder, String invokeBlockEval) { @Override public void createScratch(MethodSpec.Builder builder) { + if (componentType.equals(INT_BLOCK) || componentType.equals(LONG_BLOCK) || componentType.equals(DOUBLE_BLOCK) + || componentType.equals(BOOLEAN_BLOCK) || componentType.equals(BYTES_REF_BLOCK)) { + // if componentType is a block, it is a multi-value field, we just assign the block + builder.addStatement("$T[] $LValues = $L", componentType, name, paramName(true)); + return; + } builder.addStatement("$T[] $LValues = new $T[$L.length]", componentType, name, componentType, name); if (componentType.equals(BYTES_REF)) { builder.addStatement("$T[] $LScratch = new $T[$L.length]", componentType, name, componentType, name); @@ -689,6 +706,12 @@ public void skipNull(MethodSpec.Builder builder) { @Override public void unpackValues(MethodSpec.Builder builder, boolean blockStyle) { + if (componentType.equals(INT_BLOCK) || componentType.equals(LONG_BLOCK) || componentType.equals(DOUBLE_BLOCK) + || componentType.equals(BOOLEAN_BLOCK) || componentType.equals(BYTES_REF_BLOCK)) { + // if componentType is a block, we don't need to unpack it, it is a multi-value field + // nothing to do + return; + } builder.addComment("unpack $L into $LValues", paramName(blockStyle), name); builder.beginControlFlow("for (int i = 0; i < $L.length; i++)", paramName(blockStyle)); String lookupVar; diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java index 35c42153f9ad6..31a2531eb982d 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java @@ -120,7 +120,6 @@ public class Types { static final ClassName SOURCE = ClassName.get("org.elasticsearch.xpack.esql.core.tree", "Source"); static final ClassName BYTES_REF = ClassName.get("org.apache.lucene.util", "BytesRef"); - static final ClassName RELEASABLE = ClassName.get("org.elasticsearch.core", "Releasable"); static final ClassName RELEASABLES = ClassName.get("org.elasticsearch.core", "Releasables"); @@ -137,8 +136,14 @@ public static TypeDef of(TypeName type, String alias, String block, String vecto TypeDef.of(TypeName.LONG, "LONG", "LongBlock", "LongVector"), TypeDef.of(TypeName.FLOAT, "FLOAT", "FloatBlock", "FloatVector"), TypeDef.of(TypeName.DOUBLE, "DOUBLE", "DoubleBlock", "DoubleVector"), - TypeDef.of(BYTES_REF, "BYTES_REF", "BytesRefBlock", "BytesRefVector") - ) + TypeDef.of(BYTES_REF, "BYTES_REF", "BytesRefBlock", "BytesRefVector"), + TypeDef.of(BOOLEAN_BLOCK, "BOOLEAN_BLOCK", "BooleanBlock", "BooleanVector"), + TypeDef.of(INT_BLOCK, "INT_BLOCK", "IntBlock", "IntVector"), + TypeDef.of(LONG_BLOCK, "LONG_BLOCK", "LongBlock", "LongVector"), + TypeDef.of(FLOAT_BLOCK, "FLOAT_BLOCK", "FloatBlock", "FloatVector"), + TypeDef.of(DOUBLE_BLOCK, "DOUBLE_BLOCK", "DoubleBlock", "DoubleVector"), + TypeDef.of(BYTES_REF_BLOCK, "BYTES_REF_BLOCK", "BytesRefBlock", "BytesRefVector") + ) .flatMap(def -> Stream.of(def.type.toString(), def.type + "[]", def.alias).map(alias -> Map.entry(alias, def))) .collect(toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec index 5e7e78fd95b7c..c9854dd2ec807 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec @@ -536,12 +536,12 @@ FROM employees i = mv_append(salary_change.int, salary_change.int), i2 = mv_append(emp_no, salary_change.int), i3 = mv_append(emp_no, emp_no), - s = mv_append(salary_change.keyword, salary_change.keyword) + s = mv_append(salary_change.keyword, salary_change.keyword, salary_change.keyword) | KEEP emp_no, salary_change, d, i, i2, i3, s | SORT emp_no; emp_no:integer | salary_change:double | d:double | i:integer | i2:integer | i3:integer | s:keyword -10008 | [-2.92,0.75,3.54,12.68] | [-2.92,0.75,3.54,12.68,-2.92,0.75,3.54,12.68] | [-2,0,3,12,-2,0,3,12] | [10008,-2,0,3,12] | [10008, 10008] | [-2.92,0.75,12.68,3.54,-2.92,0.75,12.68,3.54] +10008 | [-2.92,0.75,3.54,12.68] | [-2.92,0.75,3.54,12.68,-2.92,0.75,3.54,12.68] | [-2,0,3,12,-2,0,3,12] | [10008,-2,0,3,12] | [10008, 10008] | [-2.92,0.75,12.68,3.54,-2.92,0.75,12.68,3.54,-2.92,0.75,12.68,3.54] 10021 | null | null | null | null | [10021, 10021] | null ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index d2066c5ebe321..186552882b5e5 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -1725,7 +1725,7 @@ mvAppend required_capability: fn_mv_append ROW a = "a", b = ["b", "c"], n = null -| EVAL aa = mv_append(a, a), bb = mv_append(b, b), ab = mv_append(a, b), abb = mv_append(mv_append(a, b), b), na = mv_append(n, a), an = mv_append(a, n) +| EVAL aa = mv_append(a, a), bb = mv_append(b, b), ab = mv_append(a, b), abb = mv_append(a, b, b), na = mv_append(n, a), an = mv_append(a, n) ; a:keyword | b:keyword | n:null | aa:keyword | bb:keyword | ab:keyword | abb:keyword | na:keyword | an:keyword diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index 8b19bc589fcff..8774b9c9a3fb1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -1578,7 +1578,7 @@ FROM sample_data, sample_data_ts_long multiIndexIndirectUseOfUnionTypesInMvExpand required_capability: union_types FROM sample_data, sample_data_ts_long -| EVAL foo = MV_APPEND(message, message) +| EVAL foo = MV_APPEND(message, message, message) | SORT client_ip ASC | LIMIT 1 | MV_EXPAND foo @@ -1587,6 +1587,7 @@ FROM sample_data, sample_data_ts_long @timestamp:unsupported | client_ip:ip | event_duration:long | message:keyword | foo:keyword null | 172.21.0.5 | 1232382 | Disconnected | Disconnected null | 172.21.0.5 | 1232382 | Disconnected | Disconnected + null | 172.21.0.5 | 1232382 | Disconnected | Disconnected ; shortIntegerWidening diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec index eb0d6d75a7d07..ec7ce51f1a419 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/version.csv-spec @@ -377,7 +377,7 @@ required_capability: fn_mv_append ROW a = to_version("1.2.0"), x1 = to_version("0.0.1"), x2 = to_version("1.0.0") | EVAL b = mv_append(x1, x2) -| EVAL aa = mv_append(a, a), bb = mv_append(b, b), ab = mv_append(a, b), abb = mv_append(mv_append(a, b), b) +| EVAL aa = mv_append(a, a), bb = mv_append(b, b), ab = mv_append(a, b), abb = mv_append(a, b, b) | KEEP a, b, aa, bb, ab, abb ; diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBooleanEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBooleanEvaluator.java index c126bd7bef196..b3c34d67152a1 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBooleanEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBooleanEvaluator.java @@ -6,12 +6,14 @@ import java.lang.Override; import java.lang.String; +import java.util.Arrays; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -24,14 +26,14 @@ public final class MvAppendBooleanEvaluator implements EvalOperator.ExpressionEv private final EvalOperator.ExpressionEvaluator field1; - private final EvalOperator.ExpressionEvaluator field2; + private final EvalOperator.ExpressionEvaluator[] field2; private final DriverContext driverContext; private Warnings warnings; public MvAppendBooleanEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, - EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { + EvalOperator.ExpressionEvaluator[] field2, DriverContext driverContext) { this.source = source; this.field1 = field1; this.field2 = field2; @@ -41,27 +43,35 @@ public MvAppendBooleanEvaluator(Source source, EvalOperator.ExpressionEvaluator @Override public Block eval(Page page) { try (BooleanBlock field1Block = (BooleanBlock) field1.eval(page)) { - try (BooleanBlock field2Block = (BooleanBlock) field2.eval(page)) { - return eval(page.getPositionCount(), field1Block, field2Block); + BooleanBlock[] field2Blocks = new BooleanBlock[field2.length]; + try (Releasable field2Release = Releasables.wrap(field2Blocks)) { + for (int i = 0; i < field2Blocks.length; i++) { + field2Blocks[i] = (BooleanBlock)field2[i].eval(page); + } + return eval(page.getPositionCount(), field1Block, field2Blocks); } } } - public BooleanBlock eval(int positionCount, BooleanBlock field1Block, BooleanBlock field2Block) { + public BooleanBlock eval(int positionCount, BooleanBlock field1Block, + BooleanBlock[] field2Blocks) { try(BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + BooleanBlock[] field2Values = field2Blocks; position: for (int p = 0; p < positionCount; p++) { boolean allBlocksAreNulls = true; if (!field1Block.isNull(p)) { allBlocksAreNulls = false; } - if (!field2Block.isNull(p)) { - allBlocksAreNulls = false; + for (int i = 0; i < field2Blocks.length; i++) { + if (!field2Blocks[i].isNull(p)) { + allBlocksAreNulls = false; + } } if (allBlocksAreNulls) { result.appendNull(); continue position; } - MvAppend.process(result, p, field1Block, field2Block); + MvAppend.process(result, p, field1Block, field2Values); } return result.build(); } @@ -69,12 +79,12 @@ public BooleanBlock eval(int positionCount, BooleanBlock field1Block, BooleanBlo @Override public String toString() { - return "MvAppendBooleanEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + return "MvAppendBooleanEvaluator[" + "field1=" + field1 + ", field2=" + Arrays.toString(field2) + "]"; } @Override public void close() { - Releasables.closeExpectNoException(field1, field2); + Releasables.closeExpectNoException(field1, () -> Releasables.close(field2)); } private Warnings warnings() { @@ -94,10 +104,10 @@ static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final EvalOperator.ExpressionEvaluator.Factory field1; - private final EvalOperator.ExpressionEvaluator.Factory field2; + private final EvalOperator.ExpressionEvaluator.Factory[] field2; public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, - EvalOperator.ExpressionEvaluator.Factory field2) { + EvalOperator.ExpressionEvaluator.Factory[] field2) { this.source = source; this.field1 = field1; this.field2 = field2; @@ -105,12 +115,13 @@ public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, @Override public MvAppendBooleanEvaluator get(DriverContext context) { - return new MvAppendBooleanEvaluator(source, field1.get(context), field2.get(context), context); + EvalOperator.ExpressionEvaluator[] field2 = Arrays.stream(this.field2).map(a -> a.get(context)).toArray(EvalOperator.ExpressionEvaluator[]::new); + return new MvAppendBooleanEvaluator(source, field1.get(context), field2, context); } @Override public String toString() { - return "MvAppendBooleanEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + return "MvAppendBooleanEvaluator[" + "field1=" + field1 + ", field2=" + Arrays.toString(field2) + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBytesRefEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBytesRefEvaluator.java index 3afd3534b92f6..92f426381f431 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBytesRefEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendBytesRefEvaluator.java @@ -6,12 +6,14 @@ import java.lang.Override; import java.lang.String; +import java.util.Arrays; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -24,14 +26,14 @@ public final class MvAppendBytesRefEvaluator implements EvalOperator.ExpressionE private final EvalOperator.ExpressionEvaluator field1; - private final EvalOperator.ExpressionEvaluator field2; + private final EvalOperator.ExpressionEvaluator[] field2; private final DriverContext driverContext; private Warnings warnings; public MvAppendBytesRefEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, - EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { + EvalOperator.ExpressionEvaluator[] field2, DriverContext driverContext) { this.source = source; this.field1 = field1; this.field2 = field2; @@ -41,28 +43,35 @@ public MvAppendBytesRefEvaluator(Source source, EvalOperator.ExpressionEvaluator @Override public Block eval(Page page) { try (BytesRefBlock field1Block = (BytesRefBlock) field1.eval(page)) { - try (BytesRefBlock field2Block = (BytesRefBlock) field2.eval(page)) { - return eval(page.getPositionCount(), field1Block, field2Block); + BytesRefBlock[] field2Blocks = new BytesRefBlock[field2.length]; + try (Releasable field2Release = Releasables.wrap(field2Blocks)) { + for (int i = 0; i < field2Blocks.length; i++) { + field2Blocks[i] = (BytesRefBlock)field2[i].eval(page); + } + return eval(page.getPositionCount(), field1Block, field2Blocks); } } } public BytesRefBlock eval(int positionCount, BytesRefBlock field1Block, - BytesRefBlock field2Block) { + BytesRefBlock[] field2Blocks) { try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + BytesRefBlock[] field2Values = field2Blocks; position: for (int p = 0; p < positionCount; p++) { boolean allBlocksAreNulls = true; if (!field1Block.isNull(p)) { allBlocksAreNulls = false; } - if (!field2Block.isNull(p)) { - allBlocksAreNulls = false; + for (int i = 0; i < field2Blocks.length; i++) { + if (!field2Blocks[i].isNull(p)) { + allBlocksAreNulls = false; + } } if (allBlocksAreNulls) { result.appendNull(); continue position; } - MvAppend.process(result, p, field1Block, field2Block); + MvAppend.process(result, p, field1Block, field2Values); } return result.build(); } @@ -70,12 +79,12 @@ public BytesRefBlock eval(int positionCount, BytesRefBlock field1Block, @Override public String toString() { - return "MvAppendBytesRefEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + return "MvAppendBytesRefEvaluator[" + "field1=" + field1 + ", field2=" + Arrays.toString(field2) + "]"; } @Override public void close() { - Releasables.closeExpectNoException(field1, field2); + Releasables.closeExpectNoException(field1, () -> Releasables.close(field2)); } private Warnings warnings() { @@ -95,10 +104,10 @@ static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final EvalOperator.ExpressionEvaluator.Factory field1; - private final EvalOperator.ExpressionEvaluator.Factory field2; + private final EvalOperator.ExpressionEvaluator.Factory[] field2; public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, - EvalOperator.ExpressionEvaluator.Factory field2) { + EvalOperator.ExpressionEvaluator.Factory[] field2) { this.source = source; this.field1 = field1; this.field2 = field2; @@ -106,12 +115,13 @@ public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, @Override public MvAppendBytesRefEvaluator get(DriverContext context) { - return new MvAppendBytesRefEvaluator(source, field1.get(context), field2.get(context), context); + EvalOperator.ExpressionEvaluator[] field2 = Arrays.stream(this.field2).map(a -> a.get(context)).toArray(EvalOperator.ExpressionEvaluator[]::new); + return new MvAppendBytesRefEvaluator(source, field1.get(context), field2, context); } @Override public String toString() { - return "MvAppendBytesRefEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + return "MvAppendBytesRefEvaluator[" + "field1=" + field1 + ", field2=" + Arrays.toString(field2) + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendDoubleEvaluator.java index 315150a20e354..35393314c5361 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendDoubleEvaluator.java @@ -6,12 +6,14 @@ import java.lang.Override; import java.lang.String; +import java.util.Arrays; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -24,14 +26,14 @@ public final class MvAppendDoubleEvaluator implements EvalOperator.ExpressionEva private final EvalOperator.ExpressionEvaluator field1; - private final EvalOperator.ExpressionEvaluator field2; + private final EvalOperator.ExpressionEvaluator[] field2; private final DriverContext driverContext; private Warnings warnings; public MvAppendDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, - EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { + EvalOperator.ExpressionEvaluator[] field2, DriverContext driverContext) { this.source = source; this.field1 = field1; this.field2 = field2; @@ -41,27 +43,34 @@ public MvAppendDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator f @Override public Block eval(Page page) { try (DoubleBlock field1Block = (DoubleBlock) field1.eval(page)) { - try (DoubleBlock field2Block = (DoubleBlock) field2.eval(page)) { - return eval(page.getPositionCount(), field1Block, field2Block); + DoubleBlock[] field2Blocks = new DoubleBlock[field2.length]; + try (Releasable field2Release = Releasables.wrap(field2Blocks)) { + for (int i = 0; i < field2Blocks.length; i++) { + field2Blocks[i] = (DoubleBlock)field2[i].eval(page); + } + return eval(page.getPositionCount(), field1Block, field2Blocks); } } } - public DoubleBlock eval(int positionCount, DoubleBlock field1Block, DoubleBlock field2Block) { + public DoubleBlock eval(int positionCount, DoubleBlock field1Block, DoubleBlock[] field2Blocks) { try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + DoubleBlock[] field2Values = field2Blocks; position: for (int p = 0; p < positionCount; p++) { boolean allBlocksAreNulls = true; if (!field1Block.isNull(p)) { allBlocksAreNulls = false; } - if (!field2Block.isNull(p)) { - allBlocksAreNulls = false; + for (int i = 0; i < field2Blocks.length; i++) { + if (!field2Blocks[i].isNull(p)) { + allBlocksAreNulls = false; + } } if (allBlocksAreNulls) { result.appendNull(); continue position; } - MvAppend.process(result, p, field1Block, field2Block); + MvAppend.process(result, p, field1Block, field2Values); } return result.build(); } @@ -69,12 +78,12 @@ public DoubleBlock eval(int positionCount, DoubleBlock field1Block, DoubleBlock @Override public String toString() { - return "MvAppendDoubleEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + return "MvAppendDoubleEvaluator[" + "field1=" + field1 + ", field2=" + Arrays.toString(field2) + "]"; } @Override public void close() { - Releasables.closeExpectNoException(field1, field2); + Releasables.closeExpectNoException(field1, () -> Releasables.close(field2)); } private Warnings warnings() { @@ -94,10 +103,10 @@ static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final EvalOperator.ExpressionEvaluator.Factory field1; - private final EvalOperator.ExpressionEvaluator.Factory field2; + private final EvalOperator.ExpressionEvaluator.Factory[] field2; public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, - EvalOperator.ExpressionEvaluator.Factory field2) { + EvalOperator.ExpressionEvaluator.Factory[] field2) { this.source = source; this.field1 = field1; this.field2 = field2; @@ -105,12 +114,13 @@ public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, @Override public MvAppendDoubleEvaluator get(DriverContext context) { - return new MvAppendDoubleEvaluator(source, field1.get(context), field2.get(context), context); + EvalOperator.ExpressionEvaluator[] field2 = Arrays.stream(this.field2).map(a -> a.get(context)).toArray(EvalOperator.ExpressionEvaluator[]::new); + return new MvAppendDoubleEvaluator(source, field1.get(context), field2, context); } @Override public String toString() { - return "MvAppendDoubleEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + return "MvAppendDoubleEvaluator[" + "field1=" + field1 + ", field2=" + Arrays.toString(field2) + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendIntEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendIntEvaluator.java index 0291e8c07d9ff..c284f9c698587 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendIntEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendIntEvaluator.java @@ -6,12 +6,14 @@ import java.lang.Override; import java.lang.String; +import java.util.Arrays; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -24,14 +26,14 @@ public final class MvAppendIntEvaluator implements EvalOperator.ExpressionEvalua private final EvalOperator.ExpressionEvaluator field1; - private final EvalOperator.ExpressionEvaluator field2; + private final EvalOperator.ExpressionEvaluator[] field2; private final DriverContext driverContext; private Warnings warnings; public MvAppendIntEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, - EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { + EvalOperator.ExpressionEvaluator[] field2, DriverContext driverContext) { this.source = source; this.field1 = field1; this.field2 = field2; @@ -41,27 +43,34 @@ public MvAppendIntEvaluator(Source source, EvalOperator.ExpressionEvaluator fiel @Override public Block eval(Page page) { try (IntBlock field1Block = (IntBlock) field1.eval(page)) { - try (IntBlock field2Block = (IntBlock) field2.eval(page)) { - return eval(page.getPositionCount(), field1Block, field2Block); + IntBlock[] field2Blocks = new IntBlock[field2.length]; + try (Releasable field2Release = Releasables.wrap(field2Blocks)) { + for (int i = 0; i < field2Blocks.length; i++) { + field2Blocks[i] = (IntBlock)field2[i].eval(page); + } + return eval(page.getPositionCount(), field1Block, field2Blocks); } } } - public IntBlock eval(int positionCount, IntBlock field1Block, IntBlock field2Block) { + public IntBlock eval(int positionCount, IntBlock field1Block, IntBlock[] field2Blocks) { try(IntBlock.Builder result = driverContext.blockFactory().newIntBlockBuilder(positionCount)) { + IntBlock[] field2Values = field2Blocks; position: for (int p = 0; p < positionCount; p++) { boolean allBlocksAreNulls = true; if (!field1Block.isNull(p)) { allBlocksAreNulls = false; } - if (!field2Block.isNull(p)) { - allBlocksAreNulls = false; + for (int i = 0; i < field2Blocks.length; i++) { + if (!field2Blocks[i].isNull(p)) { + allBlocksAreNulls = false; + } } if (allBlocksAreNulls) { result.appendNull(); continue position; } - MvAppend.process(result, p, field1Block, field2Block); + MvAppend.process(result, p, field1Block, field2Values); } return result.build(); } @@ -69,12 +78,12 @@ public IntBlock eval(int positionCount, IntBlock field1Block, IntBlock field2Blo @Override public String toString() { - return "MvAppendIntEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + return "MvAppendIntEvaluator[" + "field1=" + field1 + ", field2=" + Arrays.toString(field2) + "]"; } @Override public void close() { - Releasables.closeExpectNoException(field1, field2); + Releasables.closeExpectNoException(field1, () -> Releasables.close(field2)); } private Warnings warnings() { @@ -94,10 +103,10 @@ static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final EvalOperator.ExpressionEvaluator.Factory field1; - private final EvalOperator.ExpressionEvaluator.Factory field2; + private final EvalOperator.ExpressionEvaluator.Factory[] field2; public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, - EvalOperator.ExpressionEvaluator.Factory field2) { + EvalOperator.ExpressionEvaluator.Factory[] field2) { this.source = source; this.field1 = field1; this.field2 = field2; @@ -105,12 +114,13 @@ public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, @Override public MvAppendIntEvaluator get(DriverContext context) { - return new MvAppendIntEvaluator(source, field1.get(context), field2.get(context), context); + EvalOperator.ExpressionEvaluator[] field2 = Arrays.stream(this.field2).map(a -> a.get(context)).toArray(EvalOperator.ExpressionEvaluator[]::new); + return new MvAppendIntEvaluator(source, field1.get(context), field2, context); } @Override public String toString() { - return "MvAppendIntEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + return "MvAppendIntEvaluator[" + "field1=" + field1 + ", field2=" + Arrays.toString(field2) + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendLongEvaluator.java index c23d036550fc8..be6e3f1de2663 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendLongEvaluator.java @@ -6,12 +6,14 @@ import java.lang.Override; import java.lang.String; +import java.util.Arrays; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -24,14 +26,14 @@ public final class MvAppendLongEvaluator implements EvalOperator.ExpressionEvalu private final EvalOperator.ExpressionEvaluator field1; - private final EvalOperator.ExpressionEvaluator field2; + private final EvalOperator.ExpressionEvaluator[] field2; private final DriverContext driverContext; private Warnings warnings; public MvAppendLongEvaluator(Source source, EvalOperator.ExpressionEvaluator field1, - EvalOperator.ExpressionEvaluator field2, DriverContext driverContext) { + EvalOperator.ExpressionEvaluator[] field2, DriverContext driverContext) { this.source = source; this.field1 = field1; this.field2 = field2; @@ -41,27 +43,34 @@ public MvAppendLongEvaluator(Source source, EvalOperator.ExpressionEvaluator fie @Override public Block eval(Page page) { try (LongBlock field1Block = (LongBlock) field1.eval(page)) { - try (LongBlock field2Block = (LongBlock) field2.eval(page)) { - return eval(page.getPositionCount(), field1Block, field2Block); + LongBlock[] field2Blocks = new LongBlock[field2.length]; + try (Releasable field2Release = Releasables.wrap(field2Blocks)) { + for (int i = 0; i < field2Blocks.length; i++) { + field2Blocks[i] = (LongBlock)field2[i].eval(page); + } + return eval(page.getPositionCount(), field1Block, field2Blocks); } } } - public LongBlock eval(int positionCount, LongBlock field1Block, LongBlock field2Block) { + public LongBlock eval(int positionCount, LongBlock field1Block, LongBlock[] field2Blocks) { try(LongBlock.Builder result = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + LongBlock[] field2Values = field2Blocks; position: for (int p = 0; p < positionCount; p++) { boolean allBlocksAreNulls = true; if (!field1Block.isNull(p)) { allBlocksAreNulls = false; } - if (!field2Block.isNull(p)) { - allBlocksAreNulls = false; + for (int i = 0; i < field2Blocks.length; i++) { + if (!field2Blocks[i].isNull(p)) { + allBlocksAreNulls = false; + } } if (allBlocksAreNulls) { result.appendNull(); continue position; } - MvAppend.process(result, p, field1Block, field2Block); + MvAppend.process(result, p, field1Block, field2Values); } return result.build(); } @@ -69,12 +78,12 @@ public LongBlock eval(int positionCount, LongBlock field1Block, LongBlock field2 @Override public String toString() { - return "MvAppendLongEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + return "MvAppendLongEvaluator[" + "field1=" + field1 + ", field2=" + Arrays.toString(field2) + "]"; } @Override public void close() { - Releasables.closeExpectNoException(field1, field2); + Releasables.closeExpectNoException(field1, () -> Releasables.close(field2)); } private Warnings warnings() { @@ -94,10 +103,10 @@ static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final EvalOperator.ExpressionEvaluator.Factory field1; - private final EvalOperator.ExpressionEvaluator.Factory field2; + private final EvalOperator.ExpressionEvaluator.Factory[] field2; public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, - EvalOperator.ExpressionEvaluator.Factory field2) { + EvalOperator.ExpressionEvaluator.Factory[] field2) { this.source = source; this.field1 = field1; this.field2 = field2; @@ -105,12 +114,13 @@ public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory field1, @Override public MvAppendLongEvaluator get(DriverContext context) { - return new MvAppendLongEvaluator(source, field1.get(context), field2.get(context), context); + EvalOperator.ExpressionEvaluator[] field2 = Arrays.stream(this.field2).map(a -> a.get(context)).toArray(EvalOperator.ExpressionEvaluator[]::new); + return new MvAppendLongEvaluator(source, field1.get(context), field2, context); } @Override public String toString() { - return "MvAppendLongEvaluator[" + "field1=" + field1 + ", field2=" + field2 + "]"; + return "MvAppendLongEvaluator[" + "field1=" + field1 + ", field2=" + Arrays.toString(field2) + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java index 7ec7d1b9b2eca..0b67a5f332eba 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java @@ -32,12 +32,12 @@ import org.elasticsearch.xpack.esql.planner.PlannerUtils; import java.io.IOException; -import java.util.Arrays; import java.util.List; import java.util.Objects; +import java.util.stream.Stream; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; /** @@ -46,7 +46,8 @@ public class MvAppend extends EsqlScalarFunction implements EvaluatorMapper { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvAppend", MvAppend::new); - private final Expression field1, field2; + private final Expression field1; + private final List field2; private DataType dataType; @FunctionInfo( @@ -65,7 +66,7 @@ public class MvAppend extends EsqlScalarFunction implements EvaluatorMapper { "long", "unsigned_long", "version" }, - description = "Concatenates values of two multi-value fields." + description = "Concatenates values of two or more multi-value fields." ) public MvAppend( Source source, @@ -106,22 +107,23 @@ public MvAppend( "text", "unsigned_long", "version" } - ) Expression field2 + ) List field2 ) { - super(source, Arrays.asList(field1, field2)); + super(source, Stream.concat(Stream.of(field1), field2.stream()).toList()); this.field1 = field1; this.field2 = field2; } private MvAppend(StreamInput in) throws IOException { - this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class)); + this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), + in.readNamedWriteableCollectionAsList(Expression.class)); } @Override public void writeTo(StreamOutput out) throws IOException { Source.EMPTY.writeTo(out); out.writeNamedWriteable(field1); - out.writeNamedWriteable(field2); + out.writeNamedWriteableCollection(field2); } @Override @@ -141,25 +143,39 @@ protected TypeResolution resolveType() { } dataType = field1.dataType().noText(); if (dataType == DataType.NULL) { - dataType = field2.dataType().noText(); - return isType(field2, DataType::isRepresentable, sourceText(), SECOND, "representable"); + for (Expression value : field2) { + dataType = value.dataType().noText(); + resolution = isType(value, DataType::isRepresentable, sourceText(), DEFAULT, "representable"); + if (resolution.unresolved()) { + return resolution; + } + } } - return isType(field2, t -> t.noText() == dataType, sourceText(), SECOND, dataType.typeName()); + return resolution; } @Override public boolean foldable() { - return field1.foldable() && field2.foldable(); + boolean foldable = field1.foldable(); + for (Expression expression : field2) { + foldable &= expression.foldable(); + } + return foldable; } @Override public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return switch (PlannerUtils.toElementType(dataType())) { - case BOOLEAN -> new MvAppendBooleanEvaluator.Factory(source(), toEvaluator.apply(field1), toEvaluator.apply(field2)); - case BYTES_REF -> new MvAppendBytesRefEvaluator.Factory(source(), toEvaluator.apply(field1), toEvaluator.apply(field2)); - case DOUBLE -> new MvAppendDoubleEvaluator.Factory(source(), toEvaluator.apply(field1), toEvaluator.apply(field2)); - case INT -> new MvAppendIntEvaluator.Factory(source(), toEvaluator.apply(field1), toEvaluator.apply(field2)); - case LONG -> new MvAppendLongEvaluator.Factory(source(), toEvaluator.apply(field1), toEvaluator.apply(field2)); + case BOOLEAN -> new MvAppendBooleanEvaluator.Factory(source(), toEvaluator.apply(field1), + field2.stream().map(toEvaluator::apply).toArray(EvalOperator.ExpressionEvaluator.Factory[]::new)); + case BYTES_REF -> new MvAppendBytesRefEvaluator.Factory(source(), toEvaluator.apply(field1), + field2.stream().map(toEvaluator::apply).toArray(EvalOperator.ExpressionEvaluator.Factory[]::new)); + case DOUBLE -> new MvAppendDoubleEvaluator.Factory(source(), toEvaluator.apply(field1), + field2.stream().map(toEvaluator::apply).toArray(EvalOperator.ExpressionEvaluator.Factory[]::new)); + case INT -> new MvAppendIntEvaluator.Factory(source(), toEvaluator.apply(field1), field2.stream().map(toEvaluator::apply).toArray( + EvalOperator.ExpressionEvaluator.Factory[]::new)); + case LONG -> new MvAppendLongEvaluator.Factory(source(), toEvaluator.apply(field1), field2.stream().map(toEvaluator::apply).toArray( + EvalOperator.ExpressionEvaluator.Factory[]::new)); case NULL -> EvalOperator.CONSTANT_NULL_FACTORY; default -> throw EsqlIllegalArgumentException.illegalDataType(dataType); }; @@ -167,7 +183,7 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua @Override public Expression replaceChildren(List newChildren) { - return new MvAppend(source(), newChildren.get(0), newChildren.get(1)); + return new MvAppend(source(), newChildren.get(0), newChildren.subList(1, newChildren.size())); } @Override @@ -198,20 +214,29 @@ public boolean equals(Object obj) { } @Evaluator(extraName = "Int") - static void process(IntBlock.Builder builder, int position, IntBlock field1, IntBlock field2) { + static void process(IntBlock.Builder builder, int position, IntBlock field1, IntBlock... field2) { int count1 = field1.getValueCount(position); - int count2 = field2.getValueCount(position); - if (count1 == 0 || count2 == 0) { + boolean field2AllCountZero = true; + for (IntBlock block : field2) { + if (block.getValueCount(position) > 0) { + field2AllCountZero = false; + break; + } + } + if (count1 == 0 || field2AllCountZero) { builder.appendNull(); } else { builder.beginPositionEntry(); int first1 = field1.getFirstValueIndex(position); - int first2 = field2.getFirstValueIndex(position); for (int i = 0; i < count1; i++) { builder.appendInt(field1.getInt(first1 + i)); } - for (int i = 0; i < count2; i++) { - builder.appendInt(field2.getInt(first2 + i)); + + for (int i = 0; i < field2.length; i++) { + int first = field2[i].getFirstValueIndex(position); + for (int j = 0; j < field2[i].getValueCount(position); j++) { + builder.appendInt(field2[i].getInt(first + j)); + } } builder.endPositionEntry(); } @@ -219,20 +244,29 @@ static void process(IntBlock.Builder builder, int position, IntBlock field1, Int } @Evaluator(extraName = "Boolean") - static void process(BooleanBlock.Builder builder, int position, BooleanBlock field1, BooleanBlock field2) { + static void process(BooleanBlock.Builder builder, int position, BooleanBlock field1, BooleanBlock... field2) { int count1 = field1.getValueCount(position); - int count2 = field2.getValueCount(position); - if (count1 == 0 || count2 == 0) { + boolean field2AllCountZero = true; + for (BooleanBlock block : field2) { + if (block.getValueCount(position) > 0) { + field2AllCountZero = false; + break; + } + } + if (count1 == 0 || field2AllCountZero) { builder.appendNull(); } else { int first1 = field1.getFirstValueIndex(position); - int first2 = field2.getFirstValueIndex(position); builder.beginPositionEntry(); for (int i = 0; i < count1; i++) { builder.appendBoolean(field1.getBoolean(first1 + i)); } - for (int i = 0; i < count2; i++) { - builder.appendBoolean(field2.getBoolean(first2 + i)); + + for (int i = 0; i < field2.length; i++) { + int first = field2[i].getFirstValueIndex(position); + for (int j = 0; j < field2[i].getValueCount(position); j++) { + builder.appendBoolean(field2[i].getBoolean(first + j)); + } } builder.endPositionEntry(); } @@ -240,40 +274,58 @@ static void process(BooleanBlock.Builder builder, int position, BooleanBlock fie } @Evaluator(extraName = "Long") - static void process(LongBlock.Builder builder, int position, LongBlock field1, LongBlock field2) { + static void process(LongBlock.Builder builder, int position, LongBlock field1, LongBlock... field2) { int count1 = field1.getValueCount(position); - int count2 = field2.getValueCount(position); - if (count1 == 0 || count2 == 0) { + boolean field2AllCountZero = true; + for (LongBlock block : field2) { + if (block.getValueCount(position) > 0) { + field2AllCountZero = false; + break; + } + } + if (count1 == 0 || field2AllCountZero) { builder.appendNull(); } else { int first1 = field1.getFirstValueIndex(position); - int first2 = field2.getFirstValueIndex(position); builder.beginPositionEntry(); for (int i = 0; i < count1; i++) { builder.appendLong(field1.getLong(first1 + i)); } - for (int i = 0; i < count2; i++) { - builder.appendLong(field2.getLong(first2 + i)); + + for (int i = 0; i < field2.length; i++) { + int first = field2[i].getFirstValueIndex(position); + for (int j = 0; j < field2[i].getValueCount(position); j++) { + builder.appendLong(field2[i].getLong(first + j)); + } } builder.endPositionEntry(); } } @Evaluator(extraName = "Double") - static void process(DoubleBlock.Builder builder, int position, DoubleBlock field1, DoubleBlock field2) { + static void process(DoubleBlock.Builder builder, int position, DoubleBlock field1, DoubleBlock... field2) { int count1 = field1.getValueCount(position); - int count2 = field2.getValueCount(position); - if (count1 == 0 || count2 == 0) { + boolean field2AllCountZero = true; + for (DoubleBlock block : field2) { + if (block.getValueCount(position) > 0) { + field2AllCountZero = false; + break; + } + } + if (count1 == 0 || field2AllCountZero) { builder.appendNull(); } else { int first1 = field1.getFirstValueIndex(position); - int first2 = field2.getFirstValueIndex(position); builder.beginPositionEntry(); for (int i = 0; i < count1; i++) { builder.appendDouble(field1.getDouble(first1 + i)); } - for (int i = 0; i < count2; i++) { - builder.appendDouble(field2.getDouble(first2 + i)); + + for (int i = 0; i < field2.length; i++) { + int first = field2[i].getFirstValueIndex(position); + for (int j = 0; j < field2[i].getValueCount(position); j++) { + builder.appendDouble(field2[i].getDouble(first + j)); + } } builder.endPositionEntry(); } @@ -281,21 +333,30 @@ static void process(DoubleBlock.Builder builder, int position, DoubleBlock field } @Evaluator(extraName = "BytesRef") - static void process(BytesRefBlock.Builder builder, int position, BytesRefBlock field1, BytesRefBlock field2) { + static void process(BytesRefBlock.Builder builder, int position, BytesRefBlock field1, BytesRefBlock... field2) { int count1 = field1.getValueCount(position); - int count2 = field2.getValueCount(position); - if (count1 == 0 || count2 == 0) { + boolean field2AllCountZero = true; + for (BytesRefBlock block : field2) { + if (block.getValueCount(position) > 0) { + field2AllCountZero = false; + break; + } + } + if (count1 == 0 || field2AllCountZero) { builder.appendNull(); } else { int first1 = field1.getFirstValueIndex(position); - int first2 = field2.getFirstValueIndex(position); builder.beginPositionEntry(); BytesRef spare = new BytesRef(); for (int i = 0; i < count1; i++) { builder.appendBytesRef(field1.getBytesRef(first1 + i, spare)); } - for (int i = 0; i < count2; i++) { - builder.appendBytesRef(field2.getBytesRef(first2 + i, spare)); + + for (int i = 0; i < field2.length; i++) { + int first = field2[i].getFirstValueIndex(position); + for (int j = 0; j < field2[i].getValueCount(position); j++) { + builder.appendBytesRef(field2[i].getBytesRef(first + j, spare)); + } } builder.endPositionEntry(); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendErrorTests.java index df9ab4764c879..c41177de85777 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendErrorTests.java @@ -27,7 +27,7 @@ protected List cases() { @Override protected Expression build(Source source, List args) { - return new MvAppend(source, args.get(0), args.get(1)); + return new MvAppend(source, args.get(0), args.subList(1, args.size())); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendSerializationTests.java index 9bbb4856b5e0f..21943888c21fb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendSerializationTests.java @@ -12,27 +12,41 @@ import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; public class MvAppendSerializationTests extends AbstractExpressionSerializationTests { + + private static final int randomFieldNumber = randomIntBetween(2, 10); + @Override protected MvAppend createTestInstance() { Source source = randomSource(); Expression field1 = randomChild(); - Expression field2 = randomChild(); - return new MvAppend(source, field1, field2); + + List filedList = new ArrayList<>(); + for (int i = 0; i < randomFieldNumber; i++) { + filedList.add(randomChild()); + } + return new MvAppend(source, field1, filedList); } @Override protected MvAppend mutateInstance(MvAppend instance) throws IOException { Source source = randomSource(); Expression field1 = randomChild(); - Expression field2 = randomChild(); + List filedList = new ArrayList<>(); + for (int i = 0; i < randomFieldNumber; i++) { + filedList.add(randomChild()); + } if (randomBoolean()) { field1 = randomValueOtherThan(field1, AbstractExpressionSerializationTests::randomChild); } else { - field2 = randomValueOtherThan(field2, AbstractExpressionSerializationTests::randomChild); + for (int i = 0; i < randomFieldNumber; i++) { + filedList.set(i, randomValueOtherThan(filedList.get(i), AbstractExpressionSerializationTests::randomChild)); + } } - return new MvAppend(source, field1, field2); + return new MvAppend(source, field1, filedList); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java index ca0b997fe0a4f..b5cf54f4a371d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java @@ -49,7 +49,7 @@ public static Iterable parameters() { @Override protected Expression build(Source source, List args) { - return new MvAppend(source, args.get(0), args.get(1)); + return new MvAppend(source, args.get(0), args.subList(1, args.size())); } private static void booleans(List suppliers) { @@ -63,7 +63,7 @@ private static void booleans(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.BOOLEAN, "field1"), new TestCaseSupplier.TypedData(field2, DataType.BOOLEAN, "field2") ), - "MvAppendBooleanEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvAppendBooleanEvaluator[field1=Attribute[channel=0], field2=[Attribute[channel=1]]]", DataType.BOOLEAN, equalTo(result) ); @@ -81,7 +81,7 @@ private static void ints(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.INTEGER, "field1"), new TestCaseSupplier.TypedData(field2, DataType.INTEGER, "field2") ), - "MvAppendIntEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvAppendIntEvaluator[field1=Attribute[channel=0], field2=[Attribute[channel=1]]]", DataType.INTEGER, equalTo(result) ); @@ -99,7 +99,7 @@ private static void longs(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.LONG, "field1"), new TestCaseSupplier.TypedData(field2, DataType.LONG, "field2") ), - "MvAppendLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvAppendLongEvaluator[field1=Attribute[channel=0], field2=[Attribute[channel=1]]]", DataType.LONG, equalTo(result) ); @@ -128,7 +128,7 @@ private static void longs(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.DATETIME, "field1"), new TestCaseSupplier.TypedData(field2, DataType.DATETIME, "field2") ), - "MvAppendLongEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvAppendLongEvaluator[field1=Attribute[channel=0], field2=[Attribute[channel=1]]]", DataType.DATETIME, equalTo(result) ); @@ -161,7 +161,7 @@ private static void doubles(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.DOUBLE, "field1"), new TestCaseSupplier.TypedData(field2, DataType.DOUBLE, "field2") ), - "MvAppendDoubleEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvAppendDoubleEvaluator[field1=Attribute[channel=0], field2=[Attribute[channel=1]]]", DataType.DOUBLE, equalTo(result) ); @@ -188,6 +188,54 @@ private static void bytesRefs(List suppliers) { })); } } + suppliers.add(new TestCaseSupplier(List.of(DataType.KEYWORD, DataType.KEYWORD), () -> { + List field1 = randomList(1, 10, () -> randomLiteral(DataType.KEYWORD).value()); + List field2 = randomList(1, 10, () -> randomLiteral(DataType.KEYWORD).value()); + var result = new ArrayList<>(field1); + result.addAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataType.KEYWORD, "field1"), + new TestCaseSupplier.TypedData(field2, DataType.KEYWORD, "field2") + ), + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=[Attribute[channel=1]]]", + DataType.KEYWORD, + equalTo(result) + ); + })); + + suppliers.add(new TestCaseSupplier(List.of(DataType.TEXT, DataType.TEXT), () -> { + List field1 = randomList(1, 10, () -> randomLiteral(DataType.TEXT).value()); + List field2 = randomList(1, 10, () -> randomLiteral(DataType.TEXT).value()); + var result = new ArrayList<>(field1); + result.addAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataType.TEXT, "field1"), + new TestCaseSupplier.TypedData(field2, DataType.TEXT, "field2") + ), + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=[Attribute[channel=1]]]", + DataType.TEXT, + equalTo(result) + ); + })); + + suppliers.add(new TestCaseSupplier(List.of(DataType.SEMANTIC_TEXT, DataType.SEMANTIC_TEXT), () -> { + List field1 = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); + List field2 = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); + var result = new ArrayList<>(field1); + result.addAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataType.SEMANTIC_TEXT, "field1"), + new TestCaseSupplier.TypedData(field2, DataType.SEMANTIC_TEXT, "field2") + ), + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataType.SEMANTIC_TEXT, + equalTo(result) + ); + })); + suppliers.add(new TestCaseSupplier(List.of(DataType.IP, DataType.IP), () -> { List field1 = randomList(1, 10, () -> randomLiteral(DataType.IP).value()); List field2 = randomList(1, 10, () -> randomLiteral(DataType.IP).value()); @@ -198,7 +246,7 @@ private static void bytesRefs(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.IP, "field"), new TestCaseSupplier.TypedData(field2, DataType.IP, "field") ), - "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=[Attribute[channel=1]]]", DataType.IP, equalTo(result) ); @@ -214,7 +262,7 @@ private static void bytesRefs(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.VERSION, "field"), new TestCaseSupplier.TypedData(field2, DataType.VERSION, "field") ), - "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=[Attribute[channel=1]]]", DataType.VERSION, equalTo(result) ); @@ -230,7 +278,7 @@ private static void bytesRefs(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.GEO_POINT, "field1"), new TestCaseSupplier.TypedData(field2, DataType.GEO_POINT, "field2") ), - "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=[Attribute[channel=1]]]", DataType.GEO_POINT, equalTo(result) ); @@ -246,7 +294,7 @@ private static void bytesRefs(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.CARTESIAN_POINT, "field1"), new TestCaseSupplier.TypedData(field2, DataType.CARTESIAN_POINT, "field2") ), - "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=[Attribute[channel=1]]]", DataType.CARTESIAN_POINT, equalTo(result) ); @@ -262,7 +310,7 @@ private static void bytesRefs(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.GEO_SHAPE, "field1"), new TestCaseSupplier.TypedData(field2, DataType.GEO_SHAPE, "field2") ), - "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=[Attribute[channel=1]]]", DataType.GEO_SHAPE, equalTo(result) ); @@ -278,7 +326,7 @@ private static void bytesRefs(List suppliers) { new TestCaseSupplier.TypedData(field1, DataType.CARTESIAN_SHAPE, "field1"), new TestCaseSupplier.TypedData(field2, DataType.CARTESIAN_SHAPE, "field2") ), - "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=[Attribute[channel=1]]]", DataType.CARTESIAN_SHAPE, equalTo(result) );