diff --git a/docs/changelog/136398.yaml b/docs/changelog/136398.yaml new file mode 100644 index 0000000000000..20e8148914724 --- /dev/null +++ b/docs/changelog/136398.yaml @@ -0,0 +1,8 @@ +pr: 136398 +summary: "ESQL: Push down `MvExpand` past `Project`" +area: ES|QL +type: enhancement +issues: + - 136292 + - 136596 + - 119074 diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index 24a9c88a026fe..fafcc260c84be 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -69,7 +69,6 @@ public abstract class GenerativeRestTest extends ESRestTestCase implements Query "Data too large", // Circuit breaker exceptions eg. https://github.com/elastic/elasticsearch/issues/130072 "long overflow", // https://github.com/elastic/elasticsearch/issues/135759 "cannot be cast to class", // https://github.com/elastic/elasticsearch/issues/133992 - "can't find input for", // https://github.com/elastic/elasticsearch/issues/136596 "unexpected byte", // https://github.com/elastic/elasticsearch/issues/136598 "out of bounds for length", // https://github.com/elastic/elasticsearch/issues/136851 "optimized incorrectly due to missing references", // https://github.com/elastic/elasticsearch/issues/138231 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec index c7dbe01ef6f09..9618398697c55 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec @@ -481,3 +481,157 @@ Production | false | true | -1 | 1 Success | false | true | -1 | 1 | null | 1 ; + +testPushDownMvExpandPastProject +// https://github.com/elastic/elasticsearch/issues/136596 +required_capability: push_down_mv_expand_past_project +from partial_mapping_no_source_sample_data,no_mapping_sample_data +| eval spZehshmg = false, `@timestamp` = null, xrRYYdVC = -1193517579093331092 +| rename xrRYYdVC AS `goqRirHJaE`, `@timestamp` AS gilMWmQyQ +| sort goqRirHJaE NULLS LAST +| where true +| limit 14644 +| mv_expand `spZehshmg` +| keep goqRirHJaE +; + +goqRirHJaE:long +-1193517579093331092 +-1193517579093331092 +-1193517579093331092 +-1193517579093331092 +-1193517579093331092 +-1193517579093331092 +-1193517579093331092 +-1193517579093331092 +-1193517579093331092 +-1193517579093331092 +-1193517579093331092 +-1193517579093331092 +-1193517579093331092 +-1193517579093331092 +; + + +testPushDownMvExpandPastProject2 +required_capability: push_down_mv_expand_past_project +from languages +| eval a = [1,2] +| rename a AS b +| sort b +| mv_expand b +| keep b +; + +b:integer +1 +2 +1 +2 +1 +2 +1 +2 +; + + +testPushDownMvExpandPastProject3 +required_capability: push_down_mv_expand_past_project +from languages +| eval a = 2 +| rename a AS b +| mv_expand b +| keep b +; + +b:integer +2 +2 +2 +2 +; + + +testPushDownMvExpandPastProject4 +required_capability: push_down_mv_expand_past_project +row a = [1,2,3] +| eval c = a +| eval a = 12 +| rename c as b +| keep * +| mv_expand b +; + +b:integer | a:integer +1 | 12 +2 | 12 +3 | 12 +; + + +testPushDownMvExpandPastProject5 +required_capability: fork_v9 +required_capability: subquery_in_from_command +required_capability: push_down_mv_expand_past_project +from employees, (from employees | keep salary) +| eval salary = salary::keyword +| keep salary +| mv_expand salary +| sort salary +| limit 6 +; + +salary:keyword +25324 +25324 +25945 +25945 +25976 +25976 +; + + +testPushDownMvExpandPastProject6 +required_capability: fork_v9 +required_capability: subquery_in_from_command +required_capability: push_down_mv_expand_past_project +from employees, (from employees | keep salary) +| eval salary = salary::keyword +| eval tmp = salary +| keep salary, tmp +| mv_expand salary +| sort salary +| limit 6 +; + +salary:keyword | tmp:keyword +25324 | 25324 +25324 | 25324 +25945 | 25945 +25945 | 25945 +25976 | 25976 +25976 | 25976 +; + + +testPushDownMvExpandPastProject7 +required_capability: fork_v9 +required_capability: subquery_in_from_command +required_capability: push_down_mv_expand_past_project +from employees, (from employees | keep salary) +| eval salary = salary::keyword +| eval tmp = salary +| keep salary, tmp +| mv_expand tmp +| sort salary +| limit 6 +; + +salary:keyword | tmp:keyword +25324 | 25324 +25324 | 25324 +25945 | 25945 +25945 | 25945 +25976 | 25976 +25976 | 25976 +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index bf46249fc672a..a59e154735a9c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1718,6 +1718,11 @@ public enum Cap { */ KNN_FUNCTION_OPTIONS_K_VISIT_PERCENTAGE, + /** + * Support for pushing down MV_EXPAND past PROJECT with complex queries. + */ + PUSH_DOWN_MV_EXPAND_PAST_PROJECT, + /** * Enables automatically grouping by all dimension fields in TS mode queries and outputs the _timeseries column * with all the dimensions. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java index a532cebe455e6..52d439c89071e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java @@ -49,6 +49,7 @@ import org.elasticsearch.xpack.esql.optimizer.rules.logical.PushDownFilterAndLimitIntoUnionAll; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PushDownInferencePlan; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PushDownJoinPastProject; +import org.elasticsearch.xpack.esql.optimizer.rules.logical.PushDownMvExpandPastProject; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PushDownRegexExtract; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PushLimitToKnn; import org.elasticsearch.xpack.esql.optimizer.rules.logical.RemoveStatsOverride; @@ -219,6 +220,7 @@ protected static Batch operators() { new PushDownRegexExtract(), new PushDownEnrich(), new PushDownJoinPastProject(), + new PushDownMvExpandPastProject(), new PushDownAndCombineOrderBy(), new PushDownFilterAndLimitIntoUnionAll(), new PruneRedundantOrderBy(), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownMvExpandPastProject.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownMvExpandPastProject.java new file mode 100644 index 0000000000000..540864d794920 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownMvExpandPastProject.java @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules.logical; + +import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; +import org.elasticsearch.xpack.esql.core.expression.Nullability; +import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; +import org.elasticsearch.xpack.esql.plan.logical.Eval; +import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.plan.logical.MvExpand; +import org.elasticsearch.xpack.esql.plan.logical.Project; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +public final class PushDownMvExpandPastProject extends OptimizerRules.OptimizerRule { + @Override + protected LogicalPlan rule(MvExpand mvExpand) { + if (mvExpand.child() instanceof Project pj) { + LogicalPlan finalChild = pj.child(); + NamedExpression finalTarget = mvExpand.target(); + Attribute finalExpanded = mvExpand.expanded(); + + String expandedFieldName = finalExpanded.name(); + List projections = new ArrayList<>(pj.projections()); + Set inputNames = pj.inputSet().stream().map(NamedExpression::name).collect(Collectors.toSet()); + + // Find if the target is aliased in the project and create an alias with temporary names for it. + for (int i = 0; i < projections.size(); i++) { + if (projections.get(i) instanceof Alias alias) { + boolean replaced = false; + /* + * If the expanded field has the same name as a field in the projection's input set, + * and the projection shadows that specific field from the projection input set. + * Pushing down the MvExpand in such cases would cause duplicate output attributes. + * To avoid this case, we create a temporary attribute for the expanded field and + * update the projection to alias this temporary attribute back to the original name. + * This can happen with aliases generated by ResolveUnionTypesInUnionAll. + * + * Example query: + * from employees, (from employees | keep salary) + * | eval salary = salary::keyword + * | keep salary + * | mv_expand salary + * + * From plan: + * MvExpand[language_code{r}#4,language_code{r}#17] + * \_Project[[$$language_code$converted_to$keyword{r$}#20 AS language_code#4]] + * \_UnionAll[[language_code{r}#15, $$language_code$converted_to$keyword{r$}#20, language_name{r}#16]] + * + * To plan: + * Project[[$$language_code$temp_name$21{r$}#22 AS language_code#17]] + * \_MvExpand[$$language_code$converted_to$keyword{r$}#20,$$language_code$temp_name$21{r$}#22] + * \_UnionAll[[language_code{r}#15, $$language_code$converted_to$keyword{r$}#20, language_name{r}#16]] + * + * + * If the original mv_expand target field is referenced elsewhere in the projections, + * a defensive eval will also be injected. + * + * Example query: + * from languages, (from languages | keep language_code) + * | eval language_code = language_code::keyword + * | eval tmp = language_code + * | keep language_code, tmp + * | mv_expand language_code + * + * From plan: + * MvExpand[language_code{r}#4,language_code{r}#22] + * \_Project[[$$language_code$converted_to$keyword{r$}#25 AS language_code#4,$$language_code$converted_to$keyword{r$}#25 + * AS tmp#7]] + * \_UnionAll[[language_code{r}#20, $$language_code$converted_to$keyword{r$}#25, language_name{r}#21]] + * + * To plan: + * Project[[$$language_code$temp_name$26{r$}#27 AS language_code#22, $$language_code$converted_to$keyword{r$}#25 + * AS tmp#7]] + * \_MvExpand[$$language_code$converted_to$keyword$language_code$0{r}#28,$$language_code$temp_name$26{r$}#27] + * \_Eval[[$$language_code$converted_to$keyword{r$}#25 AS $$language_code$converted_to$keyword$language_code$0#28]] + * \_UnionAll[[language_code{r}#20, $$language_code$converted_to$keyword{r$}#25, language_name{r}#21]] + */ + if (alias.toAttribute().semanticEquals(finalTarget.toAttribute())) { + if (inputNames.contains(expandedFieldName) && inputNames.contains(alias.toAttribute().name())) { + ReferenceAttribute tempAttribute = new ReferenceAttribute( + alias.source(), + null, + TemporaryNameUtils.locallyUniqueTemporaryName(alias.name()), + alias.dataType(), + Nullability.FALSE, + null, + true + ); + projections.set(i, new Alias(alias.source(), expandedFieldName, tempAttribute, finalExpanded.id())); + finalExpanded = tempAttribute; + replaced = true; + } + + // Check if the alias's original field (child) is referenced elsewhere in the projections. + // If the original field is not referenced by any other projection or alias, + // we don't need to inject an Eval to preserve it, and can safely resolve renames and push down. + if (projections.stream() + .anyMatch( + ne -> ne.semanticEquals(alias.child()) + || ne instanceof Alias as && as.child().semanticEquals(alias.child()) && as != alias + ) == false) { + // The alias's original field is not referenced elsewhere, no need to preserve it, + finalTarget = (NamedExpression) alias.child(); + break; + } + + // for query like: row a = 2 | eval b = a | keep * | mv_expand b + Alias aliasAlias = new Alias( + alias.source(), + TemporaryNameUtils.temporaryName(alias.child(), alias.toAttribute(), 0), + alias.child() + ); + if (replaced == false) { + projections.set(i, finalExpanded); + } + finalChild = new Eval(aliasAlias.source(), finalChild, List.of(aliasAlias)); + finalTarget = aliasAlias.toAttribute(); + break; + } else if (alias.child().semanticEquals(finalTarget.toAttribute())) { + // for query like: row a = 2 | eval b = a | keep * | mv_expand a + Alias aliasAlias = new Alias( + alias.source(), + TemporaryNameUtils.temporaryName(alias.child(), alias.toAttribute(), 0), + alias.child() + ); + projections.set(i, alias.replaceChild(aliasAlias.toAttribute())); + finalChild = new Eval(aliasAlias.source(), finalChild, List.of(aliasAlias)); + break; + } + } + } + + // Push down the MvExpand past the Project + MvExpand pushedDownMvExpand = new MvExpand(mvExpand.source(), finalChild, finalTarget, finalExpanded); + + // Update projections to point to the expanded attribute + Attribute target = finalTarget.toAttribute(); + for (int i = 0; i < projections.size(); i++) { + NamedExpression ne = projections.get(i); + if (ne instanceof Alias alias && alias.child().semanticEquals(target)) { + projections.set(i, alias.replaceChild(finalExpanded)); + } else if (ne.semanticEquals(target)) { + projections.set(i, finalExpanded); + } + } + + return new Project(pj.source(), pushedDownMvExpand, projections); + } + return mvExpand; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownUtils.java index 33ec4fcb21561..e8a1fce280ba2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownUtils.java @@ -195,7 +195,7 @@ public static Project pushDownPastProject(UnaryPlan parent) { } } - private static

P resolveRenamesFromProject(P plan, Project project) { + public static

P resolveRenamesFromProject(P plan, Project project) { AttributeMap.Builder aliasBuilder = AttributeMap.builder(); project.forEachExpression(Alias.class, a -> aliasBuilder.put(a.toAttribute(), a.child())); var aliases = aliasBuilder.build(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 1e9ec372520ca..c00e93ce7bc10 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -134,9 +134,11 @@ import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.Row; import org.elasticsearch.xpack.esql.plan.logical.Sample; +import org.elasticsearch.xpack.esql.plan.logical.Subquery; import org.elasticsearch.xpack.esql.plan.logical.TimeSeriesAggregate; import org.elasticsearch.xpack.esql.plan.logical.TopN; import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; +import org.elasticsearch.xpack.esql.plan.logical.UnionAll; import org.elasticsearch.xpack.esql.plan.logical.inference.Completion; import org.elasticsearch.xpack.esql.plan.logical.inference.Rerank; import org.elasticsearch.xpack.esql.plan.logical.join.InlineJoin; @@ -1604,11 +1606,11 @@ public void testDontCombineOrderByThroughMvExpand() { /** * Expected * 

{@code
-     * Limit[1000[INTEGER],true]
-     * \_MvExpand[x{r}#4,x{r}#19]
-     *   \_EsqlProject[[first_name{f}#9 AS x]]
-     *     \_Limit[1000[INTEGER],false]
-     *       \_EsRelation[test][_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, ge..]
+     * Project[[x{r}#20 AS x#5]]
+     * \_Limit[1000[INTEGER],true,false]
+     *   \_MvExpand[first_name{f}#10,x{r}#20]
+     *     \_Limit[1000[INTEGER],false,false]
+     *       \_EsRelation[test][_meta_field{f}#15, emp_no{f}#9, first_name{f}#10, g..]
      * }
*/ public void testCopyDefaultLimitPastMvExpand() { @@ -1618,11 +1620,10 @@ public void testCopyDefaultLimitPastMvExpand() { | keep x | mv_expand x """); - - var limit = asLimit(plan, 1000, true); + var project = as(plan, Project.class); + var limit = asLimit(project.child(), 1000, true); var mvExpand = as(limit.child(), MvExpand.class); - var keep = as(mvExpand.child(), EsqlProject.class); - var limitPastMvExpand = asLimit(keep.child(), 1000, false); + var limitPastMvExpand = asLimit(mvExpand.child(), 1000, false); as(limitPastMvExpand.child(), EsRelation.class); } @@ -1655,11 +1656,11 @@ public void testCopyDefaultLimitPastLookupJoin() { /** * Expected * 
{@code
-     * Limit[10[INTEGER],true]
-     * \_MvExpand[first_name{f}#7,first_name{r}#17]
-     *   \_EsqlProject[[first_name{f}#7, last_name{f}#10]]
-     *     \_Limit[1[INTEGER],false]
-     *       \_EsRelation[test][_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, ge..]
+     * Project[[first_name{r}#18, last_name{f}#11]]
+     * \_Limit[10[INTEGER],true,false]
+     *   \_MvExpand[first_name{f}#8,first_name{r}#18]
+     *     \_Limit[1[INTEGER],false,false]
+     *       \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..]
      * }
*/ public void testDontPushDownLimitPastMvExpand() { @@ -1670,11 +1671,10 @@ public void testDontPushDownLimitPastMvExpand() { | mv_expand first_name | limit 10 """); - - var limit = asLimit(plan, 10, true); + var project = as(plan, Project.class); + var limit = asLimit(project.child(), 10, true); var mvExpand = as(limit.child(), MvExpand.class); - var project = as(mvExpand.child(), EsqlProject.class); - var limit2 = asLimit(project.child(), 1, false); + var limit2 = asLimit(mvExpand.child(), 1, false); as(limit2.child(), EsRelation.class); } @@ -2124,16 +2124,16 @@ public void testMultiMvExpand_SortDownBelow() { * Expected * * 
{@code
-     * Limit[10000[INTEGER],true]
-     * \_MvExpand[c{r}#7,c{r}#16]
-     *   \_EsqlProject[[c{r}#7, a{r}#3]]
-     *     \_TopN[[Order[a{r}#3,ASC,FIRST]],7300[INTEGER]]
-     *       \_Limit[7300[INTEGER],true]
-     *         \_MvExpand[b{r}#5,b{r}#15]
-     *           \_Limit[7300[INTEGER],false]
-     *             \_LocalRelation[[a{r}#3, b{r}#5, c{r}#7],[ConstantNullBlock[positions=1],
-     *               IntVectorBlock[vector=ConstantIntVector[positions=1, value=123]],
-     *               IntVectorBlock[vector=ConstantIntVector[positions=1, value=234]]]]
+     * EsqlProject[[c{r}#17, a{r}#4]]
+     * \_Limit[10000[INTEGER],true,false]
+     *   \_MvExpand[c{r}#8,c{r}#17]
+     *     \_TopN[[Order[a{r}#4,ASC,FIRST]],7300[INTEGER],false]
+     *       \_Limit[7300[INTEGER],true,false]
+     *         \_MvExpand[b{r}#6,b{r}#16]
+     *           \_Limit[7300[INTEGER],false,false]
+     *             \_LocalRelation[[a{r}#4, b{r}#6, c{r}#8],Page{blocks=[ConstantNullBlock[positions=1],
+     *             IntVectorBlock[vector=ConstantIntVector[positions=1, value=123]],
+     *             IntVectorBlock[vector=ConstantIntVector[positions=1, value=234]]]}]
      * }
*/ public void testLimitThenSortBeforeMvExpand() { @@ -2145,10 +2145,10 @@ public void testLimitThenSortBeforeMvExpand() { | sort a NULLS FIRST | mv_expand c"""); - var limit10kBefore = asLimit(plan, 10000, true); + var project = as(plan, Project.class); + var limit10kBefore = asLimit(project.child(), 10000, true); var mvExpand = as(limit10kBefore.child(), MvExpand.class); - var project = as(mvExpand.child(), EsqlProject.class); - var topN = as(project.child(), TopN.class); + var topN = as(mvExpand.child(), TopN.class); assertThat(topN.limit().fold(FoldContext.small()), equalTo(7300)); assertThat(orderNames(topN), contains("a")); var limit7300Before = asLimit(topN.child(), 7300, true); @@ -3106,6 +3106,277 @@ public void testPushDownEnrichPastProject() { as(keep.child(), Enrich.class); } + /** + * 
{@code
+     * EsqlProject[[_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, gender{f}#8, hire_date{f}#13, job{f}#14, job.raw{f}#15, lang
+     * uages{f}#9, last_name{f}#10, long_noidx{f}#16, salary{r}#17]]
+     * \_Limit[1000[INTEGER],false]
+     *   \_Filter[salary{r}#17 > 20[INTEGER]]
+     *     \_MvExpand[salary{f}#11,salary{r}#17]
+     *       \_EsRelation[test][_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, ge..]
+     * }
+ */ + public void testPushDownMvExpandPastProject() { + LogicalPlan plan = optimizedPlan(""" + from test + | keep * + | mv_expand salary + | where salary > 20 + """); + + var keep = as(plan, Project.class); + var limit = asLimit(keep.child(), 1000, false); + var filter = as(limit.child(), Filter.class); + var mvExpand = as(filter.child(), MvExpand.class); + as(mvExpand.child(), EsRelation.class); + } + + /** + * 
{@code
+     * Project[[a{r}#13, b{r}#14 AS b#8]]
+     * \_Limit[1000[INTEGER],true,false]
+     *   \_MvExpand[$$a$b$0{r}#15,b{r}#14]
+     *     \_Limit[1000[INTEGER],true,false]
+     *       \_MvExpand[a{r}#6,a{r}#13]
+     *         \_Eval[[a{r}#6 AS $$a$b$0#15]]
+     *           \_Limit[1000[INTEGER],false,false]
+     *             \_Aggregate[[],[COUNT(*[KEYWORD],true[BOOLEAN],PT0S[TIME_DURATION]) AS a#6]]
+     *               \_LocalRelation[[a{r}#4],Page{blocks=[IntVectorBlock[vector=ConstantIntVector[positions=1, value=1]]]}]
+     * }
+ */ + public void testPushDownMvExpandPastProject2() { + LogicalPlan plan = optimizedPlan(""" + row a = 1 + | stats a = count(*), b = count(*) + | mv_expand a + | mv_expand b + """); + var project = as(plan, Project.class); + var limit = asLimit(project.child(), 1000, true); + var mvExpand = as(limit.child(), MvExpand.class); + limit = asLimit(mvExpand.child(), 1000, true); + mvExpand = as(limit.child(), MvExpand.class); + var eval = as(mvExpand.child(), Eval.class); + limit = asLimit(eval.child(), 1000, false); + var agg = as(limit.child(), Aggregate.class); + as(agg.child(), LocalRelation.class); + } + + /** + * 
{@code
+     * Project[[b{r}#22]]
+     * \_Limit[1000[INTEGER],true,false]
+     *   \_MvExpand[$$a$b$0{r}#23,b{r}#22]
+     *     \_Eval[[2[INTEGER] AS $$a$b$0#23]]
+     *       \_Limit[1000[INTEGER],false,false]
+     *         \_EsRelation[test][_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, ..]
+     * }
+ */ + public void testPushDownMvExpandPastProject3() { + LogicalPlan plan = optimizedPlan(""" + from test + | eval a = 2 + | rename a AS b + | mv_expand b + | keep b + """); + var project = as(plan, Project.class); + var limit = asLimit(project.child(), 1000, true); + var mvExpand = as(limit.child(), MvExpand.class); + var eval = as(mvExpand.child(), Eval.class); + limit = asLimit(eval.child(), 1000, false); + as(limit.child(), EsRelation.class); + } + + /** + * 
{@code
+     * Project[[b{r}#16 AS b#12, $$a$temp_name$17{r}#18 AS a#9]]
+     * \_Limit[1000[INTEGER],true,false]
+     *   \_MvExpand[a{r}#4,b{r}#16]
+     *     \_Eval[[12[INTEGER] AS $$a$temp_name$17#18]]
+     *       \_Limit[1000[INTEGER],false,false]
+     *         \_LocalRelation[[a{r}#4],Page{blocks=[IntVectorBlock[vector=ConstantIntVector[positions=1, value=2]]]}]
+     * }
+ */ + public void testPushDownMvExpandPastProject4() { + LogicalPlan plan = optimizedPlan(""" + row a = 2 + | eval c = a + | eval a = 12 + | rename c as b + | keep * + | mv_expand b + """); + var project = as(plan, Project.class); + var limit = asLimit(project.child(), 1000, true); + var mvExpand = as(limit.child(), MvExpand.class); + var eval = as(mvExpand.child(), Eval.class); + limit = asLimit(eval.child(), 1000, false); + as(limit.child(), LocalRelation.class); + } + + /** + * 
{@code
+     * Project[[$$salary$temp_name$57{r$}#58 AS salary#53]]
+     * \_Limit[1000[INTEGER],true,false]
+     *   \_MvExpand[$$salary$converted_to$keyword{r$}#56,$$salary$temp_name$57{r$}#58]
+     *     \_Limit[1000[INTEGER],false,false]
+     *       \_UnionAll[[_meta_field{r}#42, emp_no{r}#43, first_name{r}#44, gender{r}#45, hire_date{r}#46, job{r}#47, job.raw{r}#48, l
+     * anguages{r}#49, last_name{r}#50, long_noidx{r}#51, salary{r}#52, $$salary$converted_to$keyword{r$}#56]]
+     *         |_EsqlProject[[_meta_field{f}#16, emp_no{f}#10, first_name{f}#11, gender{f}#12, hire_date{f}#17, job{f}#18, job.raw{f}#19, l
+     * anguages{f}#13, last_name{f}#14, long_noidx{f}#20, salary{f}#15, $$salary$converted_to$keyword{r}#54]]
+     *         | \_Eval[[TOSTRING(salary{f}#15) AS $$salary$converted_to$keyword#54]]
+     *         |   \_Limit[1000[INTEGER],false,false]
+     *         |     \_EsRelation[employees][_meta_field{f}#16, emp_no{f}#10, first_name{f}#11, ..]
+     *         \_EsqlProject[[_meta_field{r}#32, emp_no{r}#33, first_name{r}#34, gender{r}#35, hire_date{r}#36, job{r}#37, job.raw{r}#38, l
+     * anguages{r}#39, last_name{r}#40, long_noidx{r}#41, salary{f}#26, $$salary$converted_to$keyword{r}#55]]
+     *           \_Eval[[null[KEYWORD] AS _meta_field#32, null[INTEGER] AS emp_no#33, null[KEYWORD] AS first_name#34, null[TEXT] AS ge
+     * nder#35, null[DATETIME] AS hire_date#36, null[TEXT] AS job#37, null[KEYWORD] AS job.raw#38, null[INTEGER] AS languages#39,
+     * null[KEYWORD] AS last_name#40, null[LONG] AS long_noidx#41, TOSTRING(salary{f}#26) AS $$salary$converted_to$keyword#55]]
+     *             \_Subquery[]
+     *               \_EsqlProject[[salary{f}#26]]
+     *                 \_Limit[1000[INTEGER],false,false]
+     *                   \_EsRelation[employees][_meta_field{f}#27, emp_no{f}#21, first_name{f}#22, ..]
+     * }
+ */ + public void testPushDownMvExpandPastProject5() { + assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled()); + LogicalPlan plan = optimizedPlan(""" + from employees, (from employees | keep salary) + | eval salary = salary::keyword + | keep salary + | mv_expand salary + """); + var project = as(plan, Project.class); + var limit = asLimit(project.child(), 1000, true); + var mvExpand = as(limit.child(), MvExpand.class); + limit = asLimit(mvExpand.child(), 1000, false); + var union = as(limit.child(), UnionAll.class); + assertThat(union.children(), hasSize(2)); + + var branch1 = as(union.children().getFirst(), EsqlProject.class); + var eval1 = as(branch1.child(), Eval.class); + var limit1 = asLimit(eval1.child(), 1000, false); + as(limit1.child(), EsRelation.class); + + var branch2 = as(union.children().get(1), EsqlProject.class); + var eval2 = as(branch2.child(), Eval.class); + var subquery = as(eval2.child(), Subquery.class); + var subProject = as(subquery.child(), EsqlProject.class); + var subLimit = asLimit(subProject.child(), 1000, false); + as(subLimit.child(), EsRelation.class); + } + + /** + * 
{@code
+     * Project[[$$salary$temp_name$61{r$}#62 AS salary#57, $$salary$converted_to$keyword{r$}#60 AS tmp#9]]
+     * \_Limit[1000[INTEGER],true,false]
+     *   \_MvExpand[$$salary$converted_to$keyword$salary$0{r}#63,$$salary$temp_name$61{r$}#62]
+     *     \_Eval[[$$salary$converted_to$keyword{r$}#60 AS $$salary$converted_to$keyword$salary$0#63]]
+     *       \_Limit[1000[INTEGER],false,false]
+     *         \_UnionAll[[_meta_field{r}#46, emp_no{r}#47, first_name{r}#48, gender{r}#49, hire_date{r}#50, job{r}#51, job.raw{r}#52, l
+     * anguages{r}#53, last_name{r}#54, long_noidx{r}#55, salary{r}#56, $$salary$converted_to$keyword{r$}#60]]
+     *           |_EsqlProject[[_meta_field{f}#20, emp_no{f}#14, first_name{f}#15, gender{f}#16, hire_date{f}#21, job{f}#22, job.raw{f}#23,l
+     * anguages{f}#17, last_name{f}#18, long_noidx{f}#24, salary{f}#19, $$salary$converted_to$keyword{r}#58]]
+     *           | \_Eval[[TOSTRING(salary{f}#19) AS $$salary$converted_to$keyword#58]]
+     *           |   \_Limit[1000[INTEGER],false,false]
+     *           |     \_EsRelation[employees][_meta_field{f}#20, emp_no{f}#14, first_name{f}#15, ..]
+     *           \_EsqlProject[[_meta_field{r}#36, emp_no{r}#37, first_name{r}#38, gender{r}#39, hire_date{r}#40, job{r}#41, job.raw{r}#42,l
+     * anguages{r}#43, last_name{r}#44, long_noidx{r}#45, salary{f}#30, $$salary$converted_to$keyword{r}#59]]
+     *             \_Eval[[null[KEYWORD] AS _meta_field#36, null[INTEGER] AS emp_no#37, null[KEYWORD] AS first_name#38, null[TEXT] AS ge
+     * nder#39, null[DATETIME] AS hire_date#40, null[TEXT] AS job#41, null[KEYWORD] AS job.raw#42, null[INTEGER] AS languages#43,
+     * null[KEYWORD] AS last_name#44, null[LONG] AS long_noidx#45, TOSTRING(salary{f}#30) AS $$salary$converted_to$keyword#59]]
+     *               \_Subquery[]
+     *                 \_EsqlProject[[salary{f}#30]]
+     *                   \_Limit[1000[INTEGER],false,false]
+     *                     \_EsRelation[employees][_meta_field{f}#31, emp_no{f}#25, first_name{f}#26, ..]
+     * }
+ */ + public void testPushDownMvExpandPastProject6() { + assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled()); + LogicalPlan plan = optimizedPlan(""" + from employees, (from employees | keep salary) + | eval salary = salary::keyword + | eval tmp = salary + | keep salary, tmp + | mv_expand salary + """); + var project = as(plan, Project.class); + var limit = asLimit(project.child(), 1000, true); + var mvExpand = as(limit.child(), MvExpand.class); + var eval = as(mvExpand.child(), Eval.class); + limit = asLimit(eval.child(), 1000, false); + var union = as(limit.child(), UnionAll.class); + assertThat(union.children(), hasSize(2)); + + var branch1 = as(union.children().getFirst(), EsqlProject.class); + var eval1 = as(branch1.child(), Eval.class); + var limit1 = asLimit(eval1.child(), 1000, false); + as(limit1.child(), EsRelation.class); + + var branch2 = as(union.children().get(1), EsqlProject.class); + var eval2 = as(branch2.child(), Eval.class); + var subquery = as(eval2.child(), Subquery.class); + var subProject = as(subquery.child(), EsqlProject.class); + var subLimit = asLimit(subProject.child(), 1000, false); + as(subLimit.child(), EsRelation.class); + } + + /** + * 
{@code
+     * Project[[$$salary$converted_to$keyword{r$}#60 AS salary#6, tmp{r}#57]]
+     * \_Limit[1000[INTEGER],true,false]
+     *   \_MvExpand[$$salary$converted_to$keyword$tmp$0{r}#61,tmp{r}#57]
+     *     \_Eval[[$$salary$converted_to$keyword{r$}#60 AS $$salary$converted_to$keyword$tmp$0#61]]
+     *       \_Limit[1000[INTEGER],false,false]
+     *         \_UnionAll[[_meta_field{r}#46, emp_no{r}#47, first_name{r}#48, gender{r}#49, hire_date{r}#50, job{r}#51, job.raw{r}#52, l
+     * anguages{r}#53, last_name{r}#54, long_noidx{r}#55, salary{r}#56, $$salary$converted_to$keyword{r$}#60]]
+     *           |_EsqlProject[[_meta_field{f}#20, emp_no{f}#14, first_name{f}#15, gender{f}#16, hire_date{f}#21, job{f}#22, job.raw{f}#23,l
+     * anguages{f}#17, last_name{f}#18, long_noidx{f}#24, salary{f}#19, $$salary$converted_to$keyword{r}#58]]
+     *           | \_Eval[[TOSTRING(salary{f}#19) AS $$salary$converted_to$keyword#58]]
+     *           |   \_Limit[1000[INTEGER],false,false]
+     *           |     \_EsRelation[employees][_meta_field{f}#20, emp_no{f}#14, first_name{f}#15, ..]
+     *           \_EsqlProject[[_meta_field{r}#36, emp_no{r}#37, first_name{r}#38, gender{r}#39, hire_date{r}#40, job{r}#41, job.raw{r}#42,l
+     * anguages{r}#43, last_name{r}#44, long_noidx{r}#45, salary{f}#30, $$salary$converted_to$keyword{r}#59]]
+     *             \_Eval[[null[KEYWORD] AS _meta_field#36, null[INTEGER] AS emp_no#37, null[KEYWORD] AS first_name#38, null[TEXT] AS ge
+     * nder#39, null[DATETIME] AS hire_date#40, null[TEXT] AS job#41, null[KEYWORD] AS job.raw#42, null[INTEGER] AS languages#43,
+     * null[KEYWORD] AS last_name#44, null[LONG] AS long_noidx#45, TOSTRING(salary{f}#30) AS $$salary$converted_to$keyword#59]]
+     *               \_Subquery[]
+     *                 \_EsqlProject[[salary{f}#30]]
+     *                   \_Limit[1000[INTEGER],false,false]
+     *                     \_EsRelation[employees][_meta_field{f}#31, emp_no{f}#25, first_name{f}#26, ..]
+     * }
+ */ + public void testPushDownMvExpandPastProject7() { + assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled()); + LogicalPlan plan = optimizedPlan(""" + from employees, (from employees | keep salary) + | eval salary = salary::keyword + | eval tmp = salary + | keep salary, tmp + | mv_expand tmp + """); + var project = as(plan, Project.class); + var limit = asLimit(project.child(), 1000, true); + var mvExpand = as(limit.child(), MvExpand.class); + var eval = as(mvExpand.child(), Eval.class); + limit = asLimit(eval.child(), 1000, false); + var union = as(limit.child(), UnionAll.class); + assertThat(union.children(), hasSize(2)); + + var branch1 = as(union.children().getFirst(), EsqlProject.class); + var eval1 = as(branch1.child(), Eval.class); + var limit1 = asLimit(eval1.child(), 1000, false); + as(limit1.child(), EsRelation.class); + + var branch2 = as(union.children().get(1), EsqlProject.class); + var eval2 = as(branch2.child(), Eval.class); + var subquery = as(eval2.child(), Subquery.class); + var subProject = as(subquery.child(), EsqlProject.class); + var subLimit = asLimit(subProject.child(), 1000, false); + as(subLimit.child(), EsRelation.class); + } + public void testTopNEnrich() { LogicalPlan plan = optimizedPlan(""" from test @@ -8541,14 +8812,14 @@ public void testRedundantSortOnMvExpandJoinEnrichGrokDissect() { * Expects * * 
{@code
-     * TopN[[Order[emp_no{f}#23,ASC,LAST]],1000[INTEGER]]
-     * \_Filter[emp_no{f}#23 > 1[INTEGER]]
-     *   \_MvExpand[languages{f}#26,languages{r}#36]
-     *     \_Project[[language_name{f}#35, foo{r}#5 AS bar#18, languages{f}#26, emp_no{f}#23]]
-     *       \_Join[LEFT,[languages{f}#26],[languages{f}#26],[language_code{f}#34]]
-     *         |_Eval[[TOSTRING(languages{f}#26) AS foo#5]]
-     *         | \_EsRelation[test][_meta_field{f}#29, emp_no{f}#23, first_name{f}#24, ..]
-     *         \_EsRelation[languages_lookup][LOOKUP][language_code{f}#34, language_name{f}#35]
+     * Project[[language_name{f}#36, foo{r}#6 AS bar#19, languages{r}#37, emp_no{f}#24]]
+     * \_TopN[[Order[emp_no{f}#24,ASC,LAST]],1000[INTEGER],false]
+     *   \_Filter[emp_no{f}#24 > 1[INTEGER]]
+     *     \_MvExpand[languages{f}#27,languages{r}#37]
+     *       \_Join[LEFT,[languages{f}#27],[language_code{f}#35],null]
+     *         |_Eval[[TOSTRING(languages{f}#27) AS foo#6]]
+     *         | \_EsRelation[test][_meta_field{f}#30, emp_no{f}#24, first_name{f}#25, ..]
+     *         \_EsRelation[languages_lookup][LOOKUP][language_code{f}#35, language_name{f}#36]
      * }
*/ public void testRedundantSortOnMvExpandJoinKeepDropRename() { @@ -8565,11 +8836,11 @@ public void testRedundantSortOnMvExpandJoinKeepDropRename() { | SORT emp_no """); - var topN = as(plan, TopN.class); + var project = as(plan, Project.class); + var topN = as(project.child(), TopN.class); var filter = as(topN.child(), Filter.class); var mvExpand = as(filter.child(), MvExpand.class); - var project = as(mvExpand.child(), Project.class); - var join = as(project.child(), Join.class); + var join = as(mvExpand.child(), Join.class); var eval = as(join.left(), Eval.class); as(eval.child(), EsRelation.class);