diff --git a/docs/changelog/135446.yaml b/docs/changelog/135446.yaml new file mode 100644 index 0000000000000..9684b1842192b --- /dev/null +++ b/docs/changelog/135446.yaml @@ -0,0 +1,5 @@ +pr: 135446 +summary: Fix projection generation when pruning left join +area: ES|QL +type: bug +issues: [] diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeMap.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeMap.java index 473882d90dad7..811a61821ea82 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeMap.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeMap.java @@ -393,6 +393,12 @@ public static AttributeMap of(Attribute key, E value) { return map; } + public static AttributeMap mapAll(Collection collection, Function keyMapper) { + final AttributeMap map = new AttributeMap<>(); + collection.forEach(e -> map.add(keyMapper.apply(e), e)); + return map; + } + public static Builder builder() { return new Builder<>(new AttributeMap<>()); } diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java index b22121cd8246d..b7585a54495cb 100644 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/AttributeMapTests.java @@ -344,4 +344,18 @@ public void testValuesIteratorRemoval() { it.remove(); assertThat(it.hasNext(), is(false)); } + + public void testMappAll() { + var one = a("one"); + var two = a("two"); + var three = a("three"); + + Collection collection = asList(one, two, three); + var map = AttributeMap.mapAll(collection, NamedExpression::toAttribute); + + var builder = AttributeMap.builder(); + collection.forEach(e -> builder.put(e, e.toAttribute())); + + assertThat(map, is(builder.build())); + } } diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java index 717d0d563658c..b098c4be0cdf3 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java @@ -89,15 +89,16 @@ public abstract class RestEsqlTestCase extends ESRestTestCase { private static final Logger LOGGER = LogManager.getLogger(RestEsqlTestCase.class); + private static final String MAPPING_FIELD; private static final String MAPPING_ALL_TYPES; - private static final String MAPPING_ALL_TYPES_LOOKUP; static { String properties = EsqlTestUtils.loadUtf8TextFile("/mapping-all-types.json"); - MAPPING_ALL_TYPES = "{\"mappings\": " + properties + "}"; - String settings = "{\"settings\" : {\"mode\" : \"lookup\"}"; - MAPPING_ALL_TYPES_LOOKUP = settings + ", " + "\"mappings\": " + properties + "}"; + MAPPING_FIELD = "\"mappings\": " + properties; + MAPPING_ALL_TYPES = "{" + MAPPING_FIELD + "}"; + String settings = "\"settings\" : {\"mode\" : \"lookup\"}"; + MAPPING_ALL_TYPES_LOOKUP = "{" + settings + ", " + MAPPING_FIELD + "}"; } private static final String DOCUMENT_TEMPLATE = """ @@ -1154,6 +1155,24 @@ public void testMultipleBatchesWithLookupJoin() throws IOException { } } + public void testPruneLeftJoinOnNullMatchingFieldAndShadowingAttributes() throws IOException { + var standardIndexName = "standard"; + createIndex(standardIndexName, false, MAPPING_FIELD); + createIndex(testIndexName(), true); + + var query = format( + null, + "FROM {}* | EVAL keyword = null::KEYWORD | LOOKUP JOIN {} ON keyword | KEEP keyword, integer, alias_integer | SORT keyword", + standardIndexName, + testIndexName() + ); + Map result = runEsql(requestObjectBuilder().query(query)); + var values = as(result.get("values"), List.class); + assertThat(values.size(), is(0)); + + assertThat(deleteIndex(standardIndexName).isAcknowledged(), is(true)); + } + public void testErrorMessageForLiteralDateMathOverflow() throws IOException { List dateMathOverflowExpressions = List.of( "2147483647 day + 1 day", diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 6d4ba53533c09..96f8ffdcccbbe 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -77,8 +77,9 @@ public class CsvTestsDataLoader { ).withSetting("lookup-settings.json"); private static final TestDataset LANGUAGES = new TestDataset("languages"); private static final TestDataset LANGUAGES_LOOKUP = LANGUAGES.withIndex("languages_lookup").withSetting("lookup-settings.json"); - private static final TestDataset LANGUAGES_LOOKUP_NON_UNIQUE_KEY = LANGUAGES_LOOKUP.withIndex("languages_lookup_non_unique_key") - .withData("languages_non_unique_key.csv"); + private static final TestDataset LANGUAGES_NON_UNIQUE_KEY = new TestDataset("languages_non_unique_key"); + private static final TestDataset LANGUAGES_LOOKUP_NON_UNIQUE_KEY = LANGUAGES_NON_UNIQUE_KEY.withIndex("languages_lookup_non_unique_key") + .withSetting("lookup-settings.json"); private static final TestDataset LANGUAGES_NESTED_FIELDS = new TestDataset( "languages_nested_fields", "mapping-languages_nested_fields.json", diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index bb64309140ac0..b05014b4e670f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -260,7 +260,11 @@ public static Range rangeOf(Expression value, Expression lower, boolean includeL } public static EsRelation relation() { - return new EsRelation(EMPTY, new EsIndex(randomAlphaOfLength(8), emptyMap()), IndexMode.STANDARD); + return relation(IndexMode.STANDARD); + } + + public static EsRelation relation(IndexMode mode) { + return new EsRelation(EMPTY, new EsIndex(randomAlphaOfLength(8), emptyMap()), mode); } /** diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec index 4b5f0e2a3739c..e6dbfa1950764 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec @@ -3970,3 +3970,23 @@ FROM k8s-downsampled 2024-05-09T23:30:00.000Z | staging | three | {"min":341.0,"max":592.0,"sum":1956.0,"value_count":5} | 824.0 2024-05-09T23:30:00.000Z | staging | two | {"min":442.0,"max":1011.0,"sum":3850.0,"value_count":7} | 1419.0 ; + +selfShadowing +required_capability: inline_stats +required_capability: fix_join_output_merging + +FROM languages_lookup_non_unique_key +| KEEP country, language_name +| EVAL language_code = null::integer +| INLINE STATS MAX(language_code) BY language_code +| SORT country +| LIMIT 5 +; + +country:text |language_name:keyword |MAX(language_code):integer |language_code:integer +Atlantis |null |null |null +[Austria, Germany]|German |null |null +Canada |English |null |null +Mv-Land |Mv-Lang |null |null +Mv-Land2 |Mv-Lang2 |null |null +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 6c46e558c49c8..7306c740319d1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -136,6 +136,22 @@ language_code:integer | language_name:keyword 6 |null ; +selfShadowing +required_capability: join_lookup_v12 +required_capability: fix_join_output_merging + +FROM languages_lookup_non_unique_key +| EVAL language_code = null::integer +| LOOKUP JOIN languages_lookup_non_unique_key ON language_code +| LIMIT 3 +; + +language_code:integer |country:text |country.keyword:keyword |language_name:keyword +null |null |null |null +null |null |null |null +null |null |null |null +; + nonUniqueLeftKeyOnTheDataNode required_capability: join_lookup_v12 @@ -240,7 +256,7 @@ emp_no:integer | language_code:integer | language_name:keyword | country:text 10091 | 1 | null | United Kingdom 10091 | 1 | English | United States of America 10091 | 1 | English | null - 10092 | 2 | German | [Germany, Austria] + 10092 | 2 | German | [Austria, Germany] 10092 | 2 | German | Switzerland 10092 | 2 | German | null 10093 | 3 | null | null @@ -265,7 +281,7 @@ emp_no:integer | language_code:integer | language_name:keyword | country:text 10001 | 1 | English | null 10001 | 1 | null | United Kingdom 10001 | 1 | English | United States of America -10002 | 2 | German | [Germany, Austria] +10002 | 2 | German | [Austria, Germany] 10002 | 2 | German | Switzerland 10002 | 2 | German | null 10003 | 3 | null | null @@ -308,7 +324,7 @@ ROW language_code = 2 ignoreOrder:true language_code:integer | country:text | language_name:keyword -2 | [Germany, Austria] | German +2 | [Austria, Germany] | German 2 | Switzerland | German 2 | null | German ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-languages_non_unique_key.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-languages_non_unique_key.json new file mode 100644 index 0000000000000..905e1083d8a27 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-languages_non_unique_key.json @@ -0,0 +1,18 @@ +{ + "properties" : { + "language_code" : { + "type" : "integer" + }, + "language_name" : { + "type" : "keyword" + }, + "country": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index bb4645075d85c..5a2c122fbd993 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1263,6 +1263,11 @@ public enum Cap { */ FIX_JOIN_MASKING_REGEX_EXTRACT, + /** + * Allow the merging of the children to use {@code Aliase}s, instead of just {@code ReferenceAttribute}s. + */ + FIX_JOIN_OUTPUT_MERGING, + /** * Avid GROK and DISSECT attributes being removed when resolving fields. * see ES|QL: Grok only supports KEYWORD or TEXT values, found expression [type] type [INTEGER] #127468 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/PruneLeftJoinOnNullMatchingField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/PruneLeftJoinOnNullMatchingField.java index 418a292c3e154..17b88e14a90bb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/PruneLeftJoinOnNullMatchingField.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/PruneLeftJoinOnNullMatchingField.java @@ -10,7 +10,6 @@ import org.elasticsearch.xpack.esql.core.expression.AttributeMap; import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.rules.RuleUtils; import org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules; @@ -59,6 +58,6 @@ private static LogicalPlan replaceJoin(Join join) { } var aliasedNulls = RuleUtils.aliasedNulls(joinRightOutput, a -> true); var eval = new Eval(join.source(), join.left(), aliasedNulls.v1()); - return new Project(join.source(), eval, join.computeOutput(join.left().output(), Expressions.asAttributes(aliasedNulls.v2()))); + return new Project(join.source(), eval, join.computeOutputExpressions(join.left().output(), aliasedNulls.v2())); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Lookup.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Lookup.java index 697eff24006d8..6341c230b2870 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Lookup.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Lookup.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.core.capabilities.Resolvables; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; @@ -139,7 +140,7 @@ public List output() { if (localRelation == null) { throw new IllegalStateException("Cannot determine output of LOOKUP with unresolved table"); } - lazyOutput = Join.computeOutput(child().output(), localRelation.output(), joinConfig()); + lazyOutput = Expressions.asAttributes(Join.computeOutputExpressions(child().output(), localRelation.output(), joinConfig())); } return lazyOutput; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/InlineJoin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/InlineJoin.java index d0bdf627db2c2..3487afe97921b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/InlineJoin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/InlineJoin.java @@ -14,7 +14,9 @@ import org.elasticsearch.compute.data.Page; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.AttributeMap; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.util.Holder; @@ -29,7 +31,7 @@ import java.util.List; import java.util.Set; -import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes; +import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputExpressions; import static org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes.LEFT; /** @@ -217,16 +219,19 @@ public Join replaceChildren(LogicalPlan left, LogicalPlan right) { } @Override - public List computeOutput(List left, List right) { + public List computeOutputExpressions(List left, List right) { JoinType joinType = config().type(); - List output; + List output; if (LEFT.equals(joinType)) { - List leftOutputWithoutKeys = left.stream().filter(attr -> config().leftFields().contains(attr) == false).toList(); - List rightWithAppendedKeys = new ArrayList<>(right); - rightWithAppendedKeys.removeAll(config().rightFields()); - rightWithAppendedKeys.addAll(config().leftFields()); - - output = mergeOutputAttributes(rightWithAppendedKeys, leftOutputWithoutKeys); + List leftOutputWithoutKeys = left.stream() + .filter(ne -> config().leftFields().contains(ne.toAttribute()) == false) + .toList(); + List rightWithAppendedLeftKeys = new ArrayList<>(right); + rightWithAppendedLeftKeys.removeIf(ne -> config().rightFields().contains(ne.toAttribute())); + AttributeMap leftAttrMap = AttributeMap.mapAll(left, NamedExpression::toAttribute); + config().leftFields().forEach(lk -> rightWithAppendedLeftKeys.add(leftAttrMap.getOrDefault(lk, lk))); + + output = mergeOutputExpressions(rightWithAppendedLeftKeys, leftOutputWithoutKeys); } else { throw new IllegalArgumentException(joinType.joinName() + " unsupported"); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java index 572eb63bd5f82..6bece6a612392 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -64,7 +65,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; -import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputAttributes; +import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputExpressions; import static org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes.LEFT; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.commonType; @@ -179,7 +180,7 @@ protected NodeInfo info() { @Override public List output() { if (lazyOutput == null) { - lazyOutput = computeOutput(left().output(), right().output()); + lazyOutput = Expressions.asAttributes(computeOutputExpressions(left().output(), right().output())); } return lazyOutput; } @@ -210,30 +211,34 @@ public List rightOutputFields() { return rightOutputFields; } - public List computeOutput(List left, List right) { - return computeOutput(left, right, config); + public List computeOutputExpressions(List left, List right) { + return computeOutputExpressions(left, right, config); } /** * Combine the two lists of attributes into one. * In case of (name) conflicts, specify which sides wins, that is overrides the other column - the left or the right. */ - public static List computeOutput(List leftOutput, List rightOutput, JoinConfig config) { + public static List computeOutputExpressions( + List leftOutput, + List rightOutput, + JoinConfig config + ) { JoinType joinType = config.type(); - List output; + List output; // TODO: make the other side nullable if (LEFT.equals(joinType)) { if (config.joinOnConditions() == null) { // right side becomes nullable and overrides left except for join keys, which we preserve from the left AttributeSet rightKeys = AttributeSet.of(config.rightFields()); - List rightOutputWithoutMatchFields = rightOutput.stream() - .filter(attr -> rightKeys.contains(attr) == false) + List rightOutputWithoutMatchFields = rightOutput.stream() + .filter(ne -> rightKeys.contains(ne.toAttribute()) == false) .toList(); - output = mergeOutputAttributes(rightOutputWithoutMatchFields, leftOutput); + output = mergeOutputExpressions(rightOutputWithoutMatchFields, leftOutput); } else { // We don't allow any attributes in the joinOnConditions that don't have unique names // so right always overwrites left in case of name clashes - output = mergeOutputAttributes(rightOutput, leftOutput); + output = mergeOutputExpressions(rightOutput, leftOutput); } } else { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java index bc19252d35b0d..f3abfa52e5ddb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java @@ -61,6 +61,9 @@ import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.Row; import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; +import org.elasticsearch.xpack.esql.plan.logical.join.Join; +import org.elasticsearch.xpack.esql.plan.logical.join.JoinConfig; +import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes; import org.elasticsearch.xpack.esql.plan.logical.local.EmptyLocalSupplier; import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject; import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; @@ -96,6 +99,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; +import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules.TransformDirection.DOWN; import static org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules.TransformDirection.UP; import static org.hamcrest.Matchers.contains; @@ -1019,6 +1023,58 @@ public List output() { assertThat(e.getMessage(), containsString("Output has changed from")); } + /** + * Input: + * Project[[key{f}#2, int{f}#3, field1{f}#7, field2{f}#8]] + * \_Join[LEFT,[key{f}#2],[key{f}#6],null] + * |_EsRelation[JLfQlKmn][key{f}#2, int{f}#3, field1{f}#4, field2{f}#5] + * \_EsRelation[HQtEBOWq][LOOKUP][key{f}#6, field1{f}#7, field2{f}#8] + * + * Output: + * Project[[key{r}#2, int{f}#3, field1{r}#7, field1{r}#7 AS field2#8]] + * \_Eval[[null[KEYWORD] AS key#2, null[INTEGER] AS field1#7]] + * \_EsRelation[JLfQlKmn][key{f}#2, int{f}#3, field1{f}#4, field2{f}#5] + */ + public void testPruneLeftJoinOnNullMatchingFieldAndShadowingAttributes() { + var keyLeft = getFieldAttribute("key", KEYWORD); + var intFieldLeft = getFieldAttribute("int"); + var fieldLeft1 = getFieldAttribute("field1"); + var fieldLeft2 = getFieldAttribute("field2"); + var leftRelation = EsqlTestUtils.relation(IndexMode.STANDARD) + .withAttributes(List.of(keyLeft, intFieldLeft, fieldLeft1, fieldLeft2)); + + var keyRight = getFieldAttribute("key", KEYWORD); + var fieldRight1 = getFieldAttribute("field1"); + var fieldRight2 = getFieldAttribute("field2"); + var rightRelation = EsqlTestUtils.relation(IndexMode.LOOKUP).withAttributes(List.of(keyRight, fieldRight1, fieldRight2)); + + JoinConfig joinConfig = new JoinConfig(JoinTypes.LEFT, List.of(keyLeft), List.of(keyRight), null); + var join = new Join(EMPTY, leftRelation, rightRelation, joinConfig); + var project = new Project(EMPTY, join, List.of(keyLeft, intFieldLeft, fieldRight1, fieldRight2)); + + var testStats = statsForMissingField("key"); + var localPlan = localPlan(project, testStats); + + var projectOut = as(localPlan, Project.class); + var projectionsOut = projectOut.projections(); + assertThat(Expressions.names(projectionsOut), contains("key", "int", "field1", "field2")); + assertThat(projectionsOut.get(0).id(), is(keyLeft.id())); + assertThat(projectionsOut.get(1).id(), is(intFieldLeft.id())); + assertThat(projectionsOut.get(2).id(), is(fieldRight1.id())); // id must remain from the RHS. + var aliasField2 = as(projectionsOut.get(3), Alias.class); // the projection must contain an alias ... + assertThat(aliasField2.id(), is(fieldRight2.id())); // ... with the same id as the original field. + + var eval = as(projectOut.child(), Eval.class); + assertThat(Expressions.names(eval.fields()), contains("key", "field1")); + var keyEval = as(Alias.unwrap(eval.fields().get(0)), Literal.class); + assertThat(keyEval.value(), is(nullValue())); + assertThat(keyEval.dataType(), is(KEYWORD)); + var field1Eval = as(Alias.unwrap(eval.fields().get(1)), Literal.class); + assertThat(field1Eval.value(), is(nullValue())); + assertThat(field1Eval.dataType(), is(INTEGER)); + var source = as(eval.child(), EsRelation.class); + } + private IsNotNull isNotNull(Expression field) { return new IsNotNull(EMPTY, field); } @@ -1078,6 +1134,6 @@ protected List filteredWarnings() { } public static EsRelation relation() { - return new EsRelation(EMPTY, new EsIndex(randomAlphaOfLength(8), emptyMap()), randomFrom(IndexMode.values())); + return EsqlTestUtils.relation(randomFrom(IndexMode.values())); } }