From 6cfb55ca54c5887d14ec0b24abeaeefe1727f275 Mon Sep 17 00:00:00 2001 From: Julian Kiryakov Date: Mon, 22 Sep 2025 15:48:43 -0400 Subject: [PATCH 01/13] Refactor to remove UsingJoinType, NaturalJoinType --- .../xpack/esql/analysis/Analyzer.java | 31 ++------ .../esql/plan/logical/join/JoinTypes.java | 78 ------------------- .../esql/plan/logical/join/LookupJoin.java | 4 +- .../xpack/esql/session/FieldNameUtils.java | 5 +- .../esql/parser/StatementParserTests.java | 23 +++--- 5 files changed, 19 insertions(+), 122 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index a5cbd69fe603d..2cae5e395e6fe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -128,7 +128,6 @@ import org.elasticsearch.xpack.esql.plan.logical.join.JoinConfig; import org.elasticsearch.xpack.esql.plan.logical.join.JoinType; import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes; -import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes.UsingJoinType; import org.elasticsearch.xpack.esql.plan.logical.join.LookupJoin; import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject; import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; @@ -775,23 +774,13 @@ private Join resolveLookupJoin(LookupJoin join) { JoinType type = config.type(); // rewrite the join into an equi-join between the field with the same name between left and right - if (type instanceof UsingJoinType using) { - List cols = using.columns(); + if (type == JoinTypes.LEFT) { // the lookup cannot be resolved, bail out - if (Expressions.anyMatch(cols, c -> c instanceof UnresolvedAttribute ua && ua.customMessage())) { + if (Expressions.anyMatch(join.config().leftFields(), c -> c instanceof UnresolvedAttribute ua && ua.customMessage())) { return join; } - - JoinType coreJoin = using.coreJoin(); - // verify the join type - if (coreJoin != JoinTypes.LEFT) { - String name = cols.get(0).name(); - UnresolvedAttribute errorAttribute = new UnresolvedAttribute( - join.source(), - name, - "Only LEFT join is supported with USING" - ); - return join.withConfig(new JoinConfig(type, singletonList(errorAttribute), emptyList(), null)); + if (Expressions.anyMatch(join.config().rightFields(), c -> c instanceof UnresolvedAttribute ua && ua.customMessage())) { + return join; } List leftKeys = new ArrayList<>(); List rightKeys = new ArrayList<>(); @@ -816,22 +805,18 @@ private Join resolveLookupJoin(LookupJoin join) { } } else { // resolve the using columns against the left and the right side then assemble the new join config - leftKeys = resolveUsingColumns(cols, join.left().output(), "left"); - rightKeys = resolveUsingColumns(cols, join.right().output(), "right"); + leftKeys = resolveUsingColumns(join.config().leftFields(), join.left().output(), "left"); + rightKeys = resolveUsingColumns(join.config().rightFields(), join.right().output(), "right"); } - config = new JoinConfig(coreJoin, leftKeys, rightKeys, Predicates.combineAnd(resolvedFilters)); + config = new JoinConfig(type, leftKeys, rightKeys, Predicates.combineAnd(resolvedFilters)); return new LookupJoin(join.source(), join.left(), join.right(), config, join.isRemote()); - } else if (type != JoinTypes.LEFT) { + } else { // everything else is unsupported for now - // LEFT can only happen by being mapped from a USING above. So we need to exclude this as well because this rule can be run - // more than once. UnresolvedAttribute errorAttribute = new UnresolvedAttribute(join.source(), "unsupported", "Unsupported join type"); // add error message return join.withConfig(new JoinConfig(type, singletonList(errorAttribute), emptyList(), null)); } - - return join; } private LogicalPlan resolveFork(Fork fork, AnalyzerContext context) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinTypes.java index 9d3471bc356f7..adaf76dbe3ff8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinTypes.java @@ -9,13 +9,9 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.Maps; -import org.elasticsearch.xpack.esql.core.capabilities.Resolvables; -import org.elasticsearch.xpack.esql.core.expression.Attribute; import java.io.IOException; -import java.util.List; import java.util.Map; -import java.util.Objects; /** * Utility class defining the concrete types of joins supported by ESQL. @@ -69,80 +65,6 @@ public void writeTo(StreamOutput out) throws IOException { } } - /** - * Join type for the USING clause - shorthand for defining an equi-join (equality join meaning the condition checks if columns across - * each side of the join are equal). - * One important difference is that the USING clause returns the join column only once, at the beginning of the result set. - */ - public static class UsingJoinType implements JoinType { - private final List columns; - private final JoinType coreJoin; - - public UsingJoinType(JoinType coreJoin, List columns) { - this.columns = columns; - this.coreJoin = coreJoin; - } - - @Override - public String joinName() { - return coreJoin.joinName() + " USING " + columns.toString(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - throw new IllegalArgumentException("USING join type should not be serialized due to being rewritten"); - } - - public JoinType coreJoin() { - return coreJoin; - } - - public List columns() { - return columns; - } - - @Override - public boolean resolved() { - return Resolvables.resolved(columns); - } - - @Override - public int hashCode() { - return Objects.hash(columns, coreJoin); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - UsingJoinType that = (UsingJoinType) o; - return Objects.equals(columns, that.columns) && coreJoin == that.coreJoin; - } - - @Override - public String toString() { - return joinName(); - } - } - - /** - * Private class so it doesn't get used yet it is defined to showcase why the join type was defined as an interface instead of a simpler - * enum. - */ - private abstract static class NaturalJoinType implements JoinType { - - private final JoinType joinType; - - private NaturalJoinType(JoinType joinType) { - this.joinType = joinType; - } - - @Override - public String joinName() { - return "NATURAL " + joinType.joinName(); - } - } - public static JoinType readFrom(StreamInput in) throws IOException { byte id = in.readByte(); JoinType type = JOIN_TYPES.get(id); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/LookupJoin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/LookupJoin.java index 41d713bf7f613..10bc8103e6458 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/LookupJoin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/LookupJoin.java @@ -18,11 +18,9 @@ import org.elasticsearch.xpack.esql.plan.logical.Limit; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.SurrogateLogicalPlan; -import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes.UsingJoinType; import java.util.List; -import static java.util.Collections.emptyList; import static org.elasticsearch.xpack.esql.common.Failure.fail; import static org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes.LEFT; @@ -40,7 +38,7 @@ public LookupJoin( boolean isRemote, @Nullable Expression joinOnConditions ) { - this(source, left, right, new UsingJoinType(LEFT, joinFields), emptyList(), emptyList(), isRemote, joinOnConditions); + this(source, left, right, LEFT, joinFields, joinFields, isRemote, joinOnConditions); } public LookupJoin( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java index c8f948303e22b..acac67e97947e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java @@ -43,7 +43,6 @@ import org.elasticsearch.xpack.esql.plan.logical.TopN; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.inference.Completion; -import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes; import org.elasticsearch.xpack.esql.plan.logical.join.LookupJoin; import org.elasticsearch.xpack.esql.session.EsqlSession.PreAnalysisResult; @@ -154,9 +153,7 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, EnrichReso enrichRefs.removeIf(attr -> attr instanceof EmptyAttribute); referencesBuilder.get().addAll(enrichRefs); } else if (p instanceof LookupJoin join) { - if (join.config().type() instanceof JoinTypes.UsingJoinType usingJoinType) { - joinRefs.addAll(usingJoinType.columns()); - } + joinRefs.addAll(join.config().leftFields()); if (keepRefs.isEmpty()) { // No KEEP commands after the JOIN, so we need to mark this index for "*" field resolution wildcardJoinIndices.add(((UnresolvedRelation) join.right()).indexPattern().indexPattern()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 95d7940ada6c4..a1f29e254e784 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -72,7 +72,6 @@ import org.elasticsearch.xpack.esql.plan.logical.fuse.Fuse; import org.elasticsearch.xpack.esql.plan.logical.inference.Completion; import org.elasticsearch.xpack.esql.plan.logical.inference.Rerank; -import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes; import org.elasticsearch.xpack.esql.plan.logical.join.LookupJoin; import java.util.ArrayList; @@ -3242,12 +3241,11 @@ public void testValidJoinPatternFieldJoin() { assertThat(as(join.left(), UnresolvedRelation.class).indexPattern().indexPattern(), equalTo(unquoteIndexPattern(basePattern))); assertThat(as(join.right(), UnresolvedRelation.class).indexPattern().indexPattern(), equalTo(unquoteIndexPattern(joinPattern))); - var joinType = as(join.config().type(), JoinTypes.UsingJoinType.class); - assertThat(joinType.columns(), hasSize(numberOfOnFields)); + assertThat(join.config().leftFields(), hasSize(numberOfOnFields)); for (int i = 0; i < numberOfOnFields; i++) { - assertThat(as(joinType.columns().get(i), UnresolvedAttribute.class).name(), equalTo(existingIdentifiers.get(i))); + assertThat(as(join.config().leftFields().get(i), UnresolvedAttribute.class).name(), equalTo(existingIdentifiers.get(i))); } - assertThat(joinType.coreJoin().joinName(), equalTo("LEFT OUTER")); + assertThat(join.config().type().joinName(), equalTo("LEFT OUTER")); } public void testExpressionJoinNonSnapshotBuild() { @@ -4604,9 +4602,8 @@ public void testDoubleParamsForIdentifier() { LookupJoin join = as(limit.child(), LookupJoin.class); UnresolvedRelation ur = as(join.right(), UnresolvedRelation.class); assertEquals(ur.indexPattern().indexPattern(), "idx"); - JoinTypes.UsingJoinType joinType = as(join.config().type(), JoinTypes.UsingJoinType.class); - assertEquals(joinType.coreJoin().joinName(), "LEFT OUTER"); - assertEquals(joinType.columns(), List.of(attribute("f9"))); + assertEquals(join.config().type().joinName(), "LEFT OUTER"); + assertEquals(join.config().leftFields(), List.of(attribute("f9"))); Rename rename = as(join.left(), Rename.class); assertEquals(rename.renamings(), List.of(new Alias(EMPTY, "f.8", attribute("f7*.")))); Grok grok = as(rename.child(), Grok.class); @@ -4714,9 +4711,8 @@ public void testDoubleParamsForIdentifier() { LookupJoin join = as(limit.child(), LookupJoin.class); UnresolvedRelation ur = as(join.right(), UnresolvedRelation.class); assertEquals(ur.indexPattern().indexPattern(), "idx"); - JoinTypes.UsingJoinType joinType = as(join.config().type(), JoinTypes.UsingJoinType.class); - assertEquals(joinType.coreJoin().joinName(), "LEFT OUTER"); - assertEquals(joinType.columns(), List.of(attribute("f13.f14"))); + assertEquals(join.config().type().joinName(), "LEFT OUTER"); + assertEquals(join.config().leftFields(), List.of(attribute("f13.f14"))); Rename rename = as(join.left(), Rename.class); assertEquals(rename.renamings(), List.of(new Alias(EMPTY, "f11*..f.12", attribute("f.9*.f.10.")))); Grok grok = as(rename.child(), Grok.class); @@ -4965,9 +4961,8 @@ public void testMixedSingleDoubleParams() { LookupJoin join = as(plan, LookupJoin.class); UnresolvedRelation ur = as(join.right(), UnresolvedRelation.class); assertEquals(ur.indexPattern().indexPattern(), "idx"); - JoinTypes.UsingJoinType joinType = as(join.config().type(), JoinTypes.UsingJoinType.class); - assertEquals(joinType.coreJoin().joinName(), "LEFT OUTER"); - assertEquals(joinType.columns(), List.of(attribute("f5"))); + assertEquals(join.config().type().joinName(), "LEFT OUTER"); + assertEquals(join.config().leftFields(), List.of(attribute("f5"))); Drop drop = as(join.left(), Drop.class); List removals = drop.removals(); assertEquals(removals.size(), 2); From 15b848c5a696b6fad1cdb594348b8faaa741b8ba Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 22 Sep 2025 19:57:14 +0000 Subject: [PATCH 02/13] [CI] Update transport version definitions --- server/src/main/resources/transport/upper_bounds/9.2.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index bf1a90e5be4e9..6e7d51d3d3020 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -index_request_include_tsid,9167000 +security_stats_endpoint,9168000 From 41c9fe6bd5904dfd00ef3ef3dc0d320773e45013 Mon Sep 17 00:00:00 2001 From: Julian Kiryakov Date: Tue, 23 Sep 2025 09:53:09 -0400 Subject: [PATCH 03/13] Add a test for Lookup Join with Union Types --- .../resources/lookup-join-expression.csv-spec | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec index 33146a8a465b8..19cf6be011303 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec @@ -735,3 +735,26 @@ id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:int 13 | Mia | thud | xi | 14000 14 | Nina | foo2 | omicron | 15000 ; + + +lookupJoinExpressionOnUnionTypes +required_capability: join_lookup_v12 +required_capability: lookup_join_on_boolean_expression +required_capability: metadata_fields_remote_test + +FROM apps, apps_short METADATA _index +| EVAL language_code = id::integer +| KEEP _index, language_code +| WHERE language_code < 3 +| RENAME language_code as language_code_left +| LOOKUP JOIN languages_lookup ON language_code_left == language_code +| KEEP _index, language_code, language_name +| SORT _index ASC, language_code ASC +; + +_index:keyword | language_code:integer | language_name:keyword +remote_cluster:apps | 1 | English +remote_cluster:apps | 2 | French +remote_cluster:apps_short | 1 | English +remote_cluster:apps_short | 2 | French +; From 49327d282c135a3e9a5c616b31eb1d2d6df5228e Mon Sep 17 00:00:00 2001 From: Julian Kiryakov Date: Tue, 23 Sep 2025 10:15:14 -0400 Subject: [PATCH 04/13] Update docs/changelog/135225.yaml --- docs/changelog/135225.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/135225.yaml diff --git a/docs/changelog/135225.yaml b/docs/changelog/135225.yaml new file mode 100644 index 0000000000000..70582907ef8cb --- /dev/null +++ b/docs/changelog/135225.yaml @@ -0,0 +1,5 @@ +pr: 135225 +summary: "Refactor to remove `UsingJoinType,` `NaturalJoinType`" +area: ES|QL +type: tech debt +issues: [] From f5c3f43a3ce5ee0c6d7d72bd7b8e2f29cc550b29 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 23 Sep 2025 14:26:35 +0000 Subject: [PATCH 05/13] [CI] Update transport version definitions --- server/src/main/resources/transport/upper_bounds/9.2.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index 6e7d51d3d3020..b1209b927d8a5 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -security_stats_endpoint,9168000 +inference_api_openai_embeddings_headers,9169000 From 9bb9ccf2905d6f3f9509f008f0171fe1a854bcf9 Mon Sep 17 00:00:00 2001 From: Julian Kiryakov Date: Tue, 23 Sep 2025 10:53:24 -0400 Subject: [PATCH 06/13] Don't throw exceptions in the Analyzer --- .../elasticsearch/xpack/esql/EsqlSecurityIT.java | 14 ++------------ .../xpack/esql/analysis/Analyzer.java | 14 ++++++++++---- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java index d5f9ed74c7a92..f398485768587 100644 --- a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java +++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java @@ -828,12 +828,7 @@ private void testLookupJoinFieldLevelSecurityHelper(boolean useExpressionJoin) t ResponseException error = expectThrows(ResponseException.class, () -> runESQLCommand("fls_user4_1", query)); assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST)); if (useExpressionJoin) { - assertThat( - error.getMessage(), - containsString( - "Join condition must be between one attribute on the left side and one attribute on the right side of the join" - ) - ); + assertThat(error.getMessage(), containsString("Unsupported join filter expression:value_left == value")); } else { assertThat(error.getMessage(), containsString("Unknown column [value] in right side of join")); } @@ -907,12 +902,7 @@ private void testLookupJoinFieldLevelSecurityOnAliasHelper(boolean useExpression ResponseException error = expectThrows(ResponseException.class, () -> runESQLCommand("fls_user4_1_alias", query)); assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST)); if (useExpressionJoin) { - assertThat( - error.getMessage(), - containsString( - "Join condition must be between one attribute on the left side and one attribute on the right side of the join" - ) - ); + assertThat(error.getMessage(), containsString("Unsupported join filter expression:value_left == value")); } else { assertThat(error.getMessage(), containsString("Unknown column [value] in right side of join")); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 2cae5e395e6fe..8190043812133 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -757,9 +757,9 @@ private Expression resolveAndOrientJoinCondition(Expression condition, Attribute if (leftIsFromRight && rightIsFromLeft) { return comp.swapLeftAndRight(); // Swapped orientation } - - // Invalid orientation (e.g., both from left or both from right) - throw new IllegalArgumentException( + return new UnresolvedAttribute( + condition.source(), + "unsupported", "Join condition must be between one attribute on the left side and " + "one attribute on the right side of the join, but found: " + condition.sourceText() @@ -800,7 +800,13 @@ private Join resolveLookupJoin(LookupJoin join) { leftKeys.add(leftAttribute); rightKeys.add(rightAttribute); } else { - throw new IllegalArgumentException("Unsupported join filter expression: " + expression); + UnresolvedAttribute errorAttribute = new UnresolvedAttribute( + expression.source(), + "unsupported", + "Unsupported join filter expression:" + expression.sourceText() + ); + return join.withConfig(new JoinConfig(type, singletonList(errorAttribute), emptyList(), null)); + } } } else { From f662d074f243695246ff6e1faf57099fef210bbb Mon Sep 17 00:00:00 2001 From: Julian Kiryakov Date: Tue, 23 Sep 2025 10:54:33 -0400 Subject: [PATCH 07/13] Roll back test change --- .../resources/lookup-join-expression.csv-spec | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec index 19cf6be011303..33146a8a465b8 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec @@ -735,26 +735,3 @@ id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:int 13 | Mia | thud | xi | 14000 14 | Nina | foo2 | omicron | 15000 ; - - -lookupJoinExpressionOnUnionTypes -required_capability: join_lookup_v12 -required_capability: lookup_join_on_boolean_expression -required_capability: metadata_fields_remote_test - -FROM apps, apps_short METADATA _index -| EVAL language_code = id::integer -| KEEP _index, language_code -| WHERE language_code < 3 -| RENAME language_code as language_code_left -| LOOKUP JOIN languages_lookup ON language_code_left == language_code -| KEEP _index, language_code, language_name -| SORT _index ASC, language_code ASC -; - -_index:keyword | language_code:integer | language_name:keyword -remote_cluster:apps | 1 | English -remote_cluster:apps | 2 | French -remote_cluster:apps_short | 1 | English -remote_cluster:apps_short | 2 | French -; From 34277cc224920399ea66b8c18aa3ea8aa3bb4d0b Mon Sep 17 00:00:00 2001 From: Julian Kiryakov Date: Wed, 24 Sep 2025 10:09:03 -0400 Subject: [PATCH 08/13] Delete changelog --- docs/changelog/135225.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/changelog/135225.yaml diff --git a/docs/changelog/135225.yaml b/docs/changelog/135225.yaml deleted file mode 100644 index 70582907ef8cb..0000000000000 --- a/docs/changelog/135225.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 135225 -summary: "Refactor to remove `UsingJoinType,` `NaturalJoinType`" -area: ES|QL -type: tech debt -issues: [] From 9e1d232db5c46ddaeb0c85d8998bdb97318373ca Mon Sep 17 00:00:00 2001 From: Julian Kiryakov Date: Fri, 26 Sep 2025 16:54:57 -0400 Subject: [PATCH 09/13] Update docs/changelog/135225.yaml --- docs/changelog/135225.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/135225.yaml diff --git a/docs/changelog/135225.yaml b/docs/changelog/135225.yaml new file mode 100644 index 0000000000000..e8eb872078bee --- /dev/null +++ b/docs/changelog/135225.yaml @@ -0,0 +1,5 @@ +pr: 135225 +summary: Expression based Lookup Join follow up and refactor +area: ES|QL +type: tech debt +issues: [] From 495b9a6a1f4b8b3083fd75107a6731b0f40b96ca Mon Sep 17 00:00:00 2001 From: Julian Kiryakov Date: Wed, 1 Oct 2025 08:38:49 -0400 Subject: [PATCH 10/13] Remove change log --- docs/changelog/135225.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/changelog/135225.yaml diff --git a/docs/changelog/135225.yaml b/docs/changelog/135225.yaml deleted file mode 100644 index e8eb872078bee..0000000000000 --- a/docs/changelog/135225.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 135225 -summary: Expression based Lookup Join follow up and refactor -area: ES|QL -type: tech debt -issues: [] From 0619f955111836abdfebee370f76e76e3120df6b Mon Sep 17 00:00:00 2001 From: Julian Kiryakov Date: Wed, 1 Oct 2025 10:25:42 -0400 Subject: [PATCH 11/13] Update docs/changelog/135225.yaml --- docs/changelog/135225.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/135225.yaml diff --git a/docs/changelog/135225.yaml b/docs/changelog/135225.yaml new file mode 100644 index 0000000000000..e8eb872078bee --- /dev/null +++ b/docs/changelog/135225.yaml @@ -0,0 +1,5 @@ +pr: 135225 +summary: Expression based Lookup Join follow up and refactor +area: ES|QL +type: tech debt +issues: [] From d8b4fff2d88543e1e006a1b0d149b8475f33be16 Mon Sep 17 00:00:00 2001 From: Julian Kiryakov Date: Wed, 1 Oct 2025 12:33:30 -0400 Subject: [PATCH 12/13] Delete docs/changelog/135225.yaml --- docs/changelog/135225.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/changelog/135225.yaml diff --git a/docs/changelog/135225.yaml b/docs/changelog/135225.yaml deleted file mode 100644 index e8eb872078bee..0000000000000 --- a/docs/changelog/135225.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 135225 -summary: Expression based Lookup Join follow up and refactor -area: ES|QL -type: tech debt -issues: [] From 5fb05d36d6f6db731c3f1f02af267d174e801c65 Mon Sep 17 00:00:00 2001 From: Julian Kiryakov Date: Wed, 1 Oct 2025 08:38:49 -0400 Subject: [PATCH 13/13] Address code review comments --- docs/changelog/135225.yaml | 5 ----- .../org/elasticsearch/xpack/esql/analysis/Analyzer.java | 8 ++++---- 2 files changed, 4 insertions(+), 9 deletions(-) delete mode 100644 docs/changelog/135225.yaml diff --git a/docs/changelog/135225.yaml b/docs/changelog/135225.yaml deleted file mode 100644 index e8eb872078bee..0000000000000 --- a/docs/changelog/135225.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 135225 -summary: Expression based Lookup Join follow up and refactor -area: ES|QL -type: tech debt -issues: [] diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 798a39b2a9438..6275bc6d37a60 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -777,10 +777,10 @@ private Join resolveLookupJoin(LookupJoin join) { // rewrite the join into an equi-join between the field with the same name between left and right if (type == JoinTypes.LEFT) { // the lookup cannot be resolved, bail out - if (Expressions.anyMatch(join.config().leftFields(), c -> c instanceof UnresolvedAttribute ua && ua.customMessage())) { - return join; - } - if (Expressions.anyMatch(join.config().rightFields(), c -> c instanceof UnresolvedAttribute ua && ua.customMessage())) { + if (Expressions.anyMatch( + join.references().stream().toList(), + c -> c instanceof UnresolvedAttribute ua && ua.customMessage() + )) { return join; } List leftKeys = new ArrayList<>();