From 89fc200556f6075c4e61dbe95a98edd5e96bb52d Mon Sep 17 00:00:00 2001 From: Julian Kiryakov Date: Thu, 30 Oct 2025 13:21:33 -0400 Subject: [PATCH 1/3] Fix attribute only in full text function not found --- .../resources/lookup-join-expression.csv-spec | 19 ++++++++++ .../xpack/esql/session/FieldNameUtils.java | 3 ++ .../esql/analysis/AnalyzerTestUtils.java | 12 ++++++ .../xpack/esql/analysis/AnalyzerTests.java | 37 +++++++++++++++++++ 4 files changed, 71 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec index f0f961f7d6c2d..738ef58e8cb6a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec @@ -1251,3 +1251,22 @@ FROM airports airport_code:keyword | name:text | city:keyword | country:keyword | location:geo_point | city_boundary:geo_shape LTN | London Luton | Luton | United Kingdom | POINT(-0.376227267397439 51.8802952570969) | POLYGON((-0.5059 51.9006, -0.4225 51.8545, -0.3499 51.8787, -0.3856 51.9157, -0.4191 51.9123, -0.4263 51.9267, -0.4857 51.9227, -0.4823 51.9078, -0.5059 51.9006)) ; + +lookupJoinExpressionOnRightFieldNotAnywhereElse +required_capability: join_lookup_v12 +required_capability: lookup_join_with_full_text_function + +FROM multi_column_joinable +| RENAME id_int AS id_left +| LOOKUP JOIN multi_column_joinable_lookup ON id_int == id_left AND MATCH(other1, "beta") +| WHERE other2 IS NOT NULL +| KEEP id_left, name_str, extra1, other2 +| SORT id_left, name_str, extra1, other2 +; + +warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_int == id_left AND MATCH(other1, \"beta\")] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + +id_left:integer | name_str:keyword | extra1:keyword | other2:integer +1 | Alice | foo | 2000 +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java index 983da830725a5..37c29ac630f9d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java @@ -162,6 +162,9 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, boolean ha referencesBuilder.get().addAll(enrichRefs); } else if (p instanceof LookupJoin join) { joinRefs.addAll(join.config().leftFields()); + if (join.config().joinOnConditions() != null) { + joinRefs.addAll(join.config().joinOnConditions().references()); + } if (keepRefs.isEmpty()) { // No KEEP commands after the JOIN, so we need to mark this index for "*" field resolution wildcardJoinIndices.add(((UnresolvedRelation) join.right()).indexPattern().indexPattern()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java index 850b886db33f6..ee15ed7362f8d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.analysis; +import org.elasticsearch.TransportVersion; import org.elasticsearch.index.IndexMode; import org.elasticsearch.inference.TaskType; import org.elasticsearch.test.ESTestCase; @@ -156,6 +157,17 @@ public static LogicalPlan analyze(String query, Analyzer analyzer) { return analyzed; } + public static LogicalPlan analyze(String query, TransportVersion transportVersion) { + Analyzer baseAnalyzer = expandedDefaultAnalyzer(); + if (baseAnalyzer.context() instanceof MutableAnalyzerContext mutableContext) { + try (var restore = mutableContext.setTemporaryTransportVersionOnOrAfter(transportVersion)) { + return analyze(query, baseAnalyzer); + } + } else { + throw new UnsupportedOperationException("Analyzer Context is not mutable"); + } + } + private static final Pattern indexFromPattern = Pattern.compile("(?i)FROM\\s+([\\w-]+)"); private static String indexFromQuery(String query) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 8de0f720b4d7d..6346ad71413d0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -5559,6 +5559,43 @@ public void testSubqueryWithFullTextFunctionInMainQuery() { assertEquals("sample_data", subqueryIndex.indexPattern()); } + public void testLookupJoinOnFieldNotAnywhereElse() { + assumeTrue( + "requires LOOKUP JOIN ON boolean expression capability", + EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION.isEnabled() + ); + + String query = "FROM test | LOOKUP JOIN languages_lookup " + + "ON languages == language_code AND MATCH(language_name, \"English\")" + + "| KEEP languages"; + + LogicalPlan analyzedPlan = analyze(query, Analyzer.ESQL_LOOKUP_JOIN_FULL_TEXT_FUNCTION); + + Limit limit = as(analyzedPlan, Limit.class); + assertThat(limit.limit(), instanceOf(Literal.class)); + assertEquals(1000, as(limit.limit(), Literal.class).value()); + + EsqlProject project = as(limit.child(), EsqlProject.class); + assertEquals(1, project.projections().size()); + + LookupJoin lookupJoin = as(project.child(), LookupJoin.class); + + // Verify join condition contains MATCH function + assertThat(lookupJoin.config().joinOnConditions(), notNullValue()); + Expression joinCondition = lookupJoin.config().joinOnConditions(); + // Check that the join condition contains a MATCH function (it should be an AND expression) + assertThat(joinCondition.toString(), containsString("MATCH")); + + // Verify left relation is test index + EsRelation leftRelation = as(lookupJoin.left(), EsRelation.class); + assertEquals("test", leftRelation.indexPattern()); + + // Verify right relation is languages_lookup with LOOKUP mode + EsRelation rightRelation = as(lookupJoin.right(), EsRelation.class); + assertEquals("languages_lookup", rightRelation.indexPattern()); + assertEquals(IndexMode.LOOKUP, rightRelation.indexMode()); + } + private void verifyNameAndTypeAndMultiTypeEsField( String actualName, DataType actualType, From 87338dbc63b6572e9a20b42f98c1a6f52e09d89d Mon Sep 17 00:00:00 2001 From: Julian Kiryakov Date: Thu, 30 Oct 2025 13:29:40 -0400 Subject: [PATCH 2/3] Update docs/changelog/137395.yaml --- docs/changelog/137395.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/137395.yaml diff --git a/docs/changelog/137395.yaml b/docs/changelog/137395.yaml new file mode 100644 index 0000000000000..028ba04b3e3b2 --- /dev/null +++ b/docs/changelog/137395.yaml @@ -0,0 +1,6 @@ +pr: 137395 +summary: Fix attribute only in full text function not found +area: ES|QL +type: bug +issues: + - 137396 From 59253f212e6a5269894acaeaaf4ef8c13bd57d44 Mon Sep 17 00:00:00 2001 From: Julian Kiryakov Date: Wed, 5 Nov 2025 16:07:36 -0500 Subject: [PATCH 3/3] Protect newly added UTs with capability --- .../src/main/resources/lookup-join-expression.csv-spec | 2 +- .../org/elasticsearch/xpack/esql/action/EsqlCapabilities.java | 4 ++++ .../org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec index 5adeff3b141a8..24cfaa0018752 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec @@ -1256,7 +1256,7 @@ LTN | London Luton | Luton | United Kingdom | POINT(-0.37622726 lookupJoinExpressionOnRightFieldNotAnywhereElse required_capability: join_lookup_v12 -required_capability: lookup_join_with_full_text_function +required_capability: lookup_join_with_full_text_function_bugfix FROM multi_column_joinable | RENAME id_int AS id_left diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 36f111b370b36..7a2d5f0e3d01c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1423,6 +1423,10 @@ public enum Cap { * to be applied to the lookup index used */ LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION, + /** + * Bugfix for lookup join with Full Text Function + */ + LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION_BUGFIX, /** * FORK with remote indices */ diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 680e3889638cc..76eba5fd0eb2e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -5564,7 +5564,7 @@ public void testSubqueryWithFullTextFunctionInMainQuery() { public void testLookupJoinOnFieldNotAnywhereElse() { assumeTrue( "requires LOOKUP JOIN ON boolean expression capability", - EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION.isEnabled() + EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION_BUGFIX.isEnabled() ); String query = "FROM test | LOOKUP JOIN languages_lookup "