diff --git a/docs/changelog/137395.yaml b/docs/changelog/137395.yaml new file mode 100644 index 0000000000000..028ba04b3e3b2 --- /dev/null +++ b/docs/changelog/137395.yaml @@ -0,0 +1,6 @@ +pr: 137395 +summary: Fix attribute only in full text function not found +area: ES|QL +type: bug +issues: + - 137396 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec index fdac0d79fbb24..24cfaa0018752 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec @@ -1253,3 +1253,22 @@ FROM airports airport_code:keyword | name:text | city:keyword | country:keyword | location:geo_point | city_boundary:geo_shape LTN | London Luton | Luton | United Kingdom | POINT(-0.376227267397439 51.8802952570969) | POLYGON((-0.5059 51.9006, -0.4225 51.8545, -0.3499 51.8787, -0.3856 51.9157, -0.4191 51.9123, -0.4263 51.9267, -0.4857 51.9227, -0.4823 51.9078, -0.5059 51.9006)) ; + +lookupJoinExpressionOnRightFieldNotAnywhereElse +required_capability: join_lookup_v12 +required_capability: lookup_join_with_full_text_function_bugfix + +FROM multi_column_joinable +| RENAME id_int AS id_left +| LOOKUP JOIN multi_column_joinable_lookup ON id_int == id_left AND MATCH(other1, "beta") +| WHERE other2 IS NOT NULL +| KEEP id_left, name_str, extra1, other2 +| SORT id_left, name_str, extra1, other2 +; + +warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_int == id_left AND MATCH(other1, \"beta\")] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + +id_left:integer | name_str:keyword | extra1:keyword | other2:integer +1 | Alice | foo | 2000 +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 36f111b370b36..7a2d5f0e3d01c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1423,6 +1423,10 @@ public enum Cap { * to be applied to the lookup index used */ LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION, + /** + * Bugfix for lookup join with Full Text Function + */ + LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION_BUGFIX, /** * FORK with remote indices */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java index e5587be184e16..f41c879bec6a9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java @@ -163,6 +163,9 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, boolean ha referencesBuilder.get().addAll(enrichRefs); } else if (p instanceof LookupJoin join) { joinRefs.addAll(join.config().leftFields()); + if (join.config().joinOnConditions() != null) { + joinRefs.addAll(join.config().joinOnConditions().references()); + } if (keepRefs.isEmpty()) { // No KEEP commands after the JOIN, so we need to mark this index for "*" field resolution wildcardJoinIndices.add(((UnresolvedRelation) join.right()).indexPattern().indexPattern()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java index 850b886db33f6..ee15ed7362f8d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.analysis; +import org.elasticsearch.TransportVersion; import org.elasticsearch.index.IndexMode; import org.elasticsearch.inference.TaskType; import org.elasticsearch.test.ESTestCase; @@ -156,6 +157,17 @@ public static LogicalPlan analyze(String query, Analyzer analyzer) { return analyzed; } + public static LogicalPlan analyze(String query, TransportVersion transportVersion) { + Analyzer baseAnalyzer = expandedDefaultAnalyzer(); + if (baseAnalyzer.context() instanceof MutableAnalyzerContext mutableContext) { + try (var restore = mutableContext.setTemporaryTransportVersionOnOrAfter(transportVersion)) { + return analyze(query, baseAnalyzer); + } + } else { + throw new UnsupportedOperationException("Analyzer Context is not mutable"); + } + } + private static final Pattern indexFromPattern = Pattern.compile("(?i)FROM\\s+([\\w-]+)"); private static String indexFromQuery(String query) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 75fc686cf5d0d..76eba5fd0eb2e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -5561,6 +5561,43 @@ public void testSubqueryWithFullTextFunctionInMainQuery() { assertEquals("sample_data", subqueryIndex.indexPattern()); } + public void testLookupJoinOnFieldNotAnywhereElse() { + assumeTrue( + "requires LOOKUP JOIN ON boolean expression capability", + EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION_BUGFIX.isEnabled() + ); + + String query = "FROM test | LOOKUP JOIN languages_lookup " + + "ON languages == language_code AND MATCH(language_name, \"English\")" + + "| KEEP languages"; + + LogicalPlan analyzedPlan = analyze(query, Analyzer.ESQL_LOOKUP_JOIN_FULL_TEXT_FUNCTION); + + Limit limit = as(analyzedPlan, Limit.class); + assertThat(limit.limit(), instanceOf(Literal.class)); + assertEquals(1000, as(limit.limit(), Literal.class).value()); + + EsqlProject project = as(limit.child(), EsqlProject.class); + assertEquals(1, project.projections().size()); + + LookupJoin lookupJoin = as(project.child(), LookupJoin.class); + + // Verify join condition contains MATCH function + assertThat(lookupJoin.config().joinOnConditions(), notNullValue()); + Expression joinCondition = lookupJoin.config().joinOnConditions(); + // Check that the join condition contains a MATCH function (it should be an AND expression) + assertThat(joinCondition.toString(), containsString("MATCH")); + + // Verify left relation is test index + EsRelation leftRelation = as(lookupJoin.left(), EsRelation.class); + assertEquals("test", leftRelation.indexPattern()); + + // Verify right relation is languages_lookup with LOOKUP mode + EsRelation rightRelation = as(lookupJoin.right(), EsRelation.class); + assertEquals("languages_lookup", rightRelation.indexPattern()); + assertEquals(IndexMode.LOOKUP, rightRelation.indexMode()); + } + private void verifyNameAndTypeAndMultiTypeEsField( String actualName, DataType actualType,