ESQL: Fix attribute only in full text function not found (#137395)

A verification exception is thrown if there is LOOKUP JOIN with a FULL TEXT FUNCTION on the LOOKUP INDEX. The field from the LOOKUP INDEX is only used in the full text function. There is KEEP that selects some fields, but not the field in question.

The fix involves making sure that all the fields from the JoinOnCondition of the LOOKUP JOIN are collected during resolveFieldNames() to be used for field caps.

Resolves #137396

Author: julian-elastic

docs/changelog/137395.yaml

@@ -0,0 +1,6 @@
+pr: 137395
+summary: Fix attribute only in full text function not found
+area: ES|QL
+type: bug
+issues:
+ - 137396

x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec

@@ -1253,3 +1253,22 @@ FROM airports
 airport_code:keyword | name:text | city:keyword | country:keyword | location:geo_point | city_boundary:geo_shape
 LTN                  | London Luton | Luton | United Kingdom | POINT(-0.376227267397439 51.8802952570969) | POLYGON((-0.5059 51.9006, -0.4225 51.8545, -0.3499 51.8787, -0.3856 51.9157, -0.4191 51.9123, -0.4263 51.9267, -0.4857 51.9227, -0.4823 51.9078, -0.5059 51.9006))
 ;
+
+lookupJoinExpressionOnRightFieldNotAnywhereElse
+required_capability: join_lookup_v12
+required_capability: lookup_join_with_full_text_function_bugfix
+
+FROM multi_column_joinable
+| RENAME id_int AS id_left
+| LOOKUP JOIN multi_column_joinable_lookup ON id_int == id_left AND MATCH(other1, "beta")
+| WHERE other2 IS NOT NULL
+| KEEP id_left, name_str, extra1, other2
+| SORT id_left, name_str, extra1, other2
+;
+
+warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_int == id_left AND MATCH(other1, \"beta\")] failed, treating result as null. Only first 20 failures recorded.
+warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value
+
+id_left:integer | name_str:keyword | extra1:keyword | other2:integer
+1               | Alice            | foo            | 2000
+;

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -1423,6 +1423,10 @@ public enum Cap {
          * to be applied to the lookup index used
          */
         LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION,
+        /**
+         * Bugfix for lookup join with Full Text Function
+         */
+        LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION_BUGFIX,
         /**
          * FORK with remote indices
          */

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/FieldNameUtils.java

@@ -163,6 +163,9 @@ public static PreAnalysisResult resolveFieldNames(LogicalPlan parsed, boolean ha
                 referencesBuilder.get().addAll(enrichRefs);
             } else if (p instanceof LookupJoin join) {
                 joinRefs.addAll(join.config().leftFields());
+                if (join.config().joinOnConditions() != null) {
+                    joinRefs.addAll(join.config().joinOnConditions().references());
+                }
                 if (keepRefs.isEmpty()) {
                     // No KEEP commands after the JOIN, so we need to mark this index for "*" field resolution
                     wildcardJoinIndices.add(((UnresolvedRelation) join.right()).indexPattern().indexPattern());

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java

@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.analysis;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.test.ESTestCase;
@@ -156,6 +157,17 @@ public static LogicalPlan analyze(String query, Analyzer analyzer) {
         return analyzed;
     }
 
+    public static LogicalPlan analyze(String query, TransportVersion transportVersion) {
+        Analyzer baseAnalyzer = expandedDefaultAnalyzer();
+        if (baseAnalyzer.context() instanceof MutableAnalyzerContext mutableContext) {
+            try (var restore = mutableContext.setTemporaryTransportVersionOnOrAfter(transportVersion)) {
+                return analyze(query, baseAnalyzer);
+            }
+        } else {
+            throw new UnsupportedOperationException("Analyzer Context is not mutable");
+        }
+    }
+
     private static final Pattern indexFromPattern = Pattern.compile("(?i)FROM\\s+([\\w-]+)");
 
     private static String indexFromQuery(String query) {

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java

@@ -5561,6 +5561,43 @@ public void testSubqueryWithFullTextFunctionInMainQuery() {
         assertEquals("sample_data", subqueryIndex.indexPattern());
     }
 
+    public void testLookupJoinOnFieldNotAnywhereElse() {
+        assumeTrue(
+            "requires LOOKUP JOIN ON boolean expression capability",
+            EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION_BUGFIX.isEnabled()
+        );
+
+        String query = "FROM test | LOOKUP JOIN languages_lookup "
+            + "ON languages == language_code AND MATCH(language_name, \"English\")"
+            + "| KEEP languages";
+
+        LogicalPlan analyzedPlan = analyze(query, Analyzer.ESQL_LOOKUP_JOIN_FULL_TEXT_FUNCTION);
+
+        Limit limit = as(analyzedPlan, Limit.class);
+        assertThat(limit.limit(), instanceOf(Literal.class));
+        assertEquals(1000, as(limit.limit(), Literal.class).value());
+
+        EsqlProject project = as(limit.child(), EsqlProject.class);
+        assertEquals(1, project.projections().size());
+
+        LookupJoin lookupJoin = as(project.child(), LookupJoin.class);
+
+        // Verify join condition contains MATCH function
+        assertThat(lookupJoin.config().joinOnConditions(), notNullValue());
+        Expression joinCondition = lookupJoin.config().joinOnConditions();
+        // Check that the join condition contains a MATCH function (it should be an AND expression)
+        assertThat(joinCondition.toString(), containsString("MATCH"));
+
+        // Verify left relation is test index
+        EsRelation leftRelation = as(lookupJoin.left(), EsRelation.class);
+        assertEquals("test", leftRelation.indexPattern());
+
+        // Verify right relation is languages_lookup with LOOKUP mode
+        EsRelation rightRelation = as(lookupJoin.right(), EsRelation.class);
+        assertEquals("languages_lookup", rightRelation.indexPattern());
+        assertEquals(IndexMode.LOOKUP, rightRelation.indexMode());
+    }
+
     private void verifyNameAndTypeAndMultiTypeEsField(
         String actualName,
         DataType actualType,