Bugfix

Author: julian-elastic

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/analysis/MutableAnalyzerContext.java

@@ -52,7 +52,14 @@ public TransportVersion minimumVersion() {
     public RestoreTransportVersion setTemporaryTransportVersionOnOrAfter(TransportVersion minVersion) {
         TransportVersion oldVersion = this.currentVersion;
         // Set to a random version between minVersion and current
-        this.currentVersion = TransportVersionUtils.randomVersionBetween(ESTestCase.random(), minVersion, TransportVersion.current());
+        // have 50+% probability of being the current version, to improve probability of catching a change
+        // that only affects the version we are trying to check in
+        boolean useCurrent = ESTestCase.randomBoolean();
+        if (useCurrent) {
+            this.currentVersion = TransportVersion.current();
+        } else {
+            this.currentVersion = TransportVersionUtils.randomVersionBetween(ESTestCase.random(), minVersion, TransportVersion.current());
+        }
         return new RestoreTransportVersion(oldVersion);
     }
 

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java

@@ -439,35 +439,15 @@ private PhysicalPlan buildRightPreJoinPlan(List<DataType> keyTypes, Expression f
      * This ensures consistent NameId usage across MatchConfig and join conditions.
      */
     private FieldAttribute getLeftSideAttribute(int index, DataType keyType) {
-        String name = "key" + index;
-        // Handle special case for key3 which can be INTEGER or KEYWORD
-        if (index == 3 && keyType == DataType.INTEGER) {
-            return getAttribute(name, DataType.INTEGER);
-        }
-        // Default to KEYWORD or LONG based on keyType
-        if (keyType == DataType.KEYWORD) {
-            return getAttribute(name, DataType.KEYWORD);
-        } else {
-            return getAttribute(name, DataType.LONG);
-        }
+        return getAttribute("key" + index, keyType);
     }
 
     /**
      * Gets the right-side attribute for a given index and type.
      * This ensures consistent NameId usage across join conditions and rightPreJoinPlan.
      */
     private FieldAttribute getRightSideAttribute(int index, DataType keyType) {
-        String name = "rkey" + index;
-        // Handle special case for rkey3 which can be INTEGER or KEYWORD
-        if (index == 3 && keyType == DataType.INTEGER) {
-            return getAttribute(name, DataType.INTEGER);
-        }
-        // Default to KEYWORD or LONG based on keyType
-        if (keyType == DataType.KEYWORD) {
-            return getAttribute(name, DataType.KEYWORD);
-        } else {
-            return getAttribute(name, DataType.LONG);
-        }
+        return getAttribute("rkey" + index, keyType);
     }
 
     private List<Attribute> buildRightSideAttributes(List<DataType> keyTypes) {
@@ -599,7 +579,7 @@ private void runLookup(List<DataType> keyTypes, PopulateIndices populateIndices,
                 TEST_REQUEST_TIMEOUT
             );
             final String finalNodeWithShard = nodeWithShard;
-            boolean expressionJoin = true;// EsqlCapabilities.Cap.LOOKUP_JOIN_ON_BOOLEAN_EXPRESSION.isEnabled() ? randomBoolean() : false;
+            boolean expressionJoin = EsqlCapabilities.Cap.LOOKUP_JOIN_ON_BOOLEAN_EXPRESSION.isEnabled() ? randomBoolean() : false;
             List<MatchConfig> matchFields = new ArrayList<>();
             List<Expression> joinOnConditions = new ArrayList<>();
             if (expressionJoin) {
@@ -609,9 +589,9 @@ private void runLookup(List<DataType> keyTypes, PopulateIndices populateIndices,
                     FieldAttribute rightAttr = getRightSideAttribute(i, keyTypes.get(i));
                     joinOnConditions.add(new Equals(Source.EMPTY, leftAttr, rightAttr));
                     // randomly decide to apply the filter as additional join on filter instead of pushed down filter
-                    boolean applyAsJoinOnCondition = true;// EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION.isEnabled()
-                    // ? randomBoolean()
-                    // : false;
+                    boolean applyAsJoinOnCondition = EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION.isEnabled()
+                        ? randomBoolean()
+                        : false;
                     if (applyAsJoinOnCondition
                         && pushedDownFilter instanceof FragmentExec fragmentExec
                         && fragmentExec.fragment() instanceof Filter filter) {

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/MatchConfig.java

@@ -81,10 +81,8 @@ public MatchConfig(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION)) {
-            // New format: write NamedExpression directly, same as Enrich and EnrichExec
             out.writeNamedWriteable(fieldName);
         } else {
-            // Old format: write field name as string
             out.writeString(fieldName.name());
         }
         out.writeInt(channel);

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java

@@ -784,11 +784,11 @@ private PhysicalOperation planLookupJoin(LookupJoinExec join, LocalExecutionPlan
 
             // TODO: Using exactAttribute was supposed to handle TEXT fields with KEYWORD subfields - but we don't allow these in lookup
             // indices, so the call to exactAttribute looks redundant now.
-            Attribute fieldName = right.exactAttribute();
+            Attribute matchFieldsAttribute = right.exactAttribute();
 
             // we support 2 types of joins: Field name joins and Expression joins
             // for Field name join, we do not ship any join on expression.
-            // we built the Lucene query on the field name that is passed in the MatchConfig.fieldName
+            // we built the Lucene query on the field name that is passed in the MatchConfig.matchFieldsAttribute
             // so for Field name we need to pass the attribute name from the right side, because that is needed to build the query
             // For expression joins, we pass an expression such as left_id > right_id.
             // So in this case we pass in left_id as the field name, because that is what we are shipping to the lookup node
@@ -798,9 +798,9 @@ private PhysicalOperation planLookupJoin(LookupJoinExec join, LocalExecutionPlan
             // e.g. LOOKUP JOIN ON left_id < right_id_1 and left_id >= right_id_2
             // we want to be able to optimize this in the future and only ship the left_id once
             if (join.isOnJoinExpression()) {
-                fieldName = left;
+                matchFieldsAttribute = left;
             }
-            matchFields.add(new MatchConfig(fieldName, input));
+            matchFields.add(new MatchConfig(matchFieldsAttribute, input));
             matchFieldIds.add(left.id());
         }
         // Extract additional left-side fields referenced in joinOnConditions that aren't already in matchFields

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java

@@ -53,6 +53,7 @@
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.testAnalyzerContext;
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning;
 import static org.elasticsearch.xpack.esql.analysis.Analyzer.ESQL_LOOKUP_JOIN_FULL_TEXT_FUNCTION;
+import static org.elasticsearch.xpack.esql.analysis.Analyzer.ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION;
 import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.TEXT_EMBEDDING_INFERENCE_ID;
 import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.defaultLookupResolution;
 import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.loadMapping;
@@ -2340,7 +2341,10 @@ public void testLookupJoinDataTypeMismatch() {
 
         assertEquals(
             "1:87: JOIN left field [language_code] of type [KEYWORD] is incompatible with right field [language_code] of type [INTEGER]",
-            error("FROM test | EVAL language_code = languages::keyword | LOOKUP JOIN languages_lookup ON language_code")
+            error(
+                "FROM test | EVAL language_code = languages::keyword | LOOKUP JOIN languages_lookup ON language_code",
+                ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION
+            )
         );
     }
 
@@ -2364,66 +2368,59 @@ public void testLookupJoinExpressionAmbiguousRight() {
     public void testLookupJoinExpressionRightNotPushable() {
         assumeTrue(
             "requires LOOKUP JOIN ON boolean expression capability",
-            EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION.isEnabled()
+            EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_GENERAL_EXPRESSION.isEnabled()
         );
         String queryString = """
             from test
             | rename languages as languages_left
             | lookup join languages_lookup ON languages_left == language_code and abs(salary) > 1000
             """;
 
-        assertEquals(
-            "3:71: Unsupported join filter expression:abs(salary) > 1000",
-            error(queryString, ESQL_LOOKUP_JOIN_FULL_TEXT_FUNCTION)
-        );
+        query(queryString, ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION);
     }
 
     public void testLookupJoinExpressionConstant() {
         assumeTrue(
             "requires LOOKUP JOIN ON boolean expression capability",
-            EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION.isEnabled()
+            EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_GENERAL_EXPRESSION.isEnabled()
         );
         String queryString = """
             from test
             | rename languages as languages_left
             | lookup join languages_lookup ON false and languages_left == language_code
             """;
 
-        assertEquals("3:35: Unsupported join filter expression:false", error(queryString, ESQL_LOOKUP_JOIN_FULL_TEXT_FUNCTION));
+        query(queryString, ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION);
     }
 
     public void testLookupJoinExpressionTranslatableButFromLeft() {
         assumeTrue(
             "requires LOOKUP JOIN ON boolean expression capability",
-            EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION.isEnabled()
+            EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_GENERAL_EXPRESSION.isEnabled()
         );
         String queryString = """
             from test
             | rename languages as languages_left
-            | lookup join languages_lookup ON languages_left == language_code and languages_left == "English"
+            | eval languages_left_str = to_string(languages_left)
+            | lookup join languages_lookup ON languages_left == language_code and languages_left_str == language_name
             """;
 
-        assertEquals(
-            "3:71: Unsupported join filter expression:languages_left == \"English\"",
-            error(queryString, ESQL_LOOKUP_JOIN_FULL_TEXT_FUNCTION)
-        );
+        query(queryString, ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION);
     }
 
     public void testLookupJoinExpressionTranslatableButMixedLeftRight() {
         assumeTrue(
             "requires LOOKUP JOIN ON boolean expression capability",
-            EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION.isEnabled()
+            EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_GENERAL_EXPRESSION.isEnabled()
         );
         String queryString = """
             from test
             | rename languages as languages_left
-            | lookup join languages_lookup ON languages_left == language_code and CONCAT(languages_left, language_code) == "English"
+            | eval language_name_left = "English"
+            | lookup join languages_lookup ON languages_left == language_code and CONCAT(language_name_left, language_name) == "English"
             """;
 
-        assertEquals(
-            "3:71: Unsupported join filter expression:CONCAT(languages_left, language_code) == \"English\"",
-            error(queryString, ESQL_LOOKUP_JOIN_FULL_TEXT_FUNCTION)
-        );
+        query(queryString, ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION);
     }
 
     public void testLookupJoinExpressionComplexFormula() {
@@ -2434,13 +2431,11 @@ public void testLookupJoinExpressionComplexFormula() {
         String queryString = """
             from test
             | rename languages as languages_left
-            | lookup join languages_lookup ON languages_left == language_code AND STARTSWITH(languages_left, language_code)
+            | eval languages_left_str = to_string(languages_left)
+            | lookup join languages_lookup ON languages_left == language_code AND starts_with(languages_left_str, language_name)
             """;
 
-        assertEquals(
-            "3:71: Unsupported join filter expression:STARTSWITH(languages_left, language_code)",
-            error(queryString, ESQL_LOOKUP_JOIN_FULL_TEXT_FUNCTION)
-        );
+        query(queryString, ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION);
     }
 
     public void testLookupJoinExpressionAmbiguousLeft() {
@@ -3348,6 +3343,13 @@ private void query(String query, Analyzer analyzer) {
         analyzer.analyze(parser.createStatement(query));
     }
 
+    private void query(String query, TransportVersion transportVersion) {
+        MutableAnalyzerContext mutableContext = (MutableAnalyzerContext) defaultAnalyzer.context();
+        try (var restore = mutableContext.setTemporaryTransportVersionOnOrAfter(transportVersion)) {
+            query(query);
+        }
+    }
+
     private String error(String query) {
         return error(query, defaultAnalyzer);
     }