Bugfix

Author: julian-elastic

test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java

@@ -758,7 +758,7 @@ public void testLookupExplosionExpression() throws IOException {
     public void testLookupExplosionManyFieldsExpression() throws IOException {
         int sensorDataCount = 400;
         int lookupEntries = 1000;
-        int joinFieldsCount = 990;
+        int joinFieldsCount = 399;// only join on 399 columns due to max expression size of 400
         Map<?, ?> map = lookupExplosion(sensorDataCount, lookupEntries, joinFieldsCount, lookupEntries, true);
         assertMap(map, matchesMap().extraOk().entry("values", List.of(List.of(sensorDataCount * lookupEntries))));
     }

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/HashJoinExec.java

@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.plan.physical;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -54,8 +55,9 @@ public HashJoinExec(
 
     private HashJoinExec(StreamInput in) throws IOException {
         super(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(PhysicalPlan.class), in.readNamedWriteable(PhysicalPlan.class));
-        // TODO: clean up, we used to read the match fields here.
-        in.readNamedWriteableCollectionAsList(Attribute.class);
+        if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_LOOKUP_JOIN_ON_EXPRESSION) == false) {
+            in.readNamedWriteableCollectionAsList(Attribute.class);
+        }
         this.leftFields = in.readNamedWriteableCollectionAsList(Attribute.class);
         this.rightFields = in.readNamedWriteableCollectionAsList(Attribute.class);
         this.addedFields = in.readNamedWriteableCollectionAsList(Attribute.class);
@@ -64,8 +66,9 @@ private HashJoinExec(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        // TODO: clean up, we used to read the match fields here.
-        out.writeNamedWriteableCollection(leftFields);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_LOOKUP_JOIN_ON_EXPRESSION) == false) {
+            out.writeNamedWriteableCollection(leftFields);
+        }
         out.writeNamedWriteableCollection(leftFields);
         out.writeNamedWriteableCollection(rightFields);
         out.writeNamedWriteableCollection(addedFields);

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperatorTests.java

@@ -91,6 +91,7 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.LongStream;
@@ -113,7 +114,10 @@ public static Iterable<Object[]> parametersFactory() {
         List<Object[]> operations = new ArrayList<>();
         operations.add(new Object[] { null });
         for (EsqlBinaryComparison.BinaryComparisonOperation operation : EsqlBinaryComparison.BinaryComparisonOperation.values()) {
-            operations.add(new Object[] { operation });
+            // we skip NEQ because there are too many matches and the test can timeout
+            if (operation != EsqlBinaryComparison.BinaryComparisonOperation.NEQ) {
+                operations.add(new Object[] { operation });
+            }
         }
         return operations;
     }
@@ -157,10 +161,6 @@ protected SourceOperator simpleInput(BlockFactory blockFactory, int size) {
 
     @Override
     protected void assertSimpleOutput(List<Page> input, List<Page> results) {
-        if (operation == EsqlBinaryComparison.BinaryComparisonOperation.NEQ) {
-            // if not equal there are too many matches and we get OOMEs when validating the output
-            return;
-        }
         /*
          * We've configured there to be just a single result per input so the total
          * row count is the same. But lookup cuts into pages of length 256 so the
@@ -448,12 +448,12 @@ private AbstractLookupService.LookupShardContextFactory lookupShardContextFactor
             };
             String suffix = (operation == null) ? "" : ("_right");
             StringBuilder props = new StringBuilder();
-            props.append(String.format("\"match0%s\": { \"type\": \"long\" }", suffix));
+            props.append(String.format(Locale.ROOT, "\"match0%s\": { \"type\": \"long\" }", suffix));
             if (numberOfJoinColumns == 2) {
-                props.append(String.format(", \"match1%s\": { \"type\": \"long\" }", suffix));
+                props.append(String.format(Locale.ROOT, ", \"match1%s\": { \"type\": \"long\" }", suffix));
             }
             props.append(", \"lkwd\": { \"type\": \"keyword\" }, \"lint\": { \"type\": \"integer\" }");
-            String mapping = String.format("{\n  \"doc\": { \"properties\": { %s } }\n}", props.toString());
+            String mapping = String.format(Locale.ROOT, "{\n  \"doc\": { \"properties\": { %s } }\n}", props.toString());
             MapperService mapperService = mapperHelper.createMapperService(mapping);
             DirectoryReader reader = DirectoryReader.open(lookupIndexDirectory);
             SearchExecutionContext executionCtx = mapperHelper.createSearchExecutionContext(mapperService, newSearcher(reader));

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java

@@ -6803,7 +6803,7 @@ public void testReplaceStringCasingWithInsensitiveEqualsLowerTrue() {
     public void testReplaceStringCasingWithInsensitiveEqualsEquals() {
         for (var fn : List.of("TO_LOWER", "TO_UPPER")) {
             var value = fn.equals("TO_LOWER") ? fn.toLowerCase(Locale.ROOT) : fn.toUpperCase(Locale.ROOT);
-            value += "�✈��"; // these should not cause folding, they're not in the upper/lower char class
+            value += "🐔✈🔥🎉"; // these should not cause folding, they're not in the upper/lower char class
             var plan = optimizedPlan("FROM test | WHERE " + fn + "(first_name) == \"" + value + "\"");
             var limit = as(plan, Limit.class);
             var filter = as(limit.child(), Filter.class);
@@ -6818,7 +6818,7 @@ public void testReplaceStringCasingWithInsensitiveEqualsEquals() {
     public void testReplaceStringCasingWithInsensitiveEqualsNotEquals() {
         for (var fn : List.of("TO_LOWER", "TO_UPPER")) {
             var value = fn.equals("TO_LOWER") ? fn.toLowerCase(Locale.ROOT) : fn.toUpperCase(Locale.ROOT);
-            value += "�✈��"; // these should not cause folding, they're not in the upper/lower char class
+            value += "🐔✈🔥🎉"; // these should not cause folding, they're not in the upper/lower char class
             var plan = optimizedPlan("FROM test | WHERE " + fn + "(first_name) != \"" + value + "\"");
             var limit = as(plan, Limit.class);
             var filter = as(limit.child(), Filter.class);