Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -325,3 +325,20 @@ count:long | type:keyword
3 | Success
1 | Disconnected
;

sortEvalBeforeLookup
required_capability: join_lookup_v4

FROM employees
| SORT emp_no
| EVAL language_code = (emp_no % 10) + 1
| LOOKUP JOIN languages_lookup ON language_code
| KEEP emp_no, language_code, language_name
| LIMIT 3
;

emp_no:integer | language_code:integer | language_name:keyword
10001 | 2 | French
10002 | 3 | Spanish
10003 | 4 | German
;
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan;
import org.elasticsearch.xpack.esql.plan.logical.join.Join;
import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes;
import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;

public final class PushDownAndCombineLimits extends OptimizerRules.OptimizerRule<Limit> {

Expand Down Expand Up @@ -63,8 +62,10 @@ public LogicalPlan rule(Limit limit) {
}
}
} else if (limit.child() instanceof Join join) {
if (join.config().type() == JoinTypes.LEFT && join.right() instanceof LocalRelation) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wondering why the previous restriction to LocalRelation (even if that's what was expected before). Anyways, should be good this way now.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That came from when we only had LOOKUP (aka 🐔 ) and INLINESTATS to consider. Both of these ended up with a LocalRelation on the right, and we didn't want to implement this in all generality because the semantics were not 100% clear.

// This is a hash join from something like a lookup.
if (join.config().type() == JoinTypes.LEFT) {
// NOTE! This is only correct because our LEFT JOINs preserve the number of rows from the left hand side.
// This deviates from SQL semantics. In SQL, multiple matches on the right hand side lead to multiple rows in the output.
// For us, multiple matches on the right hand side lead are collected into multi-values.
return join.replaceChildren(limit.replaceChild(join.left()), join.right());
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull;
import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.core.type.EsField;
import org.elasticsearch.xpack.esql.core.util.Holder;
Expand Down Expand Up @@ -113,7 +114,9 @@
import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan;
import org.elasticsearch.xpack.esql.plan.logical.join.InlineJoin;
import org.elasticsearch.xpack.esql.plan.logical.join.Join;
import org.elasticsearch.xpack.esql.plan.logical.join.JoinConfig;
import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes;
import org.elasticsearch.xpack.esql.plan.logical.join.LookupJoin;
import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject;
import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier;
Expand All @@ -139,6 +142,7 @@
import static org.elasticsearch.xpack.esql.EsqlTestUtils.TWO;
import static org.elasticsearch.xpack.esql.EsqlTestUtils.as;
import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptySource;
import static org.elasticsearch.xpack.esql.EsqlTestUtils.fieldAttribute;
import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute;
import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping;
import static org.elasticsearch.xpack.esql.EsqlTestUtils.localSource;
Expand Down Expand Up @@ -1294,6 +1298,26 @@ public void testCombineLimits() {
);
}

public void testPushdownLimitsPastLeftJoin() {
var leftChild = emptySource();
var rightChild = new LocalRelation(Source.EMPTY, List.of(fieldAttribute()), LocalSupplier.EMPTY);
assertNotEquals(leftChild, rightChild);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

😄

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Gotta be very careful here, you never know.


var joinConfig = new JoinConfig(JoinTypes.LEFT, List.of(), List.of(), List.of());
var join = switch (randomIntBetween(0, 2)) {
case 0 -> new Join(EMPTY, leftChild, rightChild, joinConfig);
case 1 -> new LookupJoin(EMPTY, leftChild, rightChild, joinConfig);
case 2 -> new InlineJoin(EMPTY, leftChild, rightChild, joinConfig);
default -> throw new IllegalArgumentException();
};

var limit = new Limit(EMPTY, L(10), join);

var optimizedPlan = new PushDownAndCombineLimits().rule(limit);

assertEquals(join.replaceChildren(limit.replaceChild(join.left()), join.right()), optimizedPlan);
}

public void testMultipleCombineLimits() {
var numberOfLimits = randomIntBetween(3, 10);
var minimum = randomIntBetween(10, 99);
Expand Down