From 0ada5c4c90389ab3ea01d3e6c4dce5904ec0e111 Mon Sep 17 00:00:00 2001 From: Julian Kiryakov Date: Thu, 18 Sep 2025 16:46:36 -0400 Subject: [PATCH 1/4] Performance improvements for Lookup Join on Expression --- .../esql/enrich/ExpressionQueryList.java | 32 ++++++++++++++----- .../esql/enrich/LookupFromIndexOperator.java | 18 +++++++++-- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ExpressionQueryList.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ExpressionQueryList.java index 9cd2bfb1b0c35..d881644849f2f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ExpressionQueryList.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ExpressionQueryList.java @@ -24,6 +24,7 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.expression.predicate.Predicates; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison; import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; import org.elasticsearch.xpack.esql.plan.physical.EsSourceExec; @@ -38,6 +39,7 @@ import java.util.List; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.LOOKUP_JOIN_ON_BOOLEAN_EXPRESSION; +import static org.elasticsearch.xpack.esql.enrich.AbstractLookupService.termQueryList; import static org.elasticsearch.xpack.esql.planner.TranslatorHandler.TRANSLATOR_HANDLER; /** @@ -154,17 +156,31 @@ private void buildJoinOnForExpressionJoin( if (right instanceof Attribute rightAttribute) { MappedFieldType fieldType = context.getFieldType(rightAttribute.name()); if (fieldType != null) { - queryLists.add( - new BinaryComparisonQueryList( + // special handle Equals operator + // TermQuery is faster than BinaryComparisonQueryList, as it does less work per row + // so here we reuse the existing logic from field based join to build a termQueryList for Equals + if (binaryComparison instanceof Equals) { + QueryList termQueryForEquals = termQueryList( fieldType, context, - block, - binaryComparison, - clusterService, aliasFilter, - warnings - ) - ); + inputPage.getBlock(matchFields.get(i).channel()), + matchFields.get(i).type() + ).onlySingleValues(warnings, "LOOKUP JOIN encountered multi-value"); + queryLists.add(termQueryForEquals); + } else { + queryLists.add( + new BinaryComparisonQueryList( + fieldType, + context, + block, + binaryComparison, + clusterService, + aliasFilter, + warnings + ) + ); + } matched = true; break; } else { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java index 3fbb54aa3a365..f5416b3083767 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java @@ -31,10 +31,12 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.function.Function; // TODO rename package @@ -145,8 +147,9 @@ public LookupFromIndexOperator( protected void performAsync(Page inputPage, ActionListener listener) { Block[] inputBlockArray = new Block[matchFields.size()]; List newMatchFields = new ArrayList<>(); - for (int i = 0; i < matchFields.size(); i++) { - MatchConfig matchField = matchFields.get(i); + List uniqueMatchFields = uniqueMatchFieldsByName(matchFields); + for (int i = 0; i < uniqueMatchFields.size(); i++) { + MatchConfig matchField = uniqueMatchFields.get(i); int inputChannel = matchField.channel(); final Block inputBlock = inputPage.getBlock(inputChannel); inputBlockArray[i] = inputBlock; @@ -176,6 +179,17 @@ protected void performAsync(Page inputPage, ActionListener listener ); } + private List uniqueMatchFieldsByName(List matchFields) { + List uniqueFields = new ArrayList<>(); + Set seenFieldNames = new HashSet<>(); + for (MatchConfig matchField : matchFields) { + if (seenFieldNames.add(matchField.fieldName())) { + uniqueFields.add(matchField); + } + } + return uniqueFields; + } + @Override public Page getOutput() { if (ongoing == null) { From 49e9d3e7d78ef376388c7a084d1019c607822d52 Mon Sep 17 00:00:00 2001 From: Julian Kiryakov Date: Fri, 19 Sep 2025 10:57:59 -0400 Subject: [PATCH 2/4] Bugfix --- .../xpack/esql/enrich/LookupFromIndexOperator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java index f5416b3083767..73ca68a36b14b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java @@ -145,9 +145,9 @@ public LookupFromIndexOperator( @Override protected void performAsync(Page inputPage, ActionListener listener) { - Block[] inputBlockArray = new Block[matchFields.size()]; List newMatchFields = new ArrayList<>(); List uniqueMatchFields = uniqueMatchFieldsByName(matchFields); + Block[] inputBlockArray = new Block[uniqueMatchFields.size()]; for (int i = 0; i < uniqueMatchFields.size(); i++) { MatchConfig matchField = uniqueMatchFields.get(i); int inputChannel = matchField.channel(); From 9ad670494e6f0cdb0e8f0ca8e9b1a0d199344198 Mon Sep 17 00:00:00 2001 From: Julian Kiryakov Date: Thu, 18 Sep 2025 16:52:10 -0400 Subject: [PATCH 3/4] Update docs/changelog/135036.yaml --- docs/changelog/135036.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/135036.yaml diff --git a/docs/changelog/135036.yaml b/docs/changelog/135036.yaml new file mode 100644 index 0000000000000..7bf09d6571a99 --- /dev/null +++ b/docs/changelog/135036.yaml @@ -0,0 +1,5 @@ +pr: 135036 +summary: Performance improvements for Lookup Join on Expression +area: ES|QL +type: enhancement +issues: [] From 20c7fa55fddac9b1cc456ba117c9712f9aa6da7f Mon Sep 17 00:00:00 2001 From: Julian Kiryakov Date: Fri, 19 Sep 2025 14:00:01 -0400 Subject: [PATCH 4/4] Disable optimization for field based join --- .../xpack/esql/enrich/LookupFromIndexOperator.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java index 73ca68a36b14b..53ebd1d89d3b4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java @@ -180,6 +180,9 @@ protected void performAsync(Page inputPage, ActionListener listener } private List uniqueMatchFieldsByName(List matchFields) { + if (joinOnConditions == null) { + return matchFields; + } List uniqueFields = new ArrayList<>(); Set seenFieldNames = new HashSet<>(); for (MatchConfig matchField : matchFields) {