Skip to content

[8.19] [ES|QL] RERANK command - Updating the syntax and behavior (#129488) #129718

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jun 20, 2025
Merged

[8.19] [ES|QL] RERANK command - Updating the syntax and behavior (#129488) #129718

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ef0f781
[ES|QL] RERANK command - Updating the syntax and behavior (#129488)
afoucret Jun 19, 2025
8e0e7bb
[CI] Auto commit changes from spotless
Jun 19, 2025
d84b862
Merge branch '8.19' of https://github.com/elastic/elasticsearch into …
afoucret Jun 19, 2025
2fc13a7
[CI] Auto commit changes from spotless
Jun 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...ver/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ public void testIndicesDontExist() throws IOException {
// currently we don't support remote clusters in LOOKUP JOIN
// this check happens before resolving actual indices and results in a different error message
RemoteClusterAware.isRemoteIndexName(pattern)
? allOf(containsString("parsing_exception"), containsString("remote clusters are not supported in LOOKUP JOIN"))
? allOf(containsString("parsing_exception"), containsString("remote clusters are not supported"))
: allOf(containsString("verification_exception"), containsString("Unknown index [foo]"))
);
}
Expand Down
116 changes: 82 additions & 34 deletions x-pack/plugin/esql/qa/testFixtures/src/main/resources/rerank.csv-spec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,62 @@
// This makes the output more predictable which is helpful here.


reranker using a single field
reranker using a single field, overwrite existing _score column
required_capability: rerank
required_capability: match_operator_colon

FROM books METADATA _score
| WHERE title:"war and peace" AND author:"Tolstoy"
| RERANK "war and peace" ON title WITH test_reranker
| KEEP book_no, title, author
| SORT _score DESC, book_no ASC
| RERANK "war and peace" ON title WITH inferenceId=test_reranker
| EVAL _score=ROUND(_score, 2)
| KEEP book_no, title, author, _score
;

book_no:keyword | title:text | author:text
5327 | War and Peace | Leo Tolstoy
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy]
9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo
2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy
book_no:keyword | title:text | author:text | _score:double
5327 | War and Peace | Leo Tolstoy | 0.08
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.03
9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.03
2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy | 0.02
;

reranker using a single field, create a mew column
required_capability: rerank
required_capability: match_operator_colon

FROM books METADATA _score
| WHERE title:"war and peace" AND author:"Tolstoy"
| SORT _score DESC, book_no ASC
| RERANK "war and peace" ON title WITH inferenceId=test_reranker, scoreColumn=rerank_score
| EVAL _score=ROUND(_score, 2), rerank_score=ROUND(rerank_score, 2)
| KEEP book_no, title, author, rerank_score
;

book_no:keyword | title:text | author:text | rerank_score:double
5327 | War and Peace | Leo Tolstoy | 0.08
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.03
9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.03
2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy | 0.02
;

reranker using a single field, create a mew column, sort by rerank_score
required_capability: rerank
required_capability: match_operator_colon

FROM books METADATA _score
| WHERE title:"war and peace" AND author:"Tolstoy"
| SORT _score DESC
| RERANK "war and peace" ON title WITH inferenceId=test_reranker, scoreColumn=rerank_score
| EVAL _score=ROUND(_score, 2), rerank_score=ROUND(rerank_score, 2)
| SORT rerank_score, _score ASC, book_no ASC
| KEEP book_no, title, author, rerank_score
;

book_no:keyword | title:text | author:text | rerank_score:double
2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy | 0.02
9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.03
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.03
5327 | War and Peace | Leo Tolstoy | 0.08
;


Expand All @@ -27,15 +68,17 @@ required_capability: match_operator_colon

FROM books METADATA _score
| WHERE title:"war and peace" AND author:"Tolstoy"
| RERANK "war and peace" ON title, author WITH test_reranker
| KEEP book_no, title, author
| RERANK "war and peace" ON title, author WITH inferenceId=test_reranker
| EVAL _score=ROUND(_score, 2)
| SORT _score DESC, book_no ASC
| KEEP book_no, title, author, _score
;

book_no:keyword | title:text | author:text
5327 | War and Peace | Leo Tolstoy
9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo
2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy]
book_no:keyword | title:text | author:text | _score:double
5327 | War and Peace | Leo Tolstoy | 0.02
2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy | 0.01
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.01
9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.01
;


Expand All @@ -45,16 +88,18 @@ required_capability: match_operator_colon

FROM books METADATA _score
| WHERE title:"war and peace" AND author:"Tolstoy"
| SORT _score DESC
| SORT _score DESC, book_no ASC
| LIMIT 3
| RERANK "war and peace" ON title WITH test_reranker
| KEEP book_no, title, author
| RERANK "war and peace" ON title WITH inferenceId=test_reranker
| EVAL _score=ROUND(_score, 2)
| SORT _score DESC, book_no ASC
| KEEP book_no, title, author, _score
;

book_no:keyword | title:text | author:text
5327 | War and Peace | Leo Tolstoy
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy]
9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo
book_no:keyword | title:text | author:text | _score:double
5327 | War and Peace | Leo Tolstoy | 0.08
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.03
9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.03
;


Expand All @@ -64,15 +109,17 @@ required_capability: match_operator_colon

FROM books METADATA _score
| WHERE title:"war and peace" AND author:"Tolstoy"
| RERANK "war and peace" ON title WITH test_reranker
| KEEP book_no, title, author
| RERANK "war and peace" ON title WITH inferenceId=test_reranker
| EVAL _score=ROUND(_score, 2)
| SORT _score DESC, book_no ASC
| KEEP book_no, title, author, _score
| LIMIT 3
;

book_no:keyword | title:text | author:text
5327 | War and Peace | Leo Tolstoy
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy]
9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo
book_no:keyword | title:text | author:text | _score:double
5327 | War and Peace | Leo Tolstoy | 0.08
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.03
9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.03
;


Expand All @@ -82,14 +129,15 @@ required_capability: match_operator_colon

FROM books
| WHERE title:"war and peace" AND author:"Tolstoy"
| RERANK "war and peace" ON title WITH test_reranker
| KEEP book_no, title, author
| RERANK "war and peace" ON title WITH inferenceId=test_reranker
| EVAL _score=ROUND(_score, 2)
| KEEP book_no, title, author, _score
| SORT author, title
| LIMIT 3
;

book_no:keyword | title:text | author:text
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy]
2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy
5327 | War and Peace | Leo Tolstoy
book_no:keyword | title:text | author:text | _score:double
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.03
2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy | 0.02
5327 | War and Peace | Leo Tolstoy | 0.08
;
15 changes: 14 additions & 1 deletion x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -387,8 +387,21 @@ joinPredicate
: valueExpression
;

inferenceCommandOptions
: inferenceCommandOption (COMMA inferenceCommandOption)*
;

inferenceCommandOption
: identifier ASSIGN inferenceCommandOptionValue
;

inferenceCommandOptionValue
: constant
| identifier
;

rerankCommand
: DEV_RERANK queryText=constant ON rerankFields (WITH inferenceId=identifierOrParameter)?
: DEV_RERANK queryText=constant ON rerankFields (WITH inferenceCommandOptions)?
;

completionCommand
Expand Down
6 changes: 5 additions & 1 deletion x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -750,7 +750,11 @@ private LogicalPlan resolveRerank(Rerank rerank, List<Attribute> childrenOutput)
if (rerank.scoreAttribute() instanceof UnresolvedAttribute ua) {
Attribute resolved = resolveAttribute(ua, childrenOutput);
if (resolved.resolved() == false || resolved.dataType() != DOUBLE) {
resolved = MetadataAttribute.create(Source.EMPTY, MetadataAttribute.SCORE);
if (ua.name().equals(MetadataAttribute.SCORE)) {
resolved = MetadataAttribute.create(Source.EMPTY, MetadataAttribute.SCORE);
} else {
resolved = new ReferenceAttribute(resolved.source(), resolved.name(), DOUBLE);
}
}
rerank = rerank.withScoreAttribute(resolved);
}
Expand Down
4 changes: 2 additions & 2 deletions ...lugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,9 @@
import org.elasticsearch.xpack.esql.optimizer.rules.logical.PushDownAndCombineLimits;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.PushDownAndCombineOrderBy;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.PushDownAndCombineSample;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.PushDownCompletion;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.PushDownEnrich;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.PushDownEval;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.PushDownInferencePlan;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.PushDownRegexExtract;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.RemoveStatsOverride;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceAggregateAggExpressionWithEval;
Expand Down Expand Up @@ -187,7 +187,7 @@ protected static Batch<LogicalPlan> operators() {
new PushDownAndCombineLimits(),
new PushDownAndCombineFilters(),
new PushDownAndCombineSample(),
new PushDownCompletion(),
new PushDownInferencePlan(),
new PushDownEval(),
new PushDownRegexExtract(),
new PushDownEnrich(),
Expand Down
8 changes: 4 additions & 4 deletions .../java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import org.elasticsearch.xpack.esql.plan.logical.Project;
import org.elasticsearch.xpack.esql.plan.logical.RegexExtract;
import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan;
import org.elasticsearch.xpack.esql.plan.logical.inference.Completion;
import org.elasticsearch.xpack.esql.plan.logical.inference.InferencePlan;
import org.elasticsearch.xpack.esql.plan.logical.join.Join;
import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes;

Expand Down Expand Up @@ -71,10 +71,10 @@ protected LogicalPlan rule(Filter filter) {
// Push down filters that do not rely on attributes created by RegexExtract
var attributes = AttributeSet.of(Expressions.asAttributes(re.extractedFields()));
plan = maybePushDownPastUnary(filter, re, attributes::contains, NO_OP);
} else if (child instanceof Completion completion) {
} else if (child instanceof InferencePlan<?> inferencePlan) {
// Push down filters that do not rely on attributes created by Cpmpletion
var attributes = AttributeSet.of(completion.generatedAttributes());
plan = maybePushDownPastUnary(filter, completion, attributes::contains, NO_OP);
var attributes = AttributeSet.of(inferencePlan.generatedAttributes());
plan = maybePushDownPastUnary(filter, inferencePlan, attributes::contains, NO_OP);
} else if (child instanceof Enrich enrich) {
// Push down filters that do not rely on attributes created by Enrich
var attributes = AttributeSet.of(Expressions.asAttributes(enrich.enrichFields()));
Expand Down
4 changes: 2 additions & 2 deletions ...n/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimits.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import org.elasticsearch.xpack.esql.plan.logical.Project;
import org.elasticsearch.xpack.esql.plan.logical.RegexExtract;
import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan;
import org.elasticsearch.xpack.esql.plan.logical.inference.Completion;
import org.elasticsearch.xpack.esql.plan.logical.inference.InferencePlan;
import org.elasticsearch.xpack.esql.plan.logical.join.Join;
import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes;

Expand All @@ -43,7 +43,7 @@ public LogicalPlan rule(Limit limit, LogicalOptimizerContext ctx) {
|| unary instanceof Project
|| unary instanceof RegexExtract
|| unary instanceof Enrich
|| unary instanceof Completion) {
|| unary instanceof InferencePlan<?>) {
return unary.replaceChild(limit.replaceChild(unary.child()));
} else if (unary instanceof MvExpand) {
// MV_EXPAND can increase the number of rows, so we cannot just push the limit down
Expand Down
6 changes: 3 additions & 3 deletions ...zer/rules/logical/PushDownCompletion.java → .../rules/logical/PushDownInferencePlan.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@
package org.elasticsearch.xpack.esql.optimizer.rules.logical;

import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
import org.elasticsearch.xpack.esql.plan.logical.inference.Completion;
import org.elasticsearch.xpack.esql.plan.logical.inference.InferencePlan;

public final class PushDownCompletion extends OptimizerRules.OptimizerRule<Completion> {
public final class PushDownInferencePlan extends OptimizerRules.OptimizerRule<InferencePlan<?>> {
@Override
protected LogicalPlan rule(Completion p) {
protected LogicalPlan rule(InferencePlan<?> p) {
return PushDownUtils.pushGeneratingPlanPastProjectAndOrderBy(p);
}
}
5 changes: 4 additions & 1 deletion x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp
View file
Open in desktop

Large diffs are not rendered by default.

Loading