Skip to content

Commit ecb3d21

Browse files
astefanastefan
authored
ESQL: Use a must boolean statement when pushing down to Lucene when scoring is also needed (#124001)
* Use a "must" instead of "filter" when building the pushed down filter AND when scoring is needed
1 parent a06c8ea commit ecb3d21

File tree

8 files changed

+159
-19
lines changed

8 files changed

+159
-19
lines changed

‎docs/changelog/124001.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
pr: 124001
2+
summary: Use a must boolean statement when pushing down to Lucene when scoring is
3+
also needed
4+
area: ES|QL
5+
type: bug
6+
issues:
7+
- 123967

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
import static org.elasticsearch.core.Tuple.tuple;
3030

3131
public class MetadataAttribute extends TypedAttribute {
32-
public static final String TIMESTAMP_FIELD = "@timestamp";
32+
public static final String TIMESTAMP_FIELD = "@timestamp"; // this is not a true metadata attribute
3333
public static final String TSID_FIELD = "_tsid";
3434
public static final String SCORE = "_score";
3535
public static final String INDEX = "_index";

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,16 @@
1111
import org.elasticsearch.action.index.IndexRequest;
1212
import org.elasticsearch.action.support.WriteRequest;
1313
import org.elasticsearch.common.settings.Settings;
14+
import org.elasticsearch.index.query.QueryBuilder;
15+
import org.elasticsearch.index.query.QueryBuilders;
1416
import org.elasticsearch.xpack.esql.VerificationException;
1517
import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase;
1618
import org.junit.Before;
1719

1820
import java.util.List;
1921

22+
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
23+
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
2024
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
2125
import static org.hamcrest.CoreMatchers.containsString;
2226

@@ -120,6 +124,119 @@ public void testWhereMatchWithScoring() {
120124
}
121125
}
122126

127+
/**
128+
* Test for https://github.com/elastic/elasticsearch/issues/123967
129+
*/
130+
public void testWhereMatchWithScoring_AndRequestFilter() {
131+
var query = """
132+
FROM test METADATA _score
133+
| WHERE content:"fox"
134+
| SORT _score DESC
135+
| KEEP content, _score
136+
""";
137+
138+
QueryBuilder filter = boolQuery().must(matchQuery("content", "brown"));
139+
140+
try (var resp = run(query, randomPragmas(), filter)) {
141+
assertColumnNames(resp.columns(), List.of("content", "_score"));
142+
assertColumnTypes(resp.columns(), List.of("text", "double"));
143+
assertValues(
144+
resp.values(),
145+
List.of(
146+
List.of("This is a brown fox", 1.4274532794952393),
147+
List.of("The quick brown fox jumps over the lazy dog", 1.1248724460601807)
148+
)
149+
);
150+
}
151+
}
152+
153+
public void testWhereMatchWithScoring_AndNoScoreRequestFilter() {
154+
var query = """
155+
FROM test METADATA _score
156+
| WHERE content:"fox"
157+
| SORT _score DESC
158+
| KEEP content, _score
159+
""";
160+
161+
QueryBuilder filter = boolQuery().filter(matchQuery("content", "brown"));
162+
163+
try (var resp = run(query, randomPragmas(), filter)) {
164+
assertColumnNames(resp.columns(), List.of("content", "_score"));
165+
assertColumnTypes(resp.columns(), List.of("text", "double"));
166+
assertValues(
167+
resp.values(),
168+
List.of(
169+
List.of("This is a brown fox", 1.156558871269226),
170+
List.of("The quick brown fox jumps over the lazy dog", 0.9114001989364624)
171+
)
172+
);
173+
}
174+
}
175+
176+
public void testWhereMatchWithScoring_And_MatchAllRequestFilter() {
177+
var query = """
178+
FROM test METADATA _score
179+
| WHERE content:"fox"
180+
| SORT _score DESC
181+
| KEEP content, _score
182+
""";
183+
184+
QueryBuilder filter = QueryBuilders.matchAllQuery();
185+
186+
try (var resp = run(query, randomPragmas(), filter)) {
187+
assertColumnNames(resp.columns(), List.of("content", "_score"));
188+
assertColumnTypes(resp.columns(), List.of("text", "double"));
189+
assertValues(
190+
resp.values(),
191+
List.of(
192+
List.of("This is a brown fox", 2.1565589904785156),
193+
List.of("The quick brown fox jumps over the lazy dog", 1.9114001989364624)
194+
)
195+
);
196+
}
197+
}
198+
199+
public void testScoringOutsideQuery() {
200+
var query = """
201+
FROM test METADATA _score
202+
| SORT _score DESC
203+
| KEEP content, _score
204+
""";
205+
206+
QueryBuilder filter = boolQuery().must(matchQuery("content", "fox"));
207+
208+
try (var resp = run(query, randomPragmas(), filter)) {
209+
assertColumnNames(resp.columns(), List.of("content", "_score"));
210+
assertColumnTypes(resp.columns(), List.of("text", "double"));
211+
assertValues(
212+
resp.values(),
213+
List.of(
214+
List.of("This is a brown fox", 1.156558871269226),
215+
List.of("The quick brown fox jumps over the lazy dog", 0.9114001989364624)
216+
)
217+
);
218+
}
219+
}
220+
221+
public void testScoring_Zero_OutsideQuery() {
222+
var query = """
223+
FROM test METADATA _score
224+
| SORT _score DESC
225+
| KEEP content, _score
226+
""";
227+
228+
QueryBuilder filter = boolQuery().filter(matchQuery("content", "fox"));
229+
230+
try (var resp = run(query, randomPragmas(), filter)) {
231+
assertColumnNames(resp.columns(), List.of("content", "_score"));
232+
assertColumnTypes(resp.columns(), List.of("text", "double"));
233+
assertValues(
234+
resp.values(),
235+
List.of(List.of("This is a brown fox", 0.0), List.of("The quick brown fox jumps over the lazy dog", 0.0))
236+
);
237+
}
238+
}
239+
123240
public void testWhereMatchWithScoringDifferentSort() {
124241
var query = """
125242
FROM test

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,8 @@ private static PhysicalPlan rewrite(
101101
if (newPushable.size() > 0) { // update the executable with pushable conditions
102102
Query queryDSL = TRANSLATOR_HANDLER.asQuery(Predicates.combineAnd(newPushable));
103103
QueryBuilder planQuery = queryDSL.asBuilder();
104-
var query = Queries.combine(Queries.Clause.FILTER, asList(queryExec.query(), planQuery));
104+
Queries.Clause combiningQueryClauseType = queryExec.hasScoring() ? Queries.Clause.MUST : Queries.Clause.FILTER;
105+
var query = Queries.combine(combiningQueryClauseType, asList(queryExec.query(), planQuery));
105106
queryExec = new EsQueryExec(
106107
queryExec.source(),
107108
queryExec.indexPattern(),

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/ReplaceSourceAttributes.java‎

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -32,27 +32,36 @@ protected PhysicalPlan rule(EsSourceExec plan) {
3232
var docId = new FieldAttribute(plan.source(), EsQueryExec.DOC_ID_FIELD.getName(), EsQueryExec.DOC_ID_FIELD);
3333
final List<Attribute> attributes = new ArrayList<>();
3434
attributes.add(docId);
35-
if (plan.indexMode() == IndexMode.TIME_SERIES) {
36-
Attribute tsid = null, timestamp = null;
37-
for (Attribute attr : plan.output()) {
38-
String name = attr.name();
39-
if (name.equals(MetadataAttribute.TSID_FIELD)) {
35+
36+
var outputIterator = plan.output().iterator();
37+
var isTimeSeries = plan.indexMode() == IndexMode.TIME_SERIES;
38+
var keepIterating = true;
39+
Attribute tsid = null, timestamp = null, score = null;
40+
41+
while (keepIterating && outputIterator.hasNext()) {
42+
Attribute attr = outputIterator.next();
43+
if (attr instanceof MetadataAttribute ma) {
44+
if (ma.name().equals(MetadataAttribute.SCORE)) {
45+
score = attr;
46+
} else if (isTimeSeries && ma.name().equals(MetadataAttribute.TSID_FIELD)) {
4047
tsid = attr;
41-
} else if (name.equals(MetadataAttribute.TIMESTAMP_FIELD)) {
42-
timestamp = attr;
4348
}
49+
} else if (attr.name().equals(MetadataAttribute.TIMESTAMP_FIELD)) {
50+
timestamp = attr;
4451
}
52+
keepIterating = score == null || (isTimeSeries && (tsid == null || timestamp == null));
53+
}
54+
if (isTimeSeries) {
4555
if (tsid == null || timestamp == null) {
4656
throw new IllegalStateException("_tsid or @timestamp are missing from the time-series source");
4757
}
4858
attributes.add(tsid);
4959
attributes.add(timestamp);
5060
}
51-
plan.output().forEach(attr -> {
52-
if (attr instanceof MetadataAttribute ma && ma.name().equals(MetadataAttribute.SCORE)) {
53-
attributes.add(ma);
54-
}
55-
});
61+
if (score != null) {
62+
attributes.add(score);
63+
}
64+
5665
return new EsQueryExec(plan.source(), plan.indexPattern(), plan.indexMode(), plan.indexNameWithModes(), attributes, plan.query());
5766
}
5867
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsQueryExec.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.elasticsearch.xpack.esql.core.expression.Attribute;
2222
import org.elasticsearch.xpack.esql.core.expression.Expression;
2323
import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
24+
import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
2425
import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
2526
import org.elasticsearch.xpack.esql.core.tree.NodeUtils;
2627
import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -204,6 +205,15 @@ public static boolean isSourceAttribute(Attribute attr) {
204205
return DOC_ID_FIELD.getName().equals(attr.name());
205206
}
206207

208+
public boolean hasScoring() {
209+
for (Attribute a : attrs()) {
210+
if (a instanceof MetadataAttribute && a.name().equals(MetadataAttribute.SCORE)) {
211+
return true;
212+
}
213+
}
214+
return false;
215+
}
216+
207217
@Override
208218
protected NodeInfo<EsQueryExec> info() {
209219
return NodeInfo.create(

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsStatsQueryExec.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ public enum StatsType {
3737
}
3838

3939
public record Stat(String name, StatsType type, QueryBuilder query) {
40-
4140
public QueryBuilder filter(QueryBuilder sourceQuery) {
4241
return query == null ? sourceQuery : Queries.combine(Queries.Clause.FILTER, asList(sourceQuery, query));
4342
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@
4949
import org.elasticsearch.xpack.esql.core.expression.Expression;
5050
import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
5151
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
52-
import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
5352
import org.elasticsearch.xpack.esql.core.type.DataType;
5453
import org.elasticsearch.xpack.esql.core.type.KeywordEsField;
5554
import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField;
@@ -186,9 +185,7 @@ public final PhysicalOperation sourcePhysicalOperation(EsQueryExec esQueryExec,
186185
assert esQueryExec.estimatedRowSize() != null : "estimated row size not initialized";
187186
int rowEstimatedSize = esQueryExec.estimatedRowSize();
188187
int limit = esQueryExec.limit() != null ? (Integer) esQueryExec.limit().fold(context.foldCtx()) : NO_LIMIT;
189-
boolean scoring = esQueryExec.attrs()
190-
.stream()
191-
.anyMatch(a -> a instanceof MetadataAttribute && a.name().equals(MetadataAttribute.SCORE));
188+
boolean scoring = esQueryExec.hasScoring();
192189
if ((sorts != null && sorts.isEmpty() == false)) {
193190
List<SortBuilder<?>> sortBuilders = new ArrayList<>(sorts.size());
194191
for (Sort sort : sorts) {

0 commit comments

Comments
 (0)