From 7d5ad3974656ae1072d666307222e08236523fd8 Mon Sep 17 00:00:00 2001 From: Carlos Delgado <6339205+carlosdelest@users.noreply.github.com> Date: Tue, 18 Mar 2025 14:37:20 +0100 Subject: [PATCH] ES\QL: Fix scoring for non full text functions (#124540) (cherry picked from commit 7950751788ab7e52ad1fa76392f5d9d4766f500c) # Conflicts: # x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/QueryBuilderResolver.java # x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java # x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java --- docs/changelog/124540.yaml | 5 + .../esql/core/querydsl/query/BoolQuery.java | 12 +- .../esql/core/querydsl/query/ExistsQuery.java | 2 +- .../core/querydsl/query/GeoDistanceQuery.java | 2 +- .../esql/core/querydsl/query/MatchAll.java | 2 +- .../esql/core/querydsl/query/NotQuery.java | 4 +- .../esql/core/querydsl/query/PrefixQuery.java | 2 +- .../xpack/esql/core/querydsl/query/Query.java | 33 +- .../core/querydsl/query/QueryStringQuery.java | 7 +- .../esql/core/querydsl/query/RangeQuery.java | 2 +- .../esql/core/querydsl/query/RegexQuery.java | 2 +- .../esql/core/querydsl/query/TermQuery.java | 22 +- .../esql/core/querydsl/query/TermsQuery.java | 2 +- .../core/querydsl/query/WildcardQuery.java | 2 +- .../core/querydsl/query/LeafQueryTests.java | 2 +- .../src/main/resources/scoring.csv-spec | 46 ++- .../xpack/esql/plugin/ScoringIT.java | 336 ++++++++++++------ .../xpack/esql/action/EsqlCapabilities.java | 7 +- .../fulltext/QueryBuilderResolver.java | 2 +- .../expression/function/fulltext/Term.java | 3 +- .../physical/local/PushFiltersToSource.java | 2 +- .../physical/local/PushStatsToSource.java | 2 +- .../esql/plan/physical/EsStatsQueryExec.java | 2 +- .../planner/EsPhysicalOperationProviders.java | 2 +- .../xpack/esql/planner/PlannerUtils.java | 2 +- .../xpack/esql/querydsl/query/KqlQuery.java | 7 +- .../xpack/esql/querydsl/query/MatchQuery.java | 7 +- .../esql/querydsl/query/MultiMatchQuery.java | 2 +- .../esql/querydsl/query/SingleValueQuery.java | 4 +- .../querydsl/query/SpatialRelatesQuery.java | 9 +- .../TranslationAwareExpressionQuery.java | 8 +- .../function/fulltext/MatchErrorTests.java | 2 +- .../function/fulltext/MatchTests.java | 2 +- .../LocalPhysicalPlanOptimizerTests.java | 89 ++--- .../optimizer/PhysicalPlanOptimizerTests.java | 36 +- .../xpack/esql/planner/FilterTests.java | 56 ++- .../esql/planner/QueryTranslatorTests.java | 16 +- .../querydsl/query/QueryStringQueryTests.java | 2 +- 38 files changed, 502 insertions(+), 243 deletions(-) create mode 100644 docs/changelog/124540.yaml diff --git a/docs/changelog/124540.yaml b/docs/changelog/124540.yaml new file mode 100644 index 0000000000000..9a6846f1bdb22 --- /dev/null +++ b/docs/changelog/124540.yaml @@ -0,0 +1,5 @@ +pr: 124540 +summary: "ES|QL: Fix scoring for full text functions" +area: ES|QL +type: bug +issues: [] diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQuery.java index dbd75c93ee0e7..2525eb8778488 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQuery.java @@ -43,13 +43,14 @@ public BoolQuery(Source source, boolean isAnd, List queries) { } @Override - public QueryBuilder asBuilder() { + protected QueryBuilder asBuilder() { BoolQueryBuilder boolQuery = boolQuery(); for (Query query : queries) { + QueryBuilder queryBuilder = query.toQueryBuilder(); if (isAnd) { - boolQuery.must(query.asBuilder()); + boolQuery.must(queryBuilder); } else { - boolQuery.should(query.asBuilder()); + boolQuery.should(queryBuilder); } } return boolQuery; @@ -94,4 +95,9 @@ public Query negate(Source source) { } return new BoolQuery(source, isAnd == false, negated); } + + @Override + public boolean scorable() { + return true; + } } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/ExistsQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/ExistsQuery.java index be585232cf8d6..42876f2a7ead3 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/ExistsQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/ExistsQuery.java @@ -21,7 +21,7 @@ public ExistsQuery(Source source, String name) { } @Override - public QueryBuilder asBuilder() { + protected QueryBuilder asBuilder() { return existsQuery(name); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/GeoDistanceQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/GeoDistanceQuery.java index f7843cec0c88c..f0806660c3f7a 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/GeoDistanceQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/GeoDistanceQuery.java @@ -45,7 +45,7 @@ public double distance() { } @Override - public QueryBuilder asBuilder() { + protected QueryBuilder asBuilder() { return QueryBuilders.geoDistanceQuery(field).distance(distance, DistanceUnit.METERS).point(lat, lon); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchAll.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchAll.java index 6415a69e2201d..3a868349a183d 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchAll.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchAll.java @@ -17,7 +17,7 @@ public MatchAll(Source source) { } @Override - public QueryBuilder asBuilder() { + protected QueryBuilder asBuilder() { return matchAllQuery(); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NotQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NotQuery.java index 4e36a4ee9f053..1a37fc8f42b9a 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NotQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NotQuery.java @@ -32,8 +32,8 @@ public Query child() { } @Override - public QueryBuilder asBuilder() { - return boolQuery().mustNot(child.asBuilder()); + protected QueryBuilder asBuilder() { + return boolQuery().mustNot(child.toQueryBuilder()); } @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/PrefixQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/PrefixQuery.java index 1d98ff53be2f2..ede4aee3d117e 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/PrefixQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/PrefixQuery.java @@ -34,7 +34,7 @@ public String query() { } @Override - public QueryBuilder asBuilder() { + protected QueryBuilder asBuilder() { return prefixQuery(field, query).caseInsensitive(caseInsensitive); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/Query.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/Query.java index f3154eb6cd377..456275a054899 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/Query.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/Query.java @@ -28,6 +28,10 @@ *

*/ public abstract class Query { + + // Boosting used to remove scoring from queries that don't contribute to it + public static final float NO_SCORE_BOOST = 0.0f; + private final Source source; protected Query(Source source) { @@ -46,9 +50,20 @@ public Source source() { /** * Convert to an Elasticsearch {@link QueryBuilder} all set up to execute - * the query. + * the query. This ensures that queries have appropriate boosting for scoring. + */ + public final QueryBuilder toQueryBuilder() { + QueryBuilder builder = asBuilder(); + if (scorable() == false) { + builder = unscore(builder); + } + return builder; + } + + /** + * Used internally to convert to retrieve a {@link QueryBuilder} by queries. */ - public abstract QueryBuilder asBuilder(); + protected abstract QueryBuilder asBuilder(); /** * Used by {@link Query#toString()} to produce a pretty string. @@ -85,4 +100,18 @@ public String toString() { public Query negate(Source source) { return new NotQuery(source, this); } + + /** + * Defines whether a query should contribute to the overall score + */ + public boolean scorable() { + return false; + } + + /** + * Removes score from a query builder, so score is not affected by the query + */ + public static QueryBuilder unscore(QueryBuilder builder) { + return builder.boost(NO_SCORE_BOOST); + } } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/QueryStringQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/QueryStringQuery.java index 8dcb87749ae48..4b459aefe95e1 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/QueryStringQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/QueryStringQuery.java @@ -64,7 +64,7 @@ public QueryStringQuery(Source source, String query, Map fields, } @Override - public QueryBuilder asBuilder() { + protected QueryBuilder asBuilder() { final QueryStringQueryBuilder queryBuilder = QueryBuilders.queryStringQuery(query); queryBuilder.fields(fields); options.forEach((k, v) -> { @@ -108,4 +108,9 @@ public boolean equals(Object obj) { protected String innerToString() { return fields + ":" + query; } + + @Override + public boolean scorable() { + return true; + } } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RangeQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RangeQuery.java index 2d66ee86d0f61..e7ddfd1735b28 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RangeQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RangeQuery.java @@ -77,7 +77,7 @@ public ZoneId zoneId() { } @Override - public QueryBuilder asBuilder() { + protected QueryBuilder asBuilder() { RangeQueryBuilder queryBuilder = rangeQuery(field).from(lower, includeLower).to(upper, includeUpper); if (Strings.hasText(format)) { queryBuilder.format(format); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RegexQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RegexQuery.java index a8e48de654196..b12802de4e715 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RegexQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RegexQuery.java @@ -42,7 +42,7 @@ public Boolean caseInsensitive() { } @Override - public QueryBuilder asBuilder() { + protected QueryBuilder asBuilder() { return regexpQuery(field, regex).caseInsensitive(caseInsensitive); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermQuery.java index 240f9f581b27e..03c3b29ba15ec 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermQuery.java @@ -14,21 +14,31 @@ import static org.elasticsearch.index.query.QueryBuilders.termQuery; +/** + * Term query. It can be considered for scoring or not - filters that use term query as implementation will not use scoring, + * but the Term full text function will + */ public class TermQuery extends Query { private final String term; private final Object value; private final boolean caseInsensitive; + private final boolean scorable; public TermQuery(Source source, String term, Object value) { this(source, term, value, false); } public TermQuery(Source source, String term, Object value, boolean caseInsensitive) { + this(source, term, value, caseInsensitive, false); + } + + public TermQuery(Source source, String term, Object value, boolean caseInsensitive, boolean scorable) { super(source); this.term = term; this.value = value; this.caseInsensitive = caseInsensitive; + this.scorable = scorable; } public String term() { @@ -44,7 +54,7 @@ public Boolean caseInsensitive() { } @Override - public QueryBuilder asBuilder() { + protected QueryBuilder asBuilder() { TermQueryBuilder qb = termQuery(term, value); // ES does not allow case_insensitive to be set to "false", it should be either "true" or not specified return caseInsensitive == false ? qb : qb.caseInsensitive(caseInsensitive); @@ -52,7 +62,7 @@ public QueryBuilder asBuilder() { @Override public int hashCode() { - return Objects.hash(term, value, caseInsensitive); + return Objects.hash(term, value, caseInsensitive, scorable); } @Override @@ -68,11 +78,17 @@ public boolean equals(Object obj) { TermQuery other = (TermQuery) obj; return Objects.equals(term, other.term) && Objects.equals(value, other.value) - && Objects.equals(caseInsensitive, other.caseInsensitive); + && Objects.equals(caseInsensitive, other.caseInsensitive) + && scorable == other.scorable; } @Override protected String innerToString() { return term + ":" + value; } + + @Override + public boolean scorable() { + return scorable; + } } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermsQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermsQuery.java index 5b0920929853a..68f8dd711f87a 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermsQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermsQuery.java @@ -26,7 +26,7 @@ public TermsQuery(Source source, String term, Set values) { } @Override - public QueryBuilder asBuilder() { + protected QueryBuilder asBuilder() { return termsQuery(term, values); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java index 9266f2b43d081..03d819cf7aa9b 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java @@ -43,7 +43,7 @@ public Boolean caseInsensitive() { } @Override - public QueryBuilder asBuilder() { + protected QueryBuilder asBuilder() { WildcardQueryBuilder wb = wildcardQuery(field, query); // ES does not allow case_insensitive to be set to "false", it should be either "true" or not specified return caseInsensitive == false ? wb : wb.caseInsensitive(caseInsensitive); diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQueryTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQueryTests.java index 15c49f58572cb..d0aa9bd073ec8 100644 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQueryTests.java +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQueryTests.java @@ -23,7 +23,7 @@ private DummyLeafQuery(Source source) { } @Override - public QueryBuilder asBuilder() { + protected QueryBuilder asBuilder() { return null; } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec index 540aec7a82e7b..ff89c4e14d4bb 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec @@ -145,6 +145,7 @@ book_no:keyword | title:text | author:text combinedMatchWithFunctionsScoring required_capability: metadata_score required_capability: match_operator_colon +required_capability: non_full_text_functions_scoring from books metadata _score | where title:"Tolkien" AND author:"Tolkien" AND year > 2000 @@ -153,7 +154,7 @@ from books metadata _score | sort book_no; book_no:keyword | title:text | author:text | year:integer | _score:double -5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 2014 | 4.733664035797119 +5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 2014 | 3.733664035797119 ; singleQstrScoring @@ -191,6 +192,7 @@ book_no:keyword | title:keyword | _score:double combinedMatchWithScoringEvalNoSort required_capability: metadata_score required_capability: match_operator_colon +required_capability: non_full_text_functions_scoring from books metadata _score | where title:"Tolkien" AND author:"Tolkien" AND year > 2000 @@ -200,7 +202,7 @@ from books metadata _score ignoreOrder:true book_no:keyword | title:text | author:text | year:integer | c_score:double -5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 2014 | 5.0 +5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 2014 | 4.0 ; singleQstrScoringRename @@ -241,6 +243,7 @@ book_no:keyword | author:text | _score:do combinedMatchWithFunctionsScoringNoSort required_capability: metadata_score required_capability: match_operator_colon +required_capability: non_full_text_functions_scoring from books metadata _score | where title:"Tolkien" AND author:"Tolkien" AND year > 2000 @@ -249,12 +252,13 @@ from books metadata _score ignoreOrder:true book_no:keyword | title:text | author:text | year:integer | _score:double -5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 2014 | 4.733664035797119 +5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 2014 | 3.733664035797119 ; combinedMatchWithScoringEval required_capability: metadata_score required_capability: match_operator_colon +required_capability: non_full_text_functions_scoring from books metadata _score | where title:"Tolkien" AND author:"Tolkien" AND year > 2000 @@ -264,7 +268,7 @@ from books metadata _score | sort book_no; book_no:keyword | title:text | author:text | year:integer | c_score:double -5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 2014 | 5.0 +5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 2014 | 4.0 ; singleQstrScoringEval @@ -361,6 +365,7 @@ _id:keyword scoresNonPushableFunctions required_capability: metadata_score +required_capability: non_full_text_functions_scoring from books metadata _score | where length(title) > 100 @@ -369,13 +374,14 @@ from books metadata _score ; book_no:keyword | _score:double -2924 | 1.0 -8678 | 1.0 +2924 | 0.0 +8678 | 0.0 ; scoresPushableFunctions required_capability: metadata_score +required_capability: non_full_text_functions_scoring from books metadata _score | where year >= 2017 @@ -384,11 +390,11 @@ from books metadata _score ; book_no:keyword | _score:double -6818 | 1.0 -7400 | 1.0 -8480 | 1.0 -8534 | 1.0 -8615 | 1.0 +6818 | 0.0 +7400 | 0.0 +8480 | 0.0 +8534 | 0.0 +8615 | 0.0 ; conjunctionScoresPushableNonPushableFunctions @@ -413,6 +419,7 @@ conjunctionScoresPushableFunctions required_capability: metadata_score required_capability: match_function +required_capability: non_full_text_functions_scoring from books metadata _score | where match(title, "Lord") and ratings > 4.6 @@ -421,8 +428,8 @@ from books metadata _score ; book_no:keyword | _score:double -7140 | 2.746896743774414 -4023 | 2.5062403678894043 +7140 | 1.746896743774414 +4023 | 1.5062403678894043 ; disjunctionScoresPushableNonPushableFunctions @@ -430,6 +437,7 @@ disjunctionScoresPushableNonPushableFunctions required_capability: metadata_score required_capability: match_operator_colon required_capability: full_text_functions_disjunctions_score +required_capability: non_full_text_functions_scoring from books metadata _score | where match(title, "Lord") or length(title) > 100 @@ -438,12 +446,12 @@ from books metadata _score ; book_no:keyword | _score:double -2675 | 3.5619282722473145 -2714 | 2.9245924949645996 -7140 | 2.746896743774414 -4023 | 2.5062403678894043 -2924 | 1.0 -8678 | 1.0 +2675 | 2.5619282722473145 +2714 | 1.9245924949645996 +7140 | 1.746896743774414 +4023 | 1.5062403678894043 +2924 | 0.0 +8678 | 0.0 ; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/ScoringIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/ScoringIT.java index 343a19f78dca5..b48045bfafaa7 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/ScoringIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/ScoringIT.java @@ -7,150 +7,272 @@ package org.elasticsearch.xpack.esql.plugin; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; +import org.elasticsearch.xpack.kql.KqlPlugin; import org.junit.Before; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Locale; +import java.util.Map; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThan; -//@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE,org.elasticsearch.compute:TRACE", reason = "debug") public class ScoringIT extends AbstractEsqlIntegTestCase { + private final String matchingClause; + + @Override + protected Collection> nodePlugins() { + return CollectionUtils.appendToCopy(super.nodePlugins(), KqlPlugin.class); + } + + @ParametersFactory + public static List params() { + List params = new ArrayList<>(); + params.add(new Object[] { "match(content, \"fox\")" }); + params.add(new Object[] { "content:\"fox\"" }); + params.add(new Object[] { "qstr(\"content: fox\")" }); + params.add(new Object[] { "kql(\"content*: fox\")" }); + params.add(new Object[] { "term(content, \"fox\")" }); + return params; + } + + public ScoringIT(String matchingClause) { + this.matchingClause = matchingClause; + } + @Before public void setupIndex() { createAndPopulateIndex(); } - public void testDefaultScoring() { - var query = """ - FROM test METADATA _score + public void testWhereMatchWithScoring() { + var query = String.format(Locale.ROOT, """ + FROM test + METADATA _score + | WHERE %s | KEEP id, _score - | SORT _score DESC, id ASC - """; + | SORT id ASC + """, matchingClause); try (var resp = run(query)) { assertColumnNames(resp.columns(), List.of("id", "_score")); assertColumnTypes(resp.columns(), List.of("integer", "double")); - List> values = getValuesList(resp); - - assertThat(values.size(), equalTo(6)); - - for (int i = 0; i < 6; i++) { - assertThat(values.get(0).get(1), equalTo(1.0)); - } + assertValues(resp.values(), List.of(List.of(1, 1.156558871269226), List.of(6, 0.9114001989364624))); } } - public void testScoringNonPushableFunctions() { - var query = """ - FROM test METADATA _score - | WHERE length(content) < 20 + public void testWhereMatchWithScoringDifferentSort() { + + var query = String.format(Locale.ROOT, """ + FROM test + METADATA _score + | WHERE %s | KEEP id, _score - | SORT _score DESC, id ASC - """; + | SORT id DESC + """, matchingClause); + ; try (var resp = run(query)) { assertColumnNames(resp.columns(), List.of("id", "_score")); assertColumnTypes(resp.columns(), List.of("integer", "double")); - List> values = getValuesList(resp); - assertThat(values.size(), equalTo(2)); + assertValues(resp.values(), List.of(List.of(6, 0.9114001989364624), List.of(1, 1.156558871269226))); + } + } - assertThat(values.get(0).get(0), equalTo(1)); - assertThat(values.get(1).get(0), equalTo(2)); + public void testWhereMatchWithScoringSortScore() { + var query = String.format(Locale.ROOT, """ + FROM test + METADATA _score + | WHERE %s + | KEEP id, _score + | SORT _score DESC + """, matchingClause); - assertThat((Double) values.get(0).get(1), is(1.0)); - assertThat((Double) values.get(1).get(1), is(1.0)); + try (var resp = run(query)) { + assertColumnNames(resp.columns(), List.of("id", "_score")); + assertColumnTypes(resp.columns(), List.of("integer", "double")); + assertValues(resp.values(), List.of(List.of(1, 1.156558871269226), List.of(6, 0.9114001989364624))); } } - public void testDisjunctionScoring() { - var query = """ - FROM test METADATA _score - | WHERE match(content, "fox") OR length(content) < 20 + public void testWhereMatchWithScoringNoSort() { + var query = String.format(Locale.ROOT, """ + FROM test + METADATA _score + | WHERE %s | KEEP id, _score - | SORT _score DESC, id ASC - """; + """, matchingClause); try (var resp = run(query)) { assertColumnNames(resp.columns(), List.of("id", "_score")); assertColumnTypes(resp.columns(), List.of("integer", "double")); - List> values = getValuesList(resp); - assertThat(values.size(), equalTo(3)); + assertValuesInAnyOrder(resp.values(), List.of(List.of(1, 1.156558871269226), List.of(6, 0.9114001989364624))); + } + } - assertThat(values.get(0).get(0), equalTo(1)); - assertThat(values.get(1).get(0), equalTo(6)); - assertThat(values.get(2).get(0), equalTo(2)); + public void testMatchAllScoring() { + var query = """ + FROM test + METADATA _score + | KEEP id, _score + """; - // Matches full text query and non pushable query - assertThat((Double) values.get(0).get(1), greaterThan(1.0)); - assertThat((Double) values.get(1).get(1), greaterThan(1.0)); - // Matches just non pushable query - assertThat((Double) values.get(2).get(1), equalTo(1.0)); - } + assertZeroScore(query); } - public void testConjunctionPushableScoring() { + public void testNonPushableFunctionsScoring() { var query = """ - FROM test METADATA _score - | WHERE match(content, "fox") AND id > 4 + FROM test + METADATA _score + | WHERE length(content) < 20 | KEEP id, _score - | SORT _score DESC, id ASC """; - try (var resp = run(query)) { - assertColumnNames(resp.columns(), List.of("id", "_score")); - assertColumnTypes(resp.columns(), List.of("integer", "double")); - List> values = getValuesList(resp); - assertThat(values.size(), equalTo(1)); + assertZeroScore(query); - assertThat(values.get(0).get(0), equalTo(6)); + query = """ + FROM test + METADATA _score + | WHERE length(content) < 20 OR id > 4 + | KEEP id, _score + """; - // Matches full text query and pushable query - assertThat((Double) values.get(0).get(1), greaterThan(1.0)); - } + assertZeroScore(query); + + query = """ + FROM test + METADATA _score + | WHERE length(content) < 20 AND id < 4 + | KEEP id, _score + """; + + assertZeroScore(query); } - public void testConjunctionNonPushableScoring() { + public void testPushableFunctionsScoring() { var query = """ - FROM test METADATA _score - | WHERE match(content, "fox") AND length(content) < 20 + FROM test + METADATA _score + | WHERE id > 4 | KEEP id, _score - | SORT _score DESC, id ASC + | SORT id ASC + """; + + assertZeroScore(query); + + query = """ + FROM test + METADATA _score + | WHERE id > 4 AND id < 7 + | KEEP id, _score + | SORT id ASC """; + assertZeroScore(query); + } + + private void assertZeroScore(String query) { try (var resp = run(query)) { - assertColumnNames(resp.columns(), List.of("id", "_score")); - assertColumnTypes(resp.columns(), List.of("integer", "double")); + List> values = EsqlTestUtils.getValuesList(resp.values()); + for (List value : values) { + assertThat((Double) value.get(1), equalTo(0.0)); + } + } + } + + public void testPushableAndFullTextFunctionsConjunctionScoring() { + var queryWithoutFilter = String.format(Locale.ROOT, """ + FROM test + METADATA _score + | WHERE %s + | KEEP id, _score + | SORT id ASC + """, matchingClause); + var query = String.format(Locale.ROOT, """ + FROM test + METADATA _score + | WHERE %s AND id > 4 + | KEEP id, _score + | SORT id ASC + """, matchingClause); + checkSameScores(queryWithoutFilter, query); + + query = String.format(Locale.ROOT, """ + FROM test + METADATA _score + | WHERE %s AND (id > 4 or id < 2) + | KEEP id, _score + | SORT id ASC + """, matchingClause); + queryWithoutFilter = String.format(Locale.ROOT, """ + FROM test + METADATA _score + | WHERE %s + | KEEP id, _score + | SORT id ASC + """, matchingClause); + checkSameScores(queryWithoutFilter, query); + } + + public void testDisjunctionScoring() { + var queryWithoutFilter = String.format(Locale.ROOT, """ + FROM test + METADATA _score + | WHERE %s OR length(content) < 20 + | KEEP id, _score + | SORT _score DESC, id ASC + """, matchingClause); + var query = String.format(Locale.ROOT, """ + FROM test + METADATA _score + | WHERE %s + | KEEP id, _score + | SORT _score DESC, id ASC + """, matchingClause); + + checkSameScores(queryWithoutFilter, query); + + try (var resp = run(queryWithoutFilter)) { List> values = getValuesList(resp); - assertThat(values.size(), equalTo(1)); + assertThat(values.size(), equalTo(3)); assertThat(values.get(0).get(0), equalTo(1)); + assertThat(values.get(1).get(0), equalTo(6)); + assertThat(values.get(2).get(0), equalTo(2)); - // Matches full text query and pushable query - assertThat((Double) values.get(0).get(1), greaterThan(1.0)); + // Matches full text query and non pushable query + assertThat((Double) values.get(0).get(1), greaterThan(0.0)); + assertThat((Double) values.get(1).get(1), greaterThan(0.0)); + // Matches just non pushable query + assertThat((Double) values.get(2).get(1), equalTo(0.0)); } } public void testDisjunctionScoringPushableFunctions() { - var query = """ + var query = String.format(Locale.ROOT, """ FROM test METADATA _score - | WHERE match(content, "fox") OR match(content, "quick") + | WHERE %s OR match(content, "quick") | KEEP id, _score | SORT _score DESC, id ASC - """; + """, matchingClause); try (var resp = run(query)) { - assertColumnNames(resp.columns(), List.of("id", "_score")); - assertColumnTypes(resp.columns(), List.of("integer", "double")); List> values = getValuesList(resp); assertThat(values.size(), equalTo(2)); @@ -165,16 +287,22 @@ public void testDisjunctionScoringPushableFunctions() { } public void testDisjunctionScoringMultipleNonPushableFunctions() { - var query = """ + var query = String.format(Locale.ROOT, """ FROM test METADATA _score - | WHERE match(content, "fox") OR length(content) < 20 AND id > 2 + | WHERE %s | KEEP id, _score | SORT _score DESC - """; + """, matchingClause); + var queryWithoutFilter = String.format(Locale.ROOT, """ + FROM test METADATA _score + | WHERE %s OR length(content) < 20 AND id > 2 + | KEEP id, _score + | SORT _score DESC + """, matchingClause); - try (var resp = run(query)) { - assertColumnNames(resp.columns(), List.of("id", "_score")); - assertColumnTypes(resp.columns(), List.of("integer", "double")); + checkSameScores(queryWithoutFilter, query); + + try (var resp = run(queryWithoutFilter)) { List> values = getValuesList(resp); assertThat(values.size(), equalTo(2)); @@ -182,57 +310,43 @@ public void testDisjunctionScoringMultipleNonPushableFunctions() { assertThat(values.get(1).get(0), equalTo(6)); // Matches the full text query and the two pushable query - assertThat((Double) values.get(0).get(1), greaterThan(2.0)); - assertThat((Double) values.get(0).get(1), lessThan(3.0)); + assertThat((Double) values.get(0).get(1), greaterThan(0.0)); // Matches just the match function - assertThat((Double) values.get(1).get(1), lessThan(2.0)); - assertThat((Double) values.get(1).get(1), greaterThan(1.0)); + assertThat((Double) values.get(1).get(1), lessThan(1.0)); + assertThat((Double) values.get(1).get(1), greaterThan(0.1)); } } public void testDisjunctionScoringWithNot() { - var query = """ + var query = String.format(Locale.ROOT, """ FROM test METADATA _score - | WHERE NOT(match(content, "dog")) OR length(content) > 50 + | WHERE NOT(%s) OR length(content) > 50 | KEEP id, _score | SORT _score DESC, id ASC - """; + """, matchingClause); try (var resp = run(query)) { - assertColumnNames(resp.columns(), List.of("id", "_score")); - assertColumnTypes(resp.columns(), List.of("integer", "double")); - List> values = getValuesList(resp); - assertThat(values.size(), equalTo(3)); - - assertThat(values.get(0).get(0), equalTo(1)); - assertThat(values.get(1).get(0), equalTo(4)); - assertThat(values.get(2).get(0), equalTo(5)); - - // Matches NOT gets 0.0 and default score is 1.0 - assertThat((Double) values.get(0).get(1), equalTo(1.0)); - assertThat((Double) values.get(1).get(1), equalTo(1.0)); - assertThat((Double) values.get(2).get(1), equalTo(1.0)); + // Matches NOT gets 0.0 + assertThat(getValuesList(resp), equalTo(List.of(List.of(2, 0.0), List.of(3, 0.0), List.of(4, 0.0), List.of(5, 0.0)))); } } - public void testScoringWithNoFullTextFunction() { - var query = """ - FROM test METADATA _score - | WHERE length(content) > 50 - | KEEP id, _score - | SORT _score DESC, id ASC - """; - + private void checkSameScores(String queryWithoutFilter, String query) { + Map expectedScores = new HashMap<>(); + try (var respWithoutFilter = run(queryWithoutFilter)) { + List> valuesList = EsqlTestUtils.getValuesList(respWithoutFilter); + for (List result : valuesList) { + expectedScores.put((Integer) result.get(0), (Double) result.get(1)); + } + } try (var resp = run(query)) { assertColumnNames(resp.columns(), List.of("id", "_score")); assertColumnTypes(resp.columns(), List.of("integer", "double")); - List> values = getValuesList(resp); - assertThat(values.size(), equalTo(1)); - - assertThat(values.get(0).get(0), equalTo(4)); - - // Non pushable query gets score of 0.0, summed with 1.0 coming from Lucene - assertThat((Double) values.get(0).get(1), equalTo(1.0)); + List> values = EsqlTestUtils.getValuesList(resp.values()); + for (List value : values) { + Double score = (Double) value.get(1); + assertThat(expectedScores.get((Integer) value.get(0)), equalTo(score)); + } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index aa30f2710957d..a70b2433ed00e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -729,7 +729,12 @@ public enum Cap { /** * Use double parameter markers to represent field or function names. */ - DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS(Build.current().isSnapshot()); + DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS(Build.current().isSnapshot()), + + /** + * Non full text functions do not contribute to score + */ + NON_FULL_TEXT_FUNCTIONS_SCORING; private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryBuilderResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryBuilderResolver.java index 14607de433630..09f8bed4f150a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryBuilderResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryBuilderResolver.java @@ -76,7 +76,7 @@ public FullTextFunctionsRewritable rewrite(QueryRewriteContext ctx) throws IOExc Holder updated = new Holder<>(false); LogicalPlan newPlan = plan.transformExpressionsDown(FullTextFunction.class, f -> { QueryBuilder builder = f.queryBuilder(), initial = builder; - builder = builder == null ? f.asQuery(TranslatorHandler.TRANSLATOR_HANDLER).asBuilder() : builder; + builder = builder == null ? f.asQuery(TranslatorHandler.TRANSLATOR_HANDLER).toQueryBuilder() : builder; try { builder = builder.rewrite(ctx); } catch (IOException e) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java index 1da28b3069675..61fcc273b9c33 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java @@ -141,7 +141,8 @@ protected TypeResolutions.ParamOrdinal queryParamOrdinal() { @Override protected Query translate(TranslatorHandler handler) { - return new TermQuery(source(), ((FieldAttribute) field()).name(), queryAsObject()); + // Uses a term query that contributes to scoring + return new TermQuery(source(), ((FieldAttribute) field()).name(), queryAsObject(), false, true); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java index f902f261e7dc9..67dc69efb2bc7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java @@ -100,7 +100,7 @@ private static PhysicalPlan rewrite( List newPushable = combineEligiblePushableToRange(pushable); if (newPushable.size() > 0) { // update the executable with pushable conditions Query queryDSL = TRANSLATOR_HANDLER.asQuery(Predicates.combineAnd(newPushable)); - QueryBuilder planQuery = queryDSL.asBuilder(); + QueryBuilder planQuery = queryDSL.toQueryBuilder(); Queries.Clause combiningQueryClauseType = queryExec.hasScoring() ? Queries.Clause.MUST : Queries.Clause.FILTER; var query = Queries.combine(combiningQueryClauseType, asList(queryExec.query(), planQuery)); queryExec = new EsQueryExec( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushStatsToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushStatsToSource.java index adc6145ce2574..1e4c12ca5a26c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushStatsToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushStatsToSource.java @@ -107,7 +107,7 @@ private Tuple, List> pushableStats( return null; // can't push down } var countFilter = TRANSLATOR_HANDLER.asQuery(count.filter()); - query = Queries.combine(Queries.Clause.MUST, asList(countFilter.asBuilder(), query)); + query = Queries.combine(Queries.Clause.MUST, asList(countFilter.toQueryBuilder(), query)); } return new EsStatsQueryExec.Stat(fieldName, COUNT, query); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsStatsQueryExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsStatsQueryExec.java index 5519e7fbc7083..397c3ade17a64 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsStatsQueryExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EsStatsQueryExec.java @@ -38,7 +38,7 @@ public enum StatsType { public record Stat(String name, StatsType type, QueryBuilder query) { public QueryBuilder filter(QueryBuilder sourceQuery) { - return query == null ? sourceQuery : Queries.combine(Queries.Clause.FILTER, asList(sourceQuery, query)); + return query == null ? sourceQuery : Queries.combine(Queries.Clause.FILTER, asList(sourceQuery, query)).boost(0.0f); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 761b526b8bb0f..05f748598838f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -149,7 +149,7 @@ private MultiTypeEsField findUnionTypes(Attribute attr) { } public Function querySupplier(QueryBuilder builder) { - QueryBuilder qb = builder == null ? QueryBuilders.matchAllQuery() : builder; + QueryBuilder qb = builder == null ? QueryBuilders.matchAllQuery().boost(0.0f) : builder; return ctx -> shardContexts.get(ctx.index()).toQuery(qb); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index a5586a8f03b4b..c3cc9e3669de1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -220,7 +220,7 @@ static QueryBuilder detectFilter(PhysicalPlan plan, Predicate fieldName) } } if (matches.isEmpty() == false) { - requestFilters.add(TRANSLATOR_HANDLER.asQuery(Predicates.combineAnd(matches)).asBuilder()); + requestFilters.add(TRANSLATOR_HANDLER.asQuery(Predicates.combineAnd(matches)).toQueryBuilder()); } }); }); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java index c388a131b9ab6..7ef272f84ec9f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java @@ -43,7 +43,7 @@ public KqlQuery(Source source, String query, Map options) { } @Override - public QueryBuilder asBuilder() { + protected QueryBuilder asBuilder() { final KqlQueryBuilder queryBuilder = new KqlQueryBuilder(query); options.forEach((k, v) -> { if (BUILDER_APPLIERS.containsKey(k)) { @@ -82,4 +82,9 @@ public boolean equals(Object obj) { protected String innerToString() { return query; } + + @Override + public boolean scorable() { + return true; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchQuery.java index 3c40db1670c15..57489cc930bf2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchQuery.java @@ -72,7 +72,7 @@ public MatchQuery(Source source, String name, Object text, Map o } @Override - public QueryBuilder asBuilder() { + protected QueryBuilder asBuilder() { final MatchQueryBuilder queryBuilder = QueryBuilders.matchQuery(name, text); options.forEach((k, v) -> { if (BUILDER_APPLIERS.containsKey(k)) { @@ -125,4 +125,9 @@ protected String innerToString() { public Map options() { return options; } + + @Override + public boolean scorable() { + return true; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MultiMatchQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MultiMatchQuery.java index 84524bad29e08..33042bd5efe4e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MultiMatchQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MultiMatchQuery.java @@ -61,7 +61,7 @@ public MultiMatchQuery(Source source, String query, Map fields, M } @Override - public QueryBuilder asBuilder() { + protected QueryBuilder asBuilder() { final MultiMatchQueryBuilder queryBuilder = QueryBuilders.multiMatchQuery(query); queryBuilder.fields(fields); queryBuilder.analyzer(predicate.analyzer()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java index a0a9d36c11000..de1149c1b6003 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java @@ -65,8 +65,8 @@ public SingleValueQuery(Query next, String field) { } @Override - public Builder asBuilder() { - return new Builder(next.asBuilder(), field, next.source()); + protected Builder asBuilder() { + return new Builder(next.toQueryBuilder(), field, next.source()); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java index 532825290af0d..bf0fa72d3af41 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java @@ -46,7 +46,7 @@ public SpatialRelatesQuery(Source source, String field, ShapeRelation queryRelat } @Override - public QueryBuilder asBuilder() { + protected QueryBuilder asBuilder() { return DataType.isSpatialGeo(dataType) ? new GeoShapeQueryBuilder() : new CartesianShapeQueryBuilder(); } @@ -94,6 +94,8 @@ public ShapeRelation shapeRelation() { */ public abstract class ShapeQueryBuilder implements QueryBuilder { + private float boost = 0.0f; + protected void doToXContent(String queryName, XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.startObject(queryName); @@ -139,12 +141,13 @@ public String queryName() { @Override public float boost() { - return 0; + return boost; } @Override public QueryBuilder boost(float boost) { - throw new UnsupportedOperationException("Unimplemented: float"); + this.boost = boost; + return this; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/TranslationAwareExpressionQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/TranslationAwareExpressionQuery.java index e92f3c899f581..d61101c2f594c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/TranslationAwareExpressionQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/TranslationAwareExpressionQuery.java @@ -26,7 +26,7 @@ public TranslationAwareExpressionQuery(Source source, QueryBuilder queryBuilder) } @Override - public QueryBuilder asBuilder() { + protected QueryBuilder asBuilder() { return queryBuilder; } @@ -34,4 +34,10 @@ public QueryBuilder asBuilder() { protected String innerToString() { return queryBuilder.toString(); } + + @Override + public boolean scorable() { + // All Full Text Functions are translated to queries using this method + return true; + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchErrorTests.java index c63e0bd6486b5..a1f4792094615 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchErrorTests.java @@ -40,7 +40,7 @@ protected Expression build(Source source, List args) { // We need to add the QueryBuilder to the match expression, as it is used to implement equals() and hashCode() and // thus test the serialization methods. But we can only do this if the parameters make sense . if (args.get(0) instanceof FieldAttribute && args.get(1).foldable()) { - QueryBuilder queryBuilder = TRANSLATOR_HANDLER.asQuery(match).asBuilder(); + QueryBuilder queryBuilder = TRANSLATOR_HANDLER.asQuery(match).toQueryBuilder(); match.replaceQueryBuilder(queryBuilder); } return match; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java index f4a4877e3317c..f790e120786dc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java @@ -80,7 +80,7 @@ protected Expression build(Source source, List args) { // We need to add the QueryBuilder to the match expression, as it is used to implement equals() and hashCode() and // thus test the serialization methods. But we can only do this if the parameters make sense . if (args.get(0) instanceof FieldAttribute && args.get(1).foldable()) { - QueryBuilder queryBuilder = TRANSLATOR_HANDLER.asQuery(match).asBuilder(); + QueryBuilder queryBuilder = TRANSLATOR_HANDLER.asQuery(match).toQueryBuilder(); match.replaceQueryBuilder(queryBuilder); } return match; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index cdbbb9d6ceddc..faf2b7c08cc0c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -91,11 +91,18 @@ import static java.util.Arrays.asList; import static org.elasticsearch.compute.aggregation.AggregatorMode.FINAL; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.existsQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.index.query.QueryBuilders.termsQuery; import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; +import static org.elasticsearch.xpack.esql.core.querydsl.query.Query.unscore; import static org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec.StatsType; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; @@ -232,7 +239,7 @@ public void testCountFieldWithFilter() { var plan = plannerOptimizer.plan("from test | where emp_no > 10040 | stats c = count(emp_no)", IS_SV_STATS); var stat = queryStatsFor(plan); assertThat(stat.type(), is(StatsType.COUNT)); - assertThat(stat.query(), is(QueryBuilders.existsQuery("emp_no"))); + assertThat(stat.query(), is(existsQuery("emp_no"))); } /** @@ -260,7 +267,7 @@ public void testCountFieldWithEval() { assertThat(esStatsQuery.limit(), is(nullValue())); assertThat(Expressions.names(esStatsQuery.output()), contains("$$c$count", "$$c$seen")); var stat = as(esStatsQuery.stats().get(0), Stat.class); - assertThat(stat.query(), is(QueryBuilders.existsQuery("salary"))); + assertThat(stat.query(), is(existsQuery("salary"))); } // optimized doesn't know yet how to push down count over field @@ -282,11 +289,11 @@ public void testCountOneFieldWithFilter() { assertThat(Expressions.names(esStatsQuery.output()), contains("$$c$count", "$$c$seen")); var stat = as(esStatsQuery.stats().get(0), Stat.class); Source source = new Source(2, 8, "salary > 1000"); - var exists = QueryBuilders.existsQuery("salary"); + var exists = existsQuery("salary"); assertThat(stat.query(), is(exists)); - var range = wrapWithSingleQuery(query, QueryBuilders.rangeQuery("salary").gt(1000), "salary", source); - var expected = QueryBuilders.boolQuery().must(range).must(exists); - assertThat(expected.toString(), is(esStatsQuery.query().toString())); + var range = wrapWithSingleQuery(query, unscore(rangeQuery("salary").gt(1000)), "salary", source); + var expected = boolQuery().must(range).must(unscore(exists)); + assertThat(esStatsQuery.query().toString(), is(expected.toString())); } // optimized doesn't know yet how to push down count over field @@ -391,7 +398,7 @@ public void testAnotherCountAllWithFilter() { assertThat(esStatsQuery.limit(), is(nullValue())); assertThat(Expressions.names(esStatsQuery.output()), contains("$$c$count", "$$c$seen")); var source = ((SingleValueQuery.Builder) esStatsQuery.query()).source(); - var expected = wrapWithSingleQuery(query, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", source); + var expected = wrapWithSingleQuery(query, unscore(rangeQuery("emp_no").gt(10010)), "emp_no", source); assertThat(expected.toString(), is(esStatsQuery.query().toString())); } @@ -442,7 +449,7 @@ protected List> batches() { { "exists": { "field": "hire_date", - "boost": 1.0 + "boost": 0.0 } }, { @@ -452,7 +459,7 @@ protected List> batches() { "range": { "emp_no": { "lt": 10042, - "boost": 1.0 + "boost": 0.0 } } }, @@ -519,9 +526,9 @@ public void testQueryStringFunctionConjunctionWhereOperands() { assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(2, 37, "emp_no > 10000"); - var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); + var range = wrapWithSingleQuery(queryText, unscore(rangeQuery("emp_no").gt(10010)), "emp_no", filterSource); var queryString = QueryBuilders.queryStringQuery("last_name: Smith"); - var expected = QueryBuilders.boolQuery().must(queryString).must(range); + var expected = boolQuery().must(queryString).must(range); assertThat(query.query().toString(), is(expected.toString())); } @@ -554,9 +561,9 @@ public void testQueryStringFunctionWithFunctionsPushedToLucene() { assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(2, 37, "cidr_match(ip, \"127.0.0.1/32\")"); - var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); + var terms = wrapWithSingleQuery(queryText, unscore(termsQuery("ip", "127.0.0.1/32")), "ip", filterSource); var queryString = QueryBuilders.queryStringQuery("last_name: Smith"); - var expected = QueryBuilders.boolQuery().must(queryString).must(terms); + var expected = boolQuery().must(queryString).must(terms); assertThat(query.query().toString(), is(expected.toString())); } @@ -589,9 +596,9 @@ public void testQueryStringFunctionMultipleWhereClauses() { assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(3, 8, "emp_no > 10000"); - var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); + var range = wrapWithSingleQuery(queryText, unscore(rangeQuery("emp_no").gt(10010)), "emp_no", filterSource); var queryString = QueryBuilders.queryStringQuery("last_name: Smith"); - var expected = QueryBuilders.boolQuery().must(queryString).must(range); + var expected = boolQuery().must(queryString).must(range); assertThat(query.query().toString(), is(expected.toString())); } @@ -623,7 +630,7 @@ public void testQueryStringFunctionMultipleQstrClauses() { var queryStringLeft = QueryBuilders.queryStringQuery("last_name: Smith"); var queryStringRight = QueryBuilders.queryStringQuery("emp_no: [10010 TO *]"); - var expected = QueryBuilders.boolQuery().must(queryStringLeft).must(queryStringRight); + var expected = boolQuery().must(queryStringLeft).must(queryStringRight); assertThat(query.query().toString(), is(expected.toString())); } @@ -680,9 +687,9 @@ public void testMatchFunctionConjunctionWhereOperands() { assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(2, 38, "emp_no > 10000"); - var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); + var range = wrapWithSingleQuery(queryText, unscore(rangeQuery("emp_no").gt(10010)), "emp_no", filterSource); var queryString = QueryBuilders.matchQuery("last_name", "Smith").lenient(true); - var expected = QueryBuilders.boolQuery().must(queryString).must(range); + var expected = boolQuery().must(queryString).must(range); assertThat(query.query().toString(), is(expected.toString())); } @@ -715,9 +722,9 @@ public void testMatchFunctionWithFunctionsPushedToLucene() { assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(2, 32, "cidr_match(ip, \"127.0.0.1/32\")"); - var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); + var terms = wrapWithSingleQuery(queryText, unscore(termsQuery("ip", "127.0.0.1/32")), "ip", filterSource); var queryString = QueryBuilders.matchQuery("text", "beta").lenient(true); - var expected = QueryBuilders.boolQuery().must(queryString).must(terms); + var expected = boolQuery().must(queryString).must(terms); assertThat(query.query().toString(), is(expected.toString())); } @@ -749,9 +756,9 @@ public void testMatchFunctionMultipleWhereClauses() { assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(3, 8, "emp_no > 10000"); - var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); + var range = wrapWithSingleQuery(queryText, unscore(rangeQuery("emp_no").gt(10010)), "emp_no", filterSource); var queryString = QueryBuilders.matchQuery("last_name", "Smith").lenient(true); - var expected = QueryBuilders.boolQuery().must(queryString).must(range); + var expected = boolQuery().must(queryString).must(range); assertThat(query.query().toString(), is(expected.toString())); } @@ -782,7 +789,7 @@ public void testMatchFunctionMultipleMatchClauses() { var queryStringLeft = QueryBuilders.matchQuery("last_name", "Smith").lenient(true); var queryStringRight = QueryBuilders.matchQuery("first_name", "John").lenient(true); - var expected = QueryBuilders.boolQuery().must(queryStringLeft).must(queryStringRight); + var expected = boolQuery().must(queryStringLeft).must(queryStringRight); assertThat(query.query().toString(), is(expected.toString())); } @@ -839,9 +846,9 @@ public void testKqlFunctionConjunctionWhereOperands() { assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(2, 36, "emp_no > 10000"); - var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); + var range = wrapWithSingleQuery(queryText, unscore(rangeQuery("emp_no").gt(10010)), "emp_no", filterSource); var kqlQuery = kqlQueryBuilder("last_name: Smith"); - var expected = QueryBuilders.boolQuery().must(kqlQuery).must(range); + var expected = boolQuery().must(kqlQuery).must(range); assertThat(query.query().toString(), is(expected.toString())); } @@ -874,9 +881,9 @@ public void testKqlFunctionWithFunctionsPushedToLucene() { assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(2, 36, "cidr_match(ip, \"127.0.0.1/32\")"); - var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); + var terms = wrapWithSingleQuery(queryText, unscore(termsQuery("ip", "127.0.0.1/32")), "ip", filterSource); var kqlQuery = kqlQueryBuilder("last_name: Smith"); - var expected = QueryBuilders.boolQuery().must(kqlQuery).must(terms); + var expected = boolQuery().must(kqlQuery).must(terms); assertThat(query.query().toString(), is(expected.toString())); } @@ -909,9 +916,9 @@ public void testKqlFunctionMultipleWhereClauses() { assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(3, 8, "emp_no > 10000"); - var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); + var range = wrapWithSingleQuery(queryText, unscore(rangeQuery("emp_no").gt(10010)), "emp_no", filterSource); var kqlQuery = kqlQueryBuilder("last_name: Smith"); - var expected = QueryBuilders.boolQuery().must(kqlQuery).must(range); + var expected = boolQuery().must(kqlQuery).must(range); assertThat(query.query().toString(), is(expected.toString())); } @@ -943,7 +950,7 @@ public void testKqlFunctionMultipleKqlClauses() { var kqlQueryLeft = kqlQueryBuilder("last_name: Smith"); var kqlQueryRight = kqlQueryBuilder("emp_no > 10010"); - var expected = QueryBuilders.boolQuery().must(kqlQueryLeft).must(kqlQueryRight); + var expected = boolQuery().must(kqlQueryLeft).must(kqlQueryRight); assertThat(query.query().toString(), is(expected.toString())); } @@ -1006,7 +1013,7 @@ public void testIsNotNullPushdownFilter() { var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); assertThat(as(query.limit(), Literal.class).value(), is(1000)); - var expected = QueryBuilders.existsQuery("emp_no"); + var expected = unscore(existsQuery("emp_no")); assertThat(query.query().toString(), is(expected.toString())); } @@ -1030,7 +1037,7 @@ public void testIsNullPushdownFilter() { var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); assertThat(as(query.limit(), Literal.class).value(), is(1000)); - var expected = QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("emp_no")); + var expected = boolQuery().mustNot(unscore(existsQuery("emp_no"))); assertThat(query.query().toString(), is(expected.toString())); } @@ -1056,7 +1063,7 @@ public void testIsNotNull_TextField_Pushdown() { var partialAgg = as(exchange.child(), AggregateExec.class); var fieldExtract = as(partialAgg.child(), FieldExtractExec.class); var query = as(fieldExtract.child(), EsQueryExec.class); - var expected = QueryBuilders.existsQuery(textField); + var expected = unscore(existsQuery(textField)); assertThat(query.query().toString(), is(expected.toString())); } @@ -1079,7 +1086,7 @@ public void testIsNull_TextField_Pushdown() { var project = as(exchange.child(), ProjectExec.class); var fieldExtract = as(project.child(), FieldExtractExec.class); var query = as(fieldExtract.child(), EsQueryExec.class); - var expected = QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(textField)); + var expected = boolQuery().mustNot(unscore(existsQuery(textField))); assertThat(query.query().toString(), is(expected.toString())); } @@ -1142,7 +1149,7 @@ public void testIsNotNull_TextField_Pushdown_WithCount() { assertThat(esStatsQuery.limit(), is(nullValue())); assertThat(Expressions.names(esStatsQuery.output()), contains("$$c$count", "$$c$seen")); var stat = as(esStatsQuery.stats().get(0), Stat.class); - assertThat(stat.query(), is(QueryBuilders.existsQuery("job"))); + assertThat(stat.query(), is(existsQuery("job"))); } private record OutOfRangeTestCase(String fieldName, String tooLow, String tooHigh) {}; @@ -1206,7 +1213,7 @@ public void testOutOfRangeFilterPushdown() { assertThat(actualLuceneQuery.field(), equalTo(testCase.fieldName)); assertThat(actualLuceneQuery.source(), equalTo(expectedSource)); - assertThat(actualLuceneQuery.next(), equalTo(QueryBuilders.matchAllQuery())); + assertThat(actualLuceneQuery.next(), equalTo(unscore(matchAllQuery()))); } for (String falsePredicate : alwaysFalsePredicates) { @@ -1221,7 +1228,7 @@ public void testOutOfRangeFilterPushdown() { assertThat(actualLuceneQuery.field(), equalTo(testCase.fieldName)); assertThat(actualLuceneQuery.source(), equalTo(expectedSource)); - var expectedInnerQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.matchAllQuery()); + var expectedInnerQuery = unscore(boolQuery().mustNot(unscore(matchAllQuery()))); assertThat(actualLuceneQuery.next(), equalTo(expectedInnerQuery)); } } @@ -1487,7 +1494,7 @@ public void testMultipleMatchFilterPushdown() { Source filterSource = new Source(4, 8, "emp_no > 10000"); var expectedLuceneQuery = new BoolQueryBuilder().must(new MatchQueryBuilder("first_name", "Anna").lenient(true)) .must(new MatchQueryBuilder("first_name", "Anneke").lenient(true)) - .must(wrapWithSingleQuery(query, QueryBuilders.rangeQuery("emp_no").gt(10000), "emp_no", filterSource)) + .must(wrapWithSingleQuery(query, unscore(rangeQuery("emp_no").gt(10000)), "emp_no", filterSource)) .must(new MatchQueryBuilder("last_name", "Xinglin").lenient(true)); assertThat(actualLuceneQuery.toString(), is(expectedLuceneQuery.toString())); } @@ -1587,7 +1594,7 @@ public void testTermFunction() { var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); assertThat(as(query.limit(), Literal.class).value(), is(1000)); - var expected = QueryBuilders.termQuery("last_name", "Smith"); + var expected = termQuery("last_name", "Smith"); assertThat(query.query().toString(), is(expected.toString())); } @@ -1650,7 +1657,7 @@ public void testMatchFunctionWithPushableConjunction() { var esQuery = as(fieldExtract.child(), EsQueryExec.class); Source source = new Source(2, 38, "salary > 10000"); BoolQueryBuilder expected = new BoolQueryBuilder().must(new MatchQueryBuilder("last_name", "Smith").lenient(true)) - .must(wrapWithSingleQuery(query, QueryBuilders.rangeQuery("salary").gt(10000), "salary", source)); + .must(wrapWithSingleQuery(query, unscore(rangeQuery("salary").gt(10000)), "salary", source)); assertThat(esQuery.query().toString(), equalTo(expected.toString())); } @@ -1687,7 +1694,7 @@ public void testMatchFunctionWithPushableDisjunction() { var esQuery = as(fieldExtract.child(), EsQueryExec.class); Source source = new Source(2, 37, "emp_no > 10"); BoolQueryBuilder expected = new BoolQueryBuilder().should(new MatchQueryBuilder("last_name", "Smith").lenient(true)) - .should(wrapWithSingleQuery(query, QueryBuilders.rangeQuery("emp_no").gt(10), "emp_no", source)); + .should(wrapWithSingleQuery(query, unscore(rangeQuery("emp_no").gt(10)), "emp_no", source)); assertThat(esQuery.query().toString(), equalTo(expected.toString())); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 3e417440c4fed..f4642213979c8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -174,6 +174,7 @@ import static org.elasticsearch.xpack.esql.core.expression.Expressions.name; import static org.elasticsearch.xpack.esql.core.expression.Expressions.names; import static org.elasticsearch.xpack.esql.core.expression.function.scalar.FunctionTestUtils.l; +import static org.elasticsearch.xpack.esql.core.querydsl.query.Query.unscore; import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT; import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE; import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; @@ -866,7 +867,7 @@ public void testQueryWithAggregation() { var query = source(extract.child()); assertThat(query.estimatedRowSize(), equalTo(Integer.BYTES * 2 /* for doc id, emp_no*/)); - assertThat(query.query(), is(existsQuery("emp_no"))); + assertThat(query.query(), is(unscore(existsQuery("emp_no")))); } /** @@ -901,7 +902,7 @@ public void testQueryWithAggAfterEval() { var query = source(extract.child()); assertThat(query.estimatedRowSize(), equalTo(Integer.BYTES * 2 /* for doc id, emp_no*/)); - assertThat(query.query(), is(existsQuery("emp_no"))); + assertThat(query.query(), is(unscore(existsQuery("emp_no")))); } public void testQueryForStatWithMultiAgg() { @@ -922,7 +923,7 @@ public void testQueryForStatWithMultiAgg() { var query = source(extract.child()); assertThat(query.estimatedRowSize(), equalTo(Integer.BYTES * 3 /* for doc id, emp_no, salary*/)); - assertThat(query.query(), is(boolQuery().should(existsQuery("emp_no")).should(existsQuery("salary")))); + assertThat(query.query(), is(boolQuery().should(unscore(existsQuery("emp_no"))).should(unscore(existsQuery("salary"))))); } /** @@ -1947,7 +1948,8 @@ public void testPushDownEqualsToUpper() { "term" : { "first_name" : { "value" : "FOO", - "case_insensitive" : true + "case_insensitive" : true, + "boost": 0.0 } } }, @@ -1991,7 +1993,8 @@ public void testPushDownEqualsToLower() { "term" : { "first_name" : { "value" : "foo", - "case_insensitive" : true + "case_insensitive" : true, + "boost": 0.0 } } }, @@ -2015,12 +2018,13 @@ public void testPushDownNotEqualsToUpper() { "term" : { "first_name" : { "value" : "FOO", - "case_insensitive" : true + "case_insensitive" : true, + "boost": 0.0 } } } ], - "boost" : 1.0 + "boost": 0.0 } }, "source" : "to_upper(first_name) != \\"FOO\\"@2:9" @@ -2043,12 +2047,13 @@ public void testPushDownNotEqualsToLower() { "term" : { "first_name" : { "value" : "foo", - "case_insensitive" : true + "case_insensitive" : true, + "boost": 0.0 } } } ], - "boost" : 1.0 + "boost" : 0.0 } }, "source" : "to_lower(first_name) != \\"foo\\"@2:9" @@ -2074,12 +2079,13 @@ public void testPushDownChangeCaseMultiplePredicates() { "term" : { "first_name" : { "value" : "foo", - "case_insensitive" : true + "case_insensitive" : true, + "boost": 0.0 } } } ], - "boost" : 1.0 + "boost": 0.0 } }, "source" : "to_lower(first_name) != \\"foo\\"@2:9" @@ -2092,7 +2098,8 @@ public void testPushDownChangeCaseMultiplePredicates() { "term" : { "first_name" : { "value" : "FOO", - "case_insensitive" : true + "case_insensitive" : true, + "boost": 0.0 } } }, @@ -2106,7 +2113,7 @@ public void testPushDownChangeCaseMultiplePredicates() { "range" : { "emp_no" : { "gt" : 10, - "boost" : 1.0 + "boost" : 0.0 } } }, @@ -2144,7 +2151,8 @@ public void testPushDownChangeCaseThroughEval() { "term" : { "first_name" : { "value" : "foo", - "case_insensitive" : true + "case_insensitive" : true, + "boost" : 0.0 } } }, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java index 90f25db232ec7..4c1b009e847ed 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java @@ -54,6 +54,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.SerializationTestUtils.assertSerialization; +import static org.elasticsearch.xpack.esql.core.querydsl.query.Query.unscore; import static org.elasticsearch.xpack.esql.core.util.Queries.Clause.FILTER; import static org.elasticsearch.xpack.esql.core.util.Queries.Clause.MUST; import static org.elasticsearch.xpack.esql.core.util.Queries.Clause.SHOULD; @@ -110,7 +111,7 @@ public void testTimestampNoRequestFilterQueryFilter() { var plan = plan(query, null); var filter = filterQueryForTransportNodes(plan); - var expected = singleValueQuery(query, rangeQuery(EMP_NO).gt(value), EMP_NO, ((SingleValueQuery.Builder) filter).source()); + var expected = singleValueQuery(query, unscore(rangeQuery(EMP_NO).gt(value)), EMP_NO, ((SingleValueQuery.Builder) filter).source()); assertEquals(expected.toString(), filter.toString()); } @@ -128,7 +129,7 @@ public void testTimestampRequestFilterQueryFilter() { var builder = ((BoolQueryBuilder) filter).filter().get(1); var queryFilter = singleValueQuery( query, - rangeQuery(EMP_NO).gt(value).includeUpper(false), + unscore(rangeQuery(EMP_NO).gt(value).includeUpper(false)), EMP_NO, ((SingleValueQuery.Builder) builder).source() ); @@ -149,8 +150,18 @@ public void testTimestampRequestFilterQueryFilterWithConjunction() { var filter = filterQueryForTransportNodes(plan); var musts = ((BoolQueryBuilder) ((BoolQueryBuilder) filter).filter().get(1)).must(); - var left = singleValueQuery(query, rangeQuery(EMP_NO).gt(lowValue), EMP_NO, ((SingleValueQuery.Builder) musts.get(0)).source()); - var right = singleValueQuery(query, rangeQuery(EMP_NO).lt(highValue), EMP_NO, ((SingleValueQuery.Builder) musts.get(1)).source()); + var left = singleValueQuery( + query, + unscore(rangeQuery(EMP_NO).gt(lowValue)), + EMP_NO, + ((SingleValueQuery.Builder) musts.get(0)).source() + ); + var right = singleValueQuery( + query, + unscore(rangeQuery(EMP_NO).lt(highValue)), + EMP_NO, + ((SingleValueQuery.Builder) musts.get(1)).source() + ); var must = Queries.combine(MUST, asList(left, right)); var expected = Queries.combine(FILTER, asList(restFilter, must)); assertEquals(expected.toString(), filter.toString()); @@ -184,8 +195,18 @@ public void testTimestampRequestFilterQueryFilterWithDisjunctionOnSameField() { var filter = filterQueryForTransportNodes(plan); var shoulds = ((BoolQueryBuilder) ((BoolQueryBuilder) filter).filter().get(1)).should(); - var left = singleValueQuery(query, rangeQuery(EMP_NO).gt(lowValue), EMP_NO, ((SingleValueQuery.Builder) shoulds.get(0)).source()); - var right = singleValueQuery(query, rangeQuery(EMP_NO).lt(highValue), EMP_NO, ((SingleValueQuery.Builder) shoulds.get(1)).source()); + var left = singleValueQuery( + query, + unscore(rangeQuery(EMP_NO).gt(lowValue)), + EMP_NO, + ((SingleValueQuery.Builder) shoulds.get(0)).source() + ); + var right = singleValueQuery( + query, + unscore(rangeQuery(EMP_NO).lt(highValue)), + EMP_NO, + ((SingleValueQuery.Builder) shoulds.get(1)).source() + ); var should = Queries.combine(SHOULD, asList(left, right)); var expected = Queries.combine(FILTER, asList(restFilter, should)); assertEquals(expected.toString(), filter.toString()); @@ -205,8 +226,18 @@ public void testTimestampRequestFilterQueryFilterWithMultiConjunction() { var filter = filterQueryForTransportNodes(plan); var musts = ((BoolQueryBuilder) ((BoolQueryBuilder) filter).filter().get(1)).must(); - var left = singleValueQuery(query, rangeQuery(EMP_NO).gt(lowValue), EMP_NO, ((SingleValueQuery.Builder) musts.get(0)).source()); - var right = singleValueQuery(query, rangeQuery(EMP_NO).lt(highValue), EMP_NO, ((SingleValueQuery.Builder) musts.get(1)).source()); + var left = singleValueQuery( + query, + unscore(rangeQuery(EMP_NO).gt(lowValue)), + EMP_NO, + ((SingleValueQuery.Builder) musts.get(0)).source() + ); + var right = singleValueQuery( + query, + unscore(rangeQuery(EMP_NO).lt(highValue)), + EMP_NO, + ((SingleValueQuery.Builder) musts.get(1)).source() + ); var must = Queries.combine(MUST, asList(left, right)); var expected = Queries.combine(FILTER, asList(restFilter, must)); assertEquals(expected.toString(), filter.toString()); @@ -229,7 +260,12 @@ public void testTimestampRequestFilterQueryMultipleFilters() { var filter = filterQueryForTransportNodes(plan); var builder = ((BoolQueryBuilder) filter).filter().get(1); - var queryFilter = singleValueQuery(query, rangeQuery(EMP_NO).gt(lowValue), EMP_NO, ((SingleValueQuery.Builder) builder).source()); + var queryFilter = singleValueQuery( + query, + unscore(rangeQuery(EMP_NO).gt(lowValue)), + EMP_NO, + ((SingleValueQuery.Builder) builder).source() + ); var expected = Queries.combine(FILTER, asList(restFilter, queryFilter)); assertEquals(expected.toString(), filter.toString()); } @@ -314,7 +350,7 @@ private PhysicalPlan plan(String query, QueryBuilder restFilter) { } private QueryBuilder restFilterQuery(String field) { - return rangeQuery(field).lt("2020-12-34"); + return unscore(rangeQuery(field).lt("2020-12-34")); } private QueryBuilder filterQueryForTransportNodes(PhysicalPlan plan) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java index f9732272dbd74..64f073310d3e6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java @@ -99,11 +99,11 @@ public void testBinaryComparisons() { assertQueryTranslation(""" FROM test | WHERE "1.2.3" == version""", containsString(""" - "esql_single_value":{"field":"version","next":{"term":{"version":{"value":"1.2.3"}""")); + "esql_single_value":{"field":"version","next":{"term":{"version":{"value":"1.2.3","boost":0.0}""")); assertQueryTranslation(""" FROM test | WHERE "foo" == keyword""", containsString(""" - "esql_single_value":{"field":"keyword","next":{"term":{"keyword":{"value":"foo"}""")); + "esql_single_value":{"field":"keyword","next":{"term":{"keyword":{"value":"foo","boost":0.0}""")); assertQueryTranslation(""" FROM test | WHERE "2007-12-03T10:15:30+01:00" == date""", containsString(""" @@ -111,7 +111,7 @@ public void testBinaryComparisons() { assertQueryTranslation(""" FROM test | WHERE ip != "127.0.0.1\"""", containsString(""" - "esql_single_value":{"field":"ip","next":{"bool":{"must_not":[{"term":{"ip":{"value":"127.0.0.1"}""")); + "esql_single_value":{"field":"ip","next":{"bool":{"must_not":[{"term":{"ip":{"value":"127.0.0.1","boost":0.0}}""")); } public void testRanges() { @@ -195,28 +195,28 @@ public void testRanges() { assertQueryTranslation(""" FROM test | WHERE "2007-12-03T10:15:30Z" <= date AND date <= "2024-01-01T10:15:30\"""", containsString(""" "esql_single_value":{"field":"date","next":{"range":{"date":{"gte":"2007-12-03T10:15:30.000Z","lte":"2024-01-01T10:15:30.000Z",\ - "time_zone":"Z","format":"strict_date_optional_time","boost":1.0}}}""")); + "time_zone":"Z","format":"strict_date_optional_time","boost":0.0}}}""")); assertQueryTranslation(""" FROM test | WHERE "2007-12-03T10:15:30" <= date AND date <= "2024-01-01T10:15:30Z\"""", containsString(""" "esql_single_value":{"field":"date","next":{"range":{"date":{"gte":"2007-12-03T10:15:30.000Z","lte":"2024-01-01T10:15:30.000Z",\ - "time_zone":"Z","format":"strict_date_optional_time","boost":1.0}}}""")); + "time_zone":"Z","format":"strict_date_optional_time","boost":0.0}}}""")); // various timezones assertQueryTranslation(""" FROM test | WHERE "2007-12-03T10:15:30+01:00" < date AND date < "2024-01-01T10:15:30+01:00\"""", containsString(""" "esql_single_value":{"field":"date","next":{"range":{"date":{"gt":"2007-12-03T09:15:30.000Z","lt":"2024-01-01T09:15:30.000Z",\ - "time_zone":"Z","format":"strict_date_optional_time","boost":1.0}}}""")); + "time_zone":"Z","format":"strict_date_optional_time","boost":0.0}}}""")); assertQueryTranslation(""" FROM test | WHERE "2007-12-03T10:15:30-01:00" <= date AND date <= "2024-01-01T10:15:30+01:00\"""", containsString(""" "esql_single_value":{"field":"date","next":{"range":{"date":{"gte":"2007-12-03T11:15:30.000Z","lte":"2024-01-01T09:15:30.000Z",\ - "time_zone":"Z","format":"strict_date_optional_time","boost":1.0}}}""")); + "time_zone":"Z","format":"strict_date_optional_time","boost":0.0}}}""")); assertQueryTranslation(""" FROM test | WHERE "2007-12-03T10:15:30" <= date AND date <= "2024-01-01T10:15:30+01:00\"""", containsString(""" "esql_single_value":{"field":"date","next":{"range":{"date":{"gte":"2007-12-03T10:15:30.000Z","lte":"2024-01-01T09:15:30.000Z",\ - "time_zone":"Z","format":"strict_date_optional_time","boost":1.0}}}""")); + "time_zone":"Z","format":"strict_date_optional_time","boost":0.0}}}""")); } public void testIPs() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/QueryStringQueryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/QueryStringQueryTests.java index 3114b852aac70..635fa65e727c3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/QueryStringQueryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/QueryStringQueryTests.java @@ -39,7 +39,7 @@ public void testQueryBuilding() { private static QueryStringQueryBuilder getBuilder(Map options) { final Source source = new Source(1, 1, StringUtils.EMPTY); final QueryStringQuery query = new QueryStringQuery(source, "eggplant", Collections.singletonMap("foo", 1.0f), options); - return (QueryStringQueryBuilder) query.asBuilder(); + return (QueryStringQueryBuilder) query.toQueryBuilder(); } public void testToString() {