ES\QL: Fix scoring for non full text functions

docs/changelog/124540.yaml

@@ -0,0 +1,5 @@
+pr: 124540
+summary: "ES|QL: Fix scoring for full text functions"
+area: ES|QL
+type: bug
+issues: []

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQuery.java

@@ -43,13 +43,14 @@ public BoolQuery(Source source, boolean isAnd, List<Query> queries) {
     }
 
     @Override
-    public QueryBuilder asBuilder() {
+    protected QueryBuilder asBuilder() {
         BoolQueryBuilder boolQuery = boolQuery();
         for (Query query : queries) {
+            QueryBuilder queryBuilder = query.toQueryBuilder();
             if (isAnd) {
-                boolQuery.must(query.asBuilder());
+                boolQuery.must(queryBuilder);
             } else {
-                boolQuery.should(query.asBuilder());
+                boolQuery.should(queryBuilder);
             }
         }
         return boolQuery;
@@ -94,4 +95,9 @@ public Query negate(Source source) {
         }
         return new BoolQuery(source, isAnd == false, negated);
     }
+
+    @Override
+    public boolean scorable() {
+        return true;
+    }
 }

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/ExistsQuery.java

@@ -21,7 +21,7 @@ public ExistsQuery(Source source, String name) {
     }
 
     @Override
-    public QueryBuilder asBuilder() {
+    protected QueryBuilder asBuilder() {
         return existsQuery(name);
     }
 

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/GeoDistanceQuery.java

@@ -45,7 +45,7 @@ public double distance() {
     }
 
     @Override
-    public QueryBuilder asBuilder() {
+    protected QueryBuilder asBuilder() {
         return QueryBuilders.geoDistanceQuery(field).distance(distance, DistanceUnit.METERS).point(lat, lon);
     }
 

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchAll.java

@@ -17,7 +17,7 @@ public MatchAll(Source source) {
     }
 
     @Override
-    public QueryBuilder asBuilder() {
+    protected QueryBuilder asBuilder() {
         return matchAllQuery();
     }
 

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NotQuery.java

@@ -32,8 +32,8 @@ public Query child() {
     }
 
     @Override
-    public QueryBuilder asBuilder() {
-        return boolQuery().mustNot(child.asBuilder());
+    protected QueryBuilder asBuilder() {
+        return boolQuery().mustNot(child.toQueryBuilder());
     }
 
     @Override

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/PrefixQuery.java

@@ -34,7 +34,7 @@ public String query() {
     }
 
     @Override
-    public QueryBuilder asBuilder() {
+    protected QueryBuilder asBuilder() {
         return prefixQuery(field, query).caseInsensitive(caseInsensitive);
     }
 

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/Query.java

@@ -46,9 +46,20 @@ public Source source() {
 
     /**
      * Convert to an Elasticsearch {@link QueryBuilder} all set up to execute
-     * the query.
+     * the query. This ensures that queries have appropriate boosting for scoring.
      */
-    public abstract QueryBuilder asBuilder();
+    public final QueryBuilder toQueryBuilder() {
+        QueryBuilder builder = asBuilder();
+        if (scorable() == false) {
+            builder.boost(0.0f);
+        }
+        return builder;
+    }
+
+    /**
+     * Used internally to convert to retrieve a {@link QueryBuilder} by queries.
+     */
+    protected abstract QueryBuilder asBuilder();
 
     /**
      * Used by {@link Query#toString()} to produce a pretty string.
@@ -85,4 +96,11 @@ public String toString() {
     public Query negate(Source source) {
         return new NotQuery(source, this);
     }
+
+    /**
+     * Defines whether a query should contribute to the overall score
+     */
+    public boolean scorable() {
+        return false;
+    }
 }

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/QueryStringQuery.java

@@ -88,7 +88,7 @@ public QueryStringQuery(Source source, String query, Map<String, Float> fields,
     }
 
     @Override
-    public QueryBuilder asBuilder() {
+    protected QueryBuilder asBuilder() {
         final QueryStringQueryBuilder queryBuilder = QueryBuilders.queryStringQuery(query);
         queryBuilder.fields(fields);
         options.forEach((k, v) -> {
@@ -132,4 +132,9 @@ public boolean equals(Object obj) {
     protected String innerToString() {
         return fields + ":" + query;
     }
+
+    @Override
+    public boolean scorable() {
+        return true;
+    }
 }

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RangeQuery.java

@@ -77,7 +77,7 @@ public ZoneId zoneId() {
     }
 
     @Override
-    public QueryBuilder asBuilder() {
+    protected QueryBuilder asBuilder() {
         RangeQueryBuilder queryBuilder = rangeQuery(field).from(lower, includeLower).to(upper, includeUpper);
         if (Strings.hasText(format)) {
             queryBuilder.format(format);

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RegexQuery.java

@@ -42,7 +42,7 @@ public Boolean caseInsensitive() {
     }
 
     @Override
-    public QueryBuilder asBuilder() {
+    protected QueryBuilder asBuilder() {
         return regexpQuery(field, regex).caseInsensitive(caseInsensitive);
     }
 

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermQuery.java

@@ -14,21 +14,31 @@
 
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 
+/**
+ * Term query. It can be considered for scoring or not - filters that use term query as implementation will not use scoring,
+ * but the Term full text function will
+ */
 public class TermQuery extends Query {
 
     private final String term;
     private final Object value;
     private final boolean caseInsensitive;
+    private final boolean scorable;
 
     public TermQuery(Source source, String term, Object value) {
         this(source, term, value, false);
     }
 
     public TermQuery(Source source, String term, Object value, boolean caseInsensitive) {
+        this(source, term, value, caseInsensitive, false);
+    }
+
+    public TermQuery(Source source, String term, Object value, boolean caseInsensitive, boolean scorable) {
         super(source);
         this.term = term;
         this.value = value;
         this.caseInsensitive = caseInsensitive;
+        this.scorable = scorable;
     }
 
     public String term() {
@@ -44,15 +54,15 @@ public Boolean caseInsensitive() {
     }
 
     @Override
-    public QueryBuilder asBuilder() {
+    protected QueryBuilder asBuilder() {
         TermQueryBuilder qb = termQuery(term, value);
         // ES does not allow case_insensitive to be set to "false", it should be either "true" or not specified
         return caseInsensitive == false ? qb : qb.caseInsensitive(caseInsensitive);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(term, value, caseInsensitive);
+        return Objects.hash(term, value, caseInsensitive, scorable);
     }
 
     @Override
@@ -68,11 +78,17 @@ public boolean equals(Object obj) {
         TermQuery other = (TermQuery) obj;
         return Objects.equals(term, other.term)
             && Objects.equals(value, other.value)
-            && Objects.equals(caseInsensitive, other.caseInsensitive);
+            && Objects.equals(caseInsensitive, other.caseInsensitive)
+            && scorable == other.scorable;
     }
 
     @Override
     protected String innerToString() {
         return term + ":" + value;
     }
+
+    @Override
+    public boolean scorable() {
+        return scorable;
+    }
 }

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermsQuery.java

@@ -26,7 +26,7 @@ public TermsQuery(Source source, String term, Set<Object> values) {
     }
 
     @Override
-    public QueryBuilder asBuilder() {
+    protected QueryBuilder asBuilder() {
         return termsQuery(term, values);
     }
 

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java

@@ -43,7 +43,7 @@ public Boolean caseInsensitive() {
     }
 
     @Override
-    public QueryBuilder asBuilder() {
+    protected QueryBuilder asBuilder() {
         WildcardQueryBuilder wb = wildcardQuery(field, query);
         // ES does not allow case_insensitive to be set to "false", it should be either "true" or not specified
         return caseInsensitive == false ? wb : wb.caseInsensitive(caseInsensitive);

x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQueryTests.java

@@ -23,7 +23,7 @@ private DummyLeafQuery(Source source) {
         }
 
         @Override
-        public QueryBuilder asBuilder() {
+        protected QueryBuilder asBuilder() {
             return null;
         }
 

x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec

@@ -145,6 +145,7 @@ book_no:keyword | title:text                                | author:text
 combinedMatchWithFunctionsScoring
 required_capability: metadata_score
 required_capability: match_operator_colon
+required_capability: non_full_text_functions_scoring
 
 from books metadata _score
 | where title:"Tolkien" AND author:"Tolkien" AND year > 2000
@@ -153,7 +154,7 @@ from books metadata _score
 | sort book_no;
 
 book_no:keyword | title:text               | author:text    | year:integer | _score:double
-5335            | Letters of J R R Tolkien | J.R.R. Tolkien | 2014         | 4.733664035797119
+5335            | Letters of J R R Tolkien | J.R.R. Tolkien | 2014         | 3.733664035797119
 ;
 
 singleQstrScoring
@@ -191,6 +192,7 @@ book_no:keyword | title:keyword | _score:double
 combinedMatchWithScoringEvalNoSort
 required_capability: metadata_score
 required_capability: match_operator_colon
+required_capability: non_full_text_functions_scoring
 
 from books metadata _score
 | where title:"Tolkien" AND author:"Tolkien" AND year > 2000
@@ -200,7 +202,7 @@ from books metadata _score
 
 ignoreOrder:true
 book_no:keyword | title:text               | author:text    | year:integer | c_score:double
-5335            | Letters of J R R Tolkien | J.R.R. Tolkien | 2014         | 5.0
+5335            | Letters of J R R Tolkien | J.R.R. Tolkien | 2014         | 4.0
 ;
 
 singleQstrScoringRename
@@ -241,6 +243,7 @@ book_no:keyword | author:text                                        | _score:do
 combinedMatchWithFunctionsScoringNoSort
 required_capability: metadata_score
 required_capability: match_operator_colon
+required_capability: non_full_text_functions_scoring
 
 from books metadata _score
 | where title:"Tolkien" AND author:"Tolkien" AND year > 2000
@@ -249,12 +252,13 @@ from books metadata _score
 
 ignoreOrder:true
 book_no:keyword | title:text               | author:text    | year:integer | _score:double
-5335            | Letters of J R R Tolkien | J.R.R. Tolkien | 2014         | 4.733664035797119
+5335            | Letters of J R R Tolkien | J.R.R. Tolkien | 2014         | 3.733664035797119
 ;
 
 combinedMatchWithScoringEval
 required_capability: metadata_score
 required_capability: match_operator_colon
+required_capability: non_full_text_functions_scoring
 
 from books metadata _score
 | where title:"Tolkien" AND author:"Tolkien" AND year > 2000
@@ -264,7 +268,7 @@ from books metadata _score
 | sort book_no;
 
 book_no:keyword | title:text               | author:text    | year:integer | c_score:double
-5335            | Letters of J R R Tolkien | J.R.R. Tolkien | 2014         | 5.0
+5335            | Letters of J R R Tolkien | J.R.R. Tolkien | 2014         | 4.0
 ;
 
 singleQstrScoringEval
@@ -361,6 +365,7 @@ _id:keyword
 scoresNonPushableFunctions
 
 required_capability: metadata_score
+required_capability: non_full_text_functions_scoring
 
 from books metadata _score 
 | where length(title) > 100
@@ -369,13 +374,14 @@ from books metadata _score
 ;
 
 book_no:keyword |  _score:double
-2924            | 1.0
-8678            | 1.0
+2924            | 0.0
+8678            | 0.0
 ;
 
 scoresPushableFunctions
 
 required_capability: metadata_score
+required_capability: non_full_text_functions_scoring
 
 from books metadata _score 
 | where year >= 2017
@@ -384,11 +390,11 @@ from books metadata _score
 ;
 
 book_no:keyword | _score:double
-6818            | 1.0
-7400            | 1.0
-8480            | 1.0
-8534            | 1.0
-8615            | 1.0
+6818            | 0.0
+7400            | 0.0
+8480            | 0.0
+8534            | 0.0
+8615            | 0.0
 ;
 
 conjunctionScoresPushableNonPushableFunctions
@@ -413,6 +419,7 @@ conjunctionScoresPushableFunctions
 
 required_capability: metadata_score
 required_capability: match_function
+required_capability: non_full_text_functions_scoring
 
 from books metadata _score 
 | where match(title, "Lord") and ratings > 4.6
@@ -421,8 +428,8 @@ from books metadata _score
 ;
 
 book_no:keyword |  _score:double
-7140            | 2.746896743774414 
-4023            | 2.5062403678894043
+7140            | 1.746896743774414 
+4023            | 1.5062403678894043
 ;
 
 disjunctionScoresPushableNonPushableFunctions
@@ -438,12 +445,12 @@ from books metadata _score
 ;
 
 book_no:keyword |  _score:double
-2675            | 3.5619282722473145
-2714            | 2.9245924949645996
-7140            | 2.746896743774414
-4023            | 2.5062403678894043
-2924            | 1.0
-8678            | 1.0
+2675            | 2.5619282722473145
+2714            | 1.9245924949645996
+7140            | 1.746896743774414
+4023            | 1.5062403678894043
+2924            | 0.0
+8678            | 0.0
 ;