Skip to content

Commit b5b216b

Browse files
committed
ESQL: Fix constant keyword optimization
Fixes the ESQL's detection of `constant_keyword` fields. We unplugged it when we changed a function signature because we didn't have an `@Override` annotation. This plugs it back in and adds it to the integration tests we use for pushing queries to lucene. When you do `| WHERE constant_keyword_field == "itsvalue"` then the whole is removed from the query plan because *all* documents are equal.
1 parent 7b8f4fb commit b5b216b

File tree

3 files changed

+150
-73
lines changed

3 files changed

+150
-73
lines changed

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/LocalSourceOperator.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,10 @@ public Page getOutput() {
8282
}
8383

8484
@Override
85-
public void close() {
85+
public void close() {}
8686

87+
@Override
88+
public String toString() {
89+
return "LocalSourceOperator";
8790
}
8891
}

x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java

Lines changed: 142 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ public class PushQueriesIT extends ESRestTestCase {
5757

5858
@ParametersFactory(argumentFormatting = "%1s")
5959
public static List<Object[]> args() {
60-
return Stream.of("auto", "text", "match_only_text", "semantic_text").map(s -> new Object[] { s }).toList();
60+
return Stream.of("auto", "text", "match_only_text", "semantic_text", "constant_keyword").map(s -> new Object[] { s }).toList();
6161
}
6262

6363
private final String type;
@@ -74,16 +74,16 @@ public void testEquality() throws IOException {
7474
""";
7575
String luceneQuery = switch (type) {
7676
case "text", "auto" -> "#test.keyword:%value -_ignored:test.keyword";
77-
case "match_only_text" -> "*:*";
77+
case "constant_keyword", "match_only_text" -> "*:*";
7878
case "semantic_text" -> "FieldExistsQuery [field=_primary_term]";
7979
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
8080
};
81-
boolean filterInCompute = switch (type) {
82-
case "text", "auto" -> false;
83-
case "match_only_text", "semantic_text" -> true;
81+
ComputeSignature dataNodeSignature = switch (type) {
82+
case "auto", "constant_keyword", "text" -> ComputeSignature.FILTER_IN_QUERY;
83+
case "match_only_text", "semantic_text" -> ComputeSignature.FILTER_IN_COMPUTE;
8484
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
8585
};
86-
testPushQuery(value, esqlQuery, List.of(luceneQuery), filterInCompute, true);
86+
testPushQuery(value, esqlQuery, List.of(luceneQuery), dataNodeSignature, true);
8787
}
8888

8989
public void testEqualityTooBigToPush() throws IOException {
@@ -93,11 +93,16 @@ public void testEqualityTooBigToPush() throws IOException {
9393
| WHERE test == "%value"
9494
""";
9595
String luceneQuery = switch (type) {
96-
case "text", "auto", "match_only_text" -> "*:*";
96+
case "auto", "constant_keyword", "match_only_text", "text" -> "*:*";
9797
case "semantic_text" -> "FieldExistsQuery [field=_primary_term]";
9898
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
9999
};
100-
testPushQuery(value, esqlQuery, List.of(luceneQuery), true, true);
100+
ComputeSignature dataNodeSignature = switch (type) {
101+
case "constant_keyword" -> ComputeSignature.FILTER_IN_QUERY;
102+
case "auto", "match_only_text", "semantic_text", "text" -> ComputeSignature.FILTER_IN_COMPUTE;
103+
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
104+
};
105+
testPushQuery(value, esqlQuery, List.of(luceneQuery), dataNodeSignature, true);
101106
}
102107

103108
/**
@@ -111,11 +116,16 @@ public void testEqualityOrTooBig() throws IOException {
111116
| WHERE test == "%value" OR test == "%tooBig"
112117
""".replace("%tooBig", tooBig);
113118
String luceneQuery = switch (type) {
114-
case "text", "auto", "match_only_text" -> "*:*";
119+
case "auto", "constant_keyword", "match_only_text", "text" -> "*:*";
115120
case "semantic_text" -> "FieldExistsQuery [field=_primary_term]";
116121
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
117122
};
118-
testPushQuery(value, esqlQuery, List.of(luceneQuery), true, true);
123+
ComputeSignature dataNodeSignature = switch (type) {
124+
case "constant_keyword" -> ComputeSignature.FILTER_IN_QUERY;
125+
case "auto", "match_only_text", "semantic_text", "text" -> ComputeSignature.FILTER_IN_COMPUTE;
126+
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
127+
};
128+
testPushQuery(value, esqlQuery, List.of(luceneQuery), dataNodeSignature, true);
119129
}
120130

121131
public void testEqualityOrOther() throws IOException {
@@ -125,17 +135,17 @@ public void testEqualityOrOther() throws IOException {
125135
| WHERE test == "%value" OR foo == 2
126136
""";
127137
String luceneQuery = switch (type) {
128-
case "text", "auto" -> "(#test.keyword:%value -_ignored:test.keyword) foo:[2 TO 2]";
129-
case "match_only_text" -> "*:*";
138+
case "auto", "text" -> "(#test.keyword:%value -_ignored:test.keyword) foo:[2 TO 2]";
139+
case "constant_keyword", "match_only_text" -> "*:*";
130140
case "semantic_text" -> "FieldExistsQuery [field=_primary_term]";
131141
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
132142
};
133-
boolean filterInCompute = switch (type) {
134-
case "text", "auto" -> false;
135-
case "match_only_text", "semantic_text" -> true;
143+
ComputeSignature dataNodeSignature = switch (type) {
144+
case "auto", "constant_keyword", "text" -> ComputeSignature.FILTER_IN_QUERY;
145+
case "match_only_text", "semantic_text" -> ComputeSignature.FILTER_IN_COMPUTE;
136146
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
137147
};
138-
testPushQuery(value, esqlQuery, List.of(luceneQuery), filterInCompute, true);
148+
testPushQuery(value, esqlQuery, List.of(luceneQuery), dataNodeSignature, true);
139149
}
140150

141151
public void testEqualityAndOther() throws IOException {
@@ -145,8 +155,8 @@ public void testEqualityAndOther() throws IOException {
145155
| WHERE test == "%value" AND foo == 1
146156
""";
147157
List<String> luceneQueryOptions = switch (type) {
148-
case "text", "auto" -> List.of("#test.keyword:%value -_ignored:test.keyword #foo:[1 TO 1]");
149-
case "match_only_text" -> List.of("foo:[1 TO 1]");
158+
case "auto", "text" -> List.of("#test.keyword:%value -_ignored:test.keyword #foo:[1 TO 1]");
159+
case "constant_keyword", "match_only_text" -> List.of("foo:[1 TO 1]");
150160
case "semantic_text" ->
151161
/*
152162
* single_value_match is here because there are extra documents hiding in the index
@@ -158,12 +168,12 @@ public void testEqualityAndOther() throws IOException {
158168
);
159169
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
160170
};
161-
boolean filterInCompute = switch (type) {
162-
case "text", "auto" -> false;
163-
case "match_only_text", "semantic_text" -> true;
171+
ComputeSignature dataNodeSignature = switch (type) {
172+
case "auto", "constant_keyword", "text" -> ComputeSignature.FILTER_IN_QUERY;
173+
case "match_only_text", "semantic_text" -> ComputeSignature.FILTER_IN_COMPUTE;
164174
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
165175
};
166-
testPushQuery(value, esqlQuery, luceneQueryOptions, filterInCompute, true);
176+
testPushQuery(value, esqlQuery, luceneQueryOptions, dataNodeSignature, true);
167177
}
168178

169179
public void testInequality() throws IOException {
@@ -173,12 +183,17 @@ public void testInequality() throws IOException {
173183
| WHERE test != "%different_value"
174184
""";
175185
String luceneQuery = switch (type) {
176-
case "text", "auto" -> "(-test.keyword:%different_value #*:*) _ignored:test.keyword";
177-
case "match_only_text" -> "*:*";
186+
case "auto", "text" -> "(-test.keyword:%different_value #*:*) _ignored:test.keyword";
187+
case "constant_keyword", "match_only_text" -> "*:*";
178188
case "semantic_text" -> "FieldExistsQuery [field=_primary_term]";
179189
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
180190
};
181-
testPushQuery(value, esqlQuery, List.of(luceneQuery), true, true);
191+
ComputeSignature dataNodeSignature = switch (type) {
192+
case "constant_keyword" -> ComputeSignature.FILTER_IN_QUERY;
193+
case "auto", "match_only_text", "semantic_text", "text" -> ComputeSignature.FILTER_IN_COMPUTE;
194+
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
195+
};
196+
testPushQuery(value, esqlQuery, List.of(luceneQuery), dataNodeSignature, true);
182197
}
183198

184199
public void testInequalityTooBigToPush() throws IOException {
@@ -188,11 +203,16 @@ public void testInequalityTooBigToPush() throws IOException {
188203
| WHERE test != "%value"
189204
""";
190205
String luceneQuery = switch (type) {
191-
case "text", "auto", "match_only_text" -> "*:*";
206+
case "auto", "constant_keyword", "match_only_text", "text" -> "*:*";
192207
case "semantic_text" -> "FieldExistsQuery [field=_primary_term]";
193208
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
194209
};
195-
testPushQuery(value, esqlQuery, List.of(luceneQuery), true, false);
210+
ComputeSignature dataNodeSignature = switch (type) {
211+
case "constant_keyword" -> ComputeSignature.FIND_NONE;
212+
case "auto", "match_only_text", "semantic_text", "text" -> ComputeSignature.FILTER_IN_COMPUTE;
213+
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
214+
};
215+
testPushQuery(value, esqlQuery, List.of(luceneQuery), dataNodeSignature, false);
196216
}
197217

198218
public void testCaseInsensitiveEquality() throws IOException {
@@ -202,15 +222,49 @@ public void testCaseInsensitiveEquality() throws IOException {
202222
| WHERE TO_LOWER(test) == "%value"
203223
""";
204224
String luceneQuery = switch (type) {
205-
case "text", "auto", "match_only_text" -> "*:*";
225+
case "auto", "constant_keyword", "match_only_text", "text" -> "*:*";
206226
case "semantic_text" -> "FieldExistsQuery [field=_primary_term]";
207227
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
208228
};
209-
testPushQuery(value, esqlQuery, List.of(luceneQuery), true, true);
229+
ComputeSignature dataNodeSignature = switch (type) {
230+
case "constant_keyword" -> ComputeSignature.FILTER_IN_QUERY;
231+
case "auto", "match_only_text", "semantic_text", "text" -> ComputeSignature.FILTER_IN_COMPUTE;
232+
default -> throw new UnsupportedOperationException("unknown type [" + type + "]");
233+
};
234+
testPushQuery(value, esqlQuery, List.of(luceneQuery), dataNodeSignature, true);
210235
}
211236

212-
private void testPushQuery(String value, String esqlQuery, List<String> luceneQueryOptions, boolean filterInCompute, boolean found)
213-
throws IOException {
237+
enum ComputeSignature {
238+
FILTER_IN_COMPUTE(
239+
matchesList().item("LuceneSourceOperator")
240+
.item("ValuesSourceReaderOperator")
241+
.item("FilterOperator")
242+
.item("LimitOperator")
243+
.item("ProjectOperator")
244+
.item("ExchangeSinkOperator")
245+
),
246+
FILTER_IN_QUERY(
247+
matchesList().item("LuceneSourceOperator")
248+
.item("ValuesSourceReaderOperator")
249+
.item("ProjectOperator")
250+
.item("ExchangeSinkOperator")
251+
),
252+
FIND_NONE(matchesList().item("LocalSourceOperator").item("ExchangeSinkOperator"));
253+
254+
private final ListMatcher matcher;
255+
256+
ComputeSignature(ListMatcher sig) {
257+
this.matcher = sig;
258+
}
259+
}
260+
261+
private void testPushQuery(
262+
String value,
263+
String esqlQuery,
264+
List<String> luceneQueryOptions,
265+
ComputeSignature dataNodeSignature,
266+
boolean found
267+
) throws IOException {
214268
indexValue(value);
215269
String differentValue = randomValueOtherThan(value, () -> randomAlphaOfLength(value.isEmpty() ? 1 : value.length()));
216270

@@ -226,7 +280,7 @@ private void testPushQuery(String value, String esqlQuery, List<String> luceneQu
226280
.entry("planning", matchesMap().extraOk())
227281
.entry("query", matchesMap().extraOk())
228282
),
229-
matchesList().item(matchesMap().entry("name", "test").entry("type", "text")),
283+
matchesList().item(matchesMap().entry("name", "test").entry("type", anyOf(equalTo("text"), equalTo("keyword")))),
230284
equalTo(found ? List.of(List.of(value)) : List.of())
231285
);
232286
Matcher<String> luceneQueryMatcher = anyOf(
@@ -250,12 +304,7 @@ private void testPushQuery(String value, String esqlQuery, List<String> luceneQu
250304
String description = p.get("description").toString();
251305
switch (description) {
252306
case "data" -> {
253-
ListMatcher matcher = matchesList().item("LuceneSourceOperator").item("ValuesSourceReaderOperator");
254-
if (filterInCompute) {
255-
matcher = matcher.item("FilterOperator").item("LimitOperator");
256-
}
257-
matcher = matcher.item("ProjectOperator").item("ExchangeSinkOperator");
258-
assertMap(sig, matcher);
307+
assertMap(sig, dataNodeSignature.matcher);
259308
}
260309
case "node_reduce" -> {
261310
if (sig.contains("LimitOperator")) {
@@ -294,38 +343,10 @@ private void indexValue(String value) throws IOException {
294343
}""";
295344
json += switch (type) {
296345
case "auto" -> "";
297-
case "semantic_text" -> """
298-
,
299-
"mappings": {
300-
"properties": {
301-
"test": {
302-
"type": "semantic_text",
303-
"inference_id": "test",
304-
"fields": {
305-
"keyword": {
306-
"type": "keyword",
307-
"ignore_above": 256
308-
}
309-
}
310-
}
311-
}
312-
}""";
313-
default -> """
314-
,
315-
"mappings": {
316-
"properties": {
317-
"test": {
318-
"type": "%type",
319-
"fields": {
320-
"keyword": {
321-
"type": "keyword",
322-
"ignore_above": 256
323-
}
324-
}
325-
}
326-
}
327-
}
328-
}""".replace("%type", type);
346+
case "constant_keyword" -> constantKeyword();
347+
case "semantic_text" -> semanticText();
348+
case "text", "match_only_text" -> subKeyword();
349+
default -> throw new UnsupportedOperationException("unsupported type config: " + type);
329350
};
330351
json += "}";
331352
createIndex.setJsonEntity(json);
@@ -345,6 +366,57 @@ private void indexValue(String value) throws IOException {
345366
assertThat(entityToMap(bulkResponse.getEntity(), XContentType.JSON), matchesMap().entry("errors", false).extraOk());
346367
}
347368

369+
private String constantKeyword() {
370+
return """
371+
,
372+
"mappings": {
373+
"properties": {
374+
"test": {
375+
"type": "constant_keyword"
376+
}
377+
}
378+
}
379+
""";
380+
}
381+
382+
private String semanticText() {
383+
return """
384+
,
385+
"mappings": {
386+
"properties": {
387+
"test": {
388+
"type": "semantic_text",
389+
"inference_id": "test",
390+
"fields": {
391+
"keyword": {
392+
"type": "keyword",
393+
"ignore_above": 256
394+
}
395+
}
396+
}
397+
}
398+
}""";
399+
}
400+
401+
private String subKeyword() {
402+
return """
403+
,
404+
"mappings": {
405+
"properties": {
406+
"test": {
407+
"type": "%type",
408+
"fields": {
409+
"keyword": {
410+
"type": "keyword",
411+
"ignore_above": 256
412+
}
413+
}
414+
}
415+
}
416+
}
417+
}""".replace("%type", type);
418+
}
419+
348420
private static final Pattern TO_NAME = Pattern.compile("\\[.+", Pattern.DOTALL);
349421

350422
private static String checkOperatorProfile(Map<String, Object> o, Matcher<String> query) {

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchContextStats.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ public boolean hasExactSubfield(FieldName field) {
149149
return cache.computeIfAbsent(field.string(), this::makeFieldStats).config.hasExactSubfield;
150150
}
151151

152+
@Override
152153
public long count() {
153154
var count = new long[] { 0 };
154155
boolean completed = doWithContexts(r -> {
@@ -322,10 +323,11 @@ public boolean canUseEqualityOnSyntheticSourceDelegate(FieldAttribute.FieldName
322323
return true;
323324
}
324325

325-
public String constantValue(String name) {
326+
@Override
327+
public String constantValue(FieldAttribute.FieldName name) {
326328
String val = null;
327329
for (SearchExecutionContext ctx : contexts) {
328-
MappedFieldType f = ctx.getFieldType(name);
330+
MappedFieldType f = ctx.getFieldType(name.string());
329331
if (f == null) {
330332
return null;
331333
}

0 commit comments

Comments
 (0)