Skip to content

Commit 04b3b12

Browse files
committed
Allow partial results by default in ES|QL (#125060)
With this change, ES|QL will return partial results instead of failing the entire query when encountering errors. Callers should check the partial_results flag in the response to determine if the result is partial or complete. If returning partial results is not desired, this option can be overridden per request via the allow_partial_results parameter in the query URL or globally via the cluster setting esql.allow_partial_results. Relates #122802
1 parent 2ff8ca2 commit 04b3b12

File tree

19 files changed

+118
-55
lines changed

19 files changed

+118
-55
lines changed

docs/changelog/125060.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
pr: 125060
2+
summary: Allow partial results by default in ES|QL
3+
area: ES|QL
4+
type: breaking
5+
issues: [122802]
6+
7+
breaking:
8+
title: Allow partial results by default in ES|QL
9+
area: ES|QL
10+
details: >-
11+
In earlier versions of {es}, ES|QL would fail the entire query if it encountered any error. ES|QL now returns partial results instead of failing when encountering errors.
12+
13+
impact: >-
14+
Callers should check the `is_partial` flag returned in the response to determine if the result is partial or complete. If returning partial results is not desired, this option can be overridden per request via an `allow_partial_results` parameter in the query URL or globally via the cluster setting `esql.query.allow_partial_results`.
15+
16+
notable: true

docs/reference/release-notes/8.19.0.asciidoc

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,16 @@ coming[8.19.0]
66
Also see <<breaking-changes-8.19,Breaking changes in 8.19>>.
77

88

9+
[[breaking-changes-8.19]]
10+
[float]
11+
=== Breaking changes
12+
13+
ES|QL::
14+
* In earlier versions of {es}, ES|QL would fail the entire query if it encountered any error.
15+
ES|QL now returns partial results instead of failing when encountering errors.
16+
Callers should check the `is_partial` flag returned in the response to determine
17+
if the result is partial or complete. If returning partial results is not desired,
18+
this option can be overridden per request via an `allow_partial_results` parameter
19+
in the query URL or globally via the cluster setting `esql.query.allow_partial_results`.
20+
{es-pull}125060[#125060] (issue: {es-issue}122802[#122802])
21+

test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/Clusters.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ static ElasticsearchCluster buildCluster() {
2121
.module("test-esql-heap-attack")
2222
.setting("xpack.security.enabled", "false")
2323
.setting("xpack.license.self_generated.type", "trial")
24+
.setting("esql.query.allow_partial_results", "false")
2425
.jvmArg("-Xmx512m");
2526
String javaVersion = JvmInfo.jvmInfo().version();
2627
if (javaVersion.equals("20") || javaVersion.equals("21")) {

x-pack/plugin/build.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,7 @@ tasks.named("yamlRestTestV7CompatTransform").configure({ task ->
223223
task.skipTest("esql/40_tsdb/from index pattern unsupported counter", "TODO: support for subset of metric fields")
224224
task.skipTest("esql/40_unsupported_types/unsupported", "TODO: support for subset of metric fields")
225225
task.skipTest("esql/40_unsupported_types/unsupported with sort", "TODO: support for subset of metric fields")
226+
task.skipTest("esql/63_enrich_int_range/Invalid age as double", "TODO: require disable allow_partial_results")
226227
})
227228

228229
tasks.named('yamlRestTestV7CompatTest').configure {

x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,10 @@ public void testIndexPatternErrorMessageComparison_ESQL_SearchDSL() throws Excep
312312
searchRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("es-security-runas-user", "metadata1_read2"));
313313

314314
// ES|QL query on the same index pattern
315-
var esqlResp = expectThrows(ResponseException.class, () -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2"));
315+
var esqlResp = expectThrows(
316+
ResponseException.class,
317+
() -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2", false)
318+
);
316319
var srchResp = expectThrows(ResponseException.class, () -> client().performRequest(searchRequest));
317320

318321
for (ResponseException r : List.of(esqlResp, srchResp)) {
@@ -331,7 +334,8 @@ public void testLimitedPrivilege() throws Exception {
331334
ResponseException.class,
332335
() -> runESQLCommand(
333336
"metadata1_read2",
334-
"FROM index-user1,index-user2 METADATA _index | STATS sum=sum(value), index=VALUES(_index)"
337+
"FROM index-user1,index-user2 METADATA _index | STATS sum=sum(value), index=VALUES(_index)",
338+
false
335339
)
336340
);
337341
assertThat(
@@ -344,7 +348,7 @@ public void testLimitedPrivilege() throws Exception {
344348

345349
resp = expectThrows(
346350
ResponseException.class,
347-
() -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2 METADATA _index | STATS index=VALUES(_index)")
351+
() -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2 METADATA _index | STATS index=VALUES(_index)", false)
348352
);
349353
assertThat(
350354
EntityUtils.toString(resp.getResponse().getEntity()),
@@ -356,7 +360,7 @@ public void testLimitedPrivilege() throws Exception {
356360

357361
resp = expectThrows(
358362
ResponseException.class,
359-
() -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2 | STATS sum=sum(value)")
363+
() -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2 | STATS sum=sum(value)", false)
360364
);
361365
assertThat(
362366
EntityUtils.toString(resp.getResponse().getEntity()),
@@ -368,7 +372,7 @@ public void testLimitedPrivilege() throws Exception {
368372

369373
resp = expectThrows(
370374
ResponseException.class,
371-
() -> runESQLCommand("alias_user1", "FROM first-alias,index-user1 METADATA _index | KEEP _index, org, value | LIMIT 10")
375+
() -> runESQLCommand("alias_user1", "FROM first-alias,index-user1 METADATA _index | KEEP _index, org, value | LIMIT 10", false)
372376
);
373377
assertThat(
374378
EntityUtils.toString(resp.getResponse().getEntity()),
@@ -382,7 +386,8 @@ public void testLimitedPrivilege() throws Exception {
382386
ResponseException.class,
383387
() -> runESQLCommand(
384388
"alias_user2",
385-
"from second-alias,index-user2 METADATA _index | stats sum=sum(value), index=VALUES(_index)"
389+
"from second-alias,index-user2 METADATA _index | stats sum=sum(value), index=VALUES(_index)",
390+
false
386391
)
387392
);
388393
assertThat(
@@ -826,6 +831,10 @@ public void testDataStream() throws IOException {
826831
}
827832

828833
protected Response runESQLCommand(String user, String command) throws IOException {
834+
return runESQLCommand(user, command, null);
835+
}
836+
837+
protected Response runESQLCommand(String user, String command, Boolean allowPartialResults) throws IOException {
829838
if (command.toLowerCase(Locale.ROOT).contains("limit") == false) {
830839
// add a (high) limit to avoid warnings on default limit
831840
command += " | limit 10000000";
@@ -839,6 +848,9 @@ protected Response runESQLCommand(String user, String command) throws IOExceptio
839848
request.setJsonEntity(Strings.toString(json));
840849
request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("es-security-runas-user", user));
841850
request.addParameter("error_trace", "true");
851+
if (allowPartialResults != null) {
852+
request.addParameter("allow_partial_results", Boolean.toString(allowPartialResults));
853+
}
842854
return client().performRequest(request);
843855
}
844856

x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/EsqlRestValidationIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,6 @@ private RestClient remoteClusterClient() throws IOException {
8383

8484
@Before
8585
public void skipTestOnOldVersions() {
86-
assumeTrue("skip on old versions", Clusters.localClusterVersion().equals(Version.V_8_16_0));
86+
assumeTrue("skip on old versions", Clusters.localClusterVersion().equals(Version.V_8_19_0));
8787
}
8888
}

x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.junit.rules.TestRule;
3131

3232
import java.io.IOException;
33+
import java.util.List;
3334
import java.util.Map;
3435

3536
import static org.elasticsearch.test.MapMatcher.assertMap;
@@ -94,6 +95,12 @@ protected String from(String... indexName) {
9495

9596
@Override
9697
public Map<String, Object> runEsql(RestEsqlTestCase.RequestObjectBuilder requestObject) throws IOException {
98+
if (requestObject.allowPartialResults() != null) {
99+
assumeTrue(
100+
"require allow_partial_results on local cluster",
101+
clusterHasCapability("POST", "/_query", List.of(), List.of("support_partial_results")).orElse(false)
102+
);
103+
}
97104
requestObject.includeCCSMetadata(true);
98105
return super.runEsql(requestObject);
99106
}

x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ public void testInvalidPragma() throws IOException {
110110
request.setJsonEntity("{\"f\":" + i + "}");
111111
assertOK(client().performRequest(request));
112112
}
113-
RequestObjectBuilder builder = requestObjectBuilder().query("from test-index | limit 1 | keep f");
113+
RequestObjectBuilder builder = requestObjectBuilder().query("from test-index | limit 1 | keep f").allowPartialResults(false);
114114
builder.pragmas(Settings.builder().put("data_partitioning", "invalid-option").build());
115115
ResponseException re = expectThrows(ResponseException.class, () -> runEsqlSync(builder));
116116
assertThat(EntityUtils.toString(re.getResponse().getEntity()), containsString("No enum constant"));

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlRestValidationTestCase.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ private Request createRequest(String indexName) throws IOException {
129129
final var request = new Request("POST", "/_query");
130130
request.addParameter("error_trace", "true");
131131
request.addParameter("pretty", "true");
132+
request.addParameter("allow_partial_results", Boolean.toString(false));
132133
request.setJsonEntity(
133134
Strings.toString(JsonXContent.contentBuilder().startObject().field("query", "from " + indexName).endObject())
134135
);

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -198,17 +198,26 @@ public void testIndicesDontExist() throws IOException {
198198
int docsTest1 = 0; // we are interested only in the created index, not necessarily that it has data
199199
indexTimestampData(docsTest1, "test1", "2024-11-26", "id1");
200200

201-
ResponseException e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo"))));
201+
ResponseException e = expectThrows(
202+
ResponseException.class,
203+
() -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo")).allowPartialResults(false))
204+
);
202205
assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
203206
assertThat(e.getMessage(), containsString("verification_exception"));
204207
assertThat(e.getMessage(), anyOf(containsString("Unknown index [foo]"), containsString("Unknown index [remote_cluster:foo]")));
205208

206-
e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo*"))));
209+
e = expectThrows(
210+
ResponseException.class,
211+
() -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo*")).allowPartialResults(false))
212+
);
207213
assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
208214
assertThat(e.getMessage(), containsString("verification_exception"));
209215
assertThat(e.getMessage(), anyOf(containsString("Unknown index [foo*]"), containsString("Unknown index [remote_cluster:foo*]")));
210216

211-
e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo", "test1"))));
217+
e = expectThrows(
218+
ResponseException.class,
219+
() -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo", "test1")).allowPartialResults(false))
220+
);
212221
assertEquals(404, e.getResponse().getStatusLine().getStatusCode());
213222
assertThat(e.getMessage(), containsString("index_not_found_exception"));
214223
assertThat(e.getMessage(), anyOf(containsString("no such index [foo]"), containsString("no such index [remote_cluster:foo]")));
@@ -217,7 +226,7 @@ public void testIndicesDontExist() throws IOException {
217226
var pattern = from("test1");
218227
e = expectThrows(
219228
ResponseException.class,
220-
() -> runEsql(timestampFilter("gte", "2020-01-01").query(pattern + " | LOOKUP JOIN foo ON id1"))
229+
() -> runEsql(timestampFilter("gte", "2020-01-01").query(pattern + " | LOOKUP JOIN foo ON id1").allowPartialResults(false))
221230
);
222231
assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
223232
assertThat(

0 commit comments

Comments
 (0)