Skip to content

Commit a7d26f8

Browse files
luigidellaquilaluigidellaquila
committed
EQL: set allow_partial_search_results=true by default (elastic#120267)
1 parent 2c8ccf7 commit a7d26f8

File tree

10 files changed

+274
-233
lines changed

10 files changed

+274
-233
lines changed

‎docs/changelog/120267.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
pr: 120267
2+
summary: Set allow_partial_search_results=true by default
3+
area: EQL
4+
type: breaking
5+
issues: []
6+
breaking:
7+
title: Set allow_partial_search_results=true by default
8+
area: REST API
9+
details:
10+
Before this change, in case of shard failures, EQL queries always returned an error.
11+
With this change, they will keep running and will return partial results.
12+
impact:
13+
EQL queries that would previously fail due to shard failures, will now succeed and return partial results.
14+
The previous defaults can be restored by setting `xpack.eql.default_allow_partial_results` cluster setting to `false`
15+
or setting with `allow_partial_search_results` to `false` in the query request.
16+
notable: false

‎docs/reference/eql/eql-search-api.asciidoc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,10 +102,10 @@ If `false`, the request returns an error if one or more shards involved in the q
102102
If `true`, the query is executed only on the available shards, ignoring shard request timeouts and
103103
<<shard-failures,shard failures>>.
104104
+
105-
Defaults to `false`.
105+
Defaults to `true`.
106106
+
107107
To override the default for this field, set the
108-
`xpack.eql.default_allow_partial_results` cluster setting to `true`.
108+
`xpack.eql.default_allow_partial_results` cluster setting to `false`.
109109

110110

111111
[IMPORTANT]

‎rest-api-spec/src/main/resources/rest-api-spec/api/eql.search.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
"allow_partial_search_results": {
4646
"type":"boolean",
4747
"description":"Control whether the query should keep running in case of shard failures, and return partial results",
48-
"default":false
48+
"default":true
4949
},
5050
"allow_partial_sequence_results": {
5151
"type":"boolean",

‎x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java‎

Lines changed: 24 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -187,29 +187,29 @@ protected ObjectPath runQuery(String index, String query) throws Exception {
187187
builder.field("max_samples_per_key", maxSamplesPerKey);
188188
}
189189
boolean allowPartialResultsInBody = randomBoolean();
190-
if (allowPartialSearchResults != null) {
191-
if (allowPartialResultsInBody) {
190+
191+
if (allowPartialResultsInBody) {
192+
if (allowPartialSearchResults != null) {
192193
builder.field("allow_partial_search_results", String.valueOf(allowPartialSearchResults));
193-
if (allowPartialSequenceResults != null) {
194-
builder.field("allow_partial_sequence_results", String.valueOf(allowPartialSequenceResults));
195-
}
196-
} else {
197-
// these will be overwritten by the path params, that have higher priority than the query (JSON body) params
198-
if (allowPartialSearchResults != null) {
199-
builder.field("allow_partial_search_results", randomBoolean());
200-
}
201-
if (allowPartialSequenceResults != null) {
202-
builder.field("allow_partial_sequence_results", randomBoolean());
203-
}
194+
} else if (randomBoolean()) {
195+
builder.field("allow_partial_search_results", true);
196+
}
197+
if (allowPartialSequenceResults != null) {
198+
builder.field("allow_partial_sequence_results", String.valueOf(allowPartialSequenceResults));
199+
} else if (randomBoolean()) {
200+
builder.field("allow_partial_sequence_results", false);
204201
}
205202
} else {
206-
// Tests that don't specify a setting for these parameters should always pass.
207-
// These params should be irrelevant.
208-
if (randomBoolean()) {
203+
// these will be overwritten by the path params, that have higher priority than the query (JSON body) params
204+
if (allowPartialSearchResults != null) {
209205
builder.field("allow_partial_search_results", randomBoolean());
206+
} else if (randomBoolean()) {
207+
builder.field("allow_partial_search_results", true);
210208
}
211-
if (randomBoolean()) {
209+
if (allowPartialSequenceResults != null) {
212210
builder.field("allow_partial_sequence_results", randomBoolean());
211+
} else if (randomBoolean()) {
212+
builder.field("allow_partial_sequence_results", false);
213213
}
214214
}
215215
builder.endObject();
@@ -219,23 +219,17 @@ protected ObjectPath runQuery(String index, String query) throws Exception {
219219
if (ccsMinimizeRoundtrips != null) {
220220
request.addParameter("ccs_minimize_roundtrips", ccsMinimizeRoundtrips.toString());
221221
}
222-
if (allowPartialSearchResults != null) {
223-
if (allowPartialResultsInBody == false) {
222+
if (allowPartialResultsInBody == false) {
223+
if (allowPartialSearchResults != null) {
224224
request.addParameter("allow_partial_search_results", String.valueOf(allowPartialSearchResults));
225-
if (allowPartialSequenceResults != null) {
226-
request.addParameter("allow_partial_sequence_results", String.valueOf(allowPartialSequenceResults));
227-
}
225+
} else if (randomBoolean()) {
226+
request.addParameter("allow_partial_search_results", String.valueOf(true));
228227
}
229-
} else {
230-
// Tests that don't specify a setting for these parameters should always pass.
231-
// These params should be irrelevant.
232-
if (randomBoolean()) {
233-
request.addParameter("allow_partial_search_results", String.valueOf(randomBoolean()));
234-
}
235-
if (randomBoolean()) {
236-
request.addParameter("allow_partial_sequence_results", String.valueOf(randomBoolean()));
228+
if (allowPartialSequenceResults != null) {
229+
request.addParameter("allow_partial_sequence_results", String.valueOf(allowPartialSequenceResults));
237230
}
238231
}
232+
239233
int timeout = Math.toIntExact(timeout().millis());
240234
RequestConfig config = RequestConfig.copy(RequestConfig.DEFAULT)
241235
.setConnectionRequestTimeout(timeout)

‎x-pack/plugin/eql/qa/common/src/main/resources/test_failing_shards.toml‎

Lines changed: 41 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,13 @@ expect_shard_failures = false
1010
[[queries]]
1111
name = "eventQueryShardFailures"
1212
query = 'process where serial_event_id == 1 or broken == 1'
13+
expected_event_ids = [1]
14+
expect_shard_failures = true
15+
16+
17+
[[queries]]
18+
name = "eventQueryShardFailuresTrue"
19+
query = 'process where serial_event_id == 1 or broken == 1'
1320
allow_partial_search_results = true
1421
expected_event_ids = [1]
1522
expect_shard_failures = true
@@ -18,15 +25,13 @@ expect_shard_failures = true
1825
[[queries]]
1926
name = "eventQueryShardFailuresOptionalField"
2027
query = 'process where serial_event_id == 1 and ?optional_field_default_null == null or broken == 1'
21-
allow_partial_search_results = true
2228
expected_event_ids = [1]
2329
expect_shard_failures = true
2430

2531

2632
[[queries]]
2733
name = "eventQueryShardFailuresOptionalFieldMatching"
2834
query = 'process where serial_event_id == 2 and ?subtype == "create" or broken == 1'
29-
allow_partial_search_results = true
3035
expected_event_ids = [2]
3136
expect_shard_failures = true
3237

@@ -64,7 +69,6 @@ sequence
6469
[process where serial_event_id == 1]
6570
[process where serial_event_id == 2]
6671
'''
67-
allow_partial_search_results = true
6872
expected_event_ids = [1, 2]
6973
expect_shard_failures = false
7074

@@ -76,6 +80,17 @@ sequence
7680
[process where serial_event_id == 1 or broken == 1]
7781
[process where serial_event_id == 2]
7882
'''
83+
expected_event_ids = []
84+
expect_shard_failures = true
85+
86+
87+
[[queries]]
88+
name = "sequenceQueryMissingShardsTrue"
89+
query = '''
90+
sequence
91+
[process where serial_event_id == 1 or broken == 1]
92+
[process where serial_event_id == 2]
93+
'''
7994
allow_partial_search_results = true
8095
expected_event_ids = []
8196
expect_shard_failures = true
@@ -88,6 +103,18 @@ sequence
88103
[process where serial_event_id == 1 or broken == 1]
89104
[process where serial_event_id == 2]
90105
'''
106+
allow_partial_sequence_results = true
107+
expected_event_ids = [1, 2]
108+
expect_shard_failures = true
109+
110+
111+
[[queries]]
112+
name = "sequenceQueryMissingShardsPartialResultsTrue"
113+
query = '''
114+
sequence
115+
[process where serial_event_id == 1 or broken == 1]
116+
[process where serial_event_id == 2]
117+
'''
91118
allow_partial_search_results = true
92119
allow_partial_sequence_results = true
93120
expected_event_ids = [1, 2]
@@ -101,7 +128,6 @@ sequence
101128
[process where ?serial_event_id == 1 or broken == 1]
102129
[process where serial_event_id == 2]
103130
'''
104-
allow_partial_search_results = true
105131
allow_partial_sequence_results = true
106132
expected_event_ids = [1, 2]
107133
expect_shard_failures = true
@@ -114,7 +140,6 @@ sequence with maxspan=100000d
114140
[process where serial_event_id == 1 and ?subtype == "create" or broken == 1]
115141
[process where serial_event_id == 2]
116142
'''
117-
allow_partial_search_results = true
118143
allow_partial_sequence_results = true
119144
expected_event_ids = [1, 2]
120145
expect_shard_failures = true
@@ -128,7 +153,6 @@ sequence with maxspan=100000d
128153
![process where broken == 1]
129154
[process where serial_event_id == 2]
130155
'''
131-
allow_partial_search_results = true
132156
allow_partial_sequence_results = true
133157
expected_event_ids = [1, -1, 2]
134158
expect_shard_failures = true
@@ -142,7 +166,6 @@ sequence with maxspan=100000d
142166
![process where broken == 1]
143167
[process where serial_event_id == 2]
144168
'''
145-
allow_partial_search_results = true
146169
allow_partial_sequence_results = true
147170
expected_event_ids = [1, -1, 2]
148171
expect_shard_failures = true
@@ -155,6 +178,17 @@ sample by event_subtype_full
155178
[process where serial_event_id == 1 or broken == 1]
156179
[process where serial_event_id == 2]
157180
'''
181+
expected_event_ids = [1, 2]
182+
expect_shard_failures = true
183+
184+
185+
[[queries]]
186+
name = "sampleQueryMissingShardsPartialResultsTrue"
187+
query = '''
188+
sample by event_subtype_full
189+
[process where serial_event_id == 1 or broken == 1]
190+
[process where serial_event_id == 2]
191+
'''
158192
allow_partial_search_results = true
159193
expected_event_ids = [1, 2]
160194
expect_shard_failures = true
@@ -167,7 +201,6 @@ sample by event_subtype_full
167201
[process where serial_event_id == 1 and ?subtype == "create" or broken == 1]
168202
[process where serial_event_id == 2]
169203
'''
170-
allow_partial_search_results = true
171204
expected_event_ids = [1, 2]
172205
expect_shard_failures = true
173206

‎x-pack/plugin/eql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -410,10 +410,10 @@ private void assertMultiValueFunctionQuery(
410410

411411
StringBuilder payload = new StringBuilder("{\"query\":\"" + query + "\"");
412412
if (randomBoolean()) {
413-
payload.append(", \"allow_partial_search_results\": true");
413+
payload.append(", \"allow_partial_search_results\": " + randomBoolean());
414414
}
415415
if (randomBoolean()) {
416-
payload.append(", \"allow_partial_sequence_results\": true");
416+
payload.append(", \"allow_partial_sequence_results\": " + randomBoolean());
417417
}
418418
payload.append("}");
419419
request.setJsonEntity(payload.toString());

‎x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -509,6 +509,30 @@ setup:
509509

510510

511511
---
512+
513+
"Execute query shard failures":
514+
- do:
515+
eql.search:
516+
index: eql_test*
517+
body:
518+
query: 'process where user == "SYSTEM" and day_of_week == "Monday"'
519+
fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"]
520+
allow_partial_search_results: true
521+
522+
- match: {timed_out: false}
523+
- match: {hits.total.value: 1}
524+
- match: {hits.total.relation: "eq"}
525+
- match: {hits.events.0._source.user: "SYSTEM"}
526+
- match: {hits.events.0._id: "1"}
527+
- match: {hits.events.0.fields.@timestamp: ["1580733296000"]}
528+
- match: {hits.events.0.fields.id: [123]}
529+
- match: {hits.events.0.fields.valid: [false]}
530+
- match: {hits.events.0.fields.day_of_week: ["Monday"]}
531+
- match: {shard_failures.0.index: "eql_test_rebel"}
532+
533+
534+
---
535+
512536
"Execute query shard failures and with allow_partial_search_results":
513537
- do:
514538
eql.search:
@@ -535,7 +559,6 @@ setup:
535559
- do:
536560
eql.search:
537561
index: eql_test*
538-
allow_partial_search_results: true
539562
body:
540563
query: 'process where user == "SYSTEM" and day_of_week == "Monday"'
541564
fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"]
@@ -575,7 +598,6 @@ setup:
575598
body:
576599
query: 'sequence [process where user == "SYSTEM" and day_of_week == "Monday"] [process where user == "SYSTEM" and day_of_week == "Tuesday"]'
577600
fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"]
578-
allow_partial_search_results: true
579601
allow_partial_sequence_results: true
580602

581603
- match: {timed_out: false}
@@ -600,7 +622,6 @@ setup:
600622
- do:
601623
eql.search:
602624
index: eql_test*
603-
allow_partial_search_results: true
604625
allow_partial_sequence_results: true
605626
body:
606627
query: 'sequence [process where user == "SYSTEM" and day_of_week == "Monday"] [process where user == "SYSTEM" and day_of_week == "Tuesday"]'

0 commit comments

Comments
 (0)