Skip to content

Commit 0f79a0f

Browse files
dhurley14dhurley14
committed
[Security Solution] [Detection Engine] Logs shard failures for eql event queries on rule details page and in event log (elastic#207396)
## Summary Related: elastic/elasticsearch#116388 Adds support for shard failures for EQL event queries in the detection engine. (cherry picked from commit 4419390)
1 parent 91dcae5 commit 0f79a0f

File tree

10 files changed

+3512
-78
lines changed

10 files changed

+3512
-78
lines changed

‎x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_eql_search_request.test.ts‎

Lines changed: 83 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ describe('buildEqlSearchRequest', () => {
5757
],
5858
},
5959
},
60+
allow_partial_search_results: true,
6061
fields: [
6162
{
6263
field: '*',
@@ -141,6 +142,7 @@ describe('buildEqlSearchRequest', () => {
141142
],
142143
},
143144
},
145+
allow_partial_search_results: true,
144146
fields: [
145147
{
146148
field: '*',
@@ -198,6 +200,7 @@ describe('buildEqlSearchRequest', () => {
198200
],
199201
},
200202
},
203+
allow_partial_search_results: true,
201204
fields: [
202205
{
203206
field: '*',
@@ -235,97 +238,98 @@ describe('buildEqlSearchRequest', () => {
235238
exceptionFilter: filter,
236239
});
237240
expect(request).toMatchInlineSnapshot(`
238-
Object {
239-
"allow_no_indices": true,
240-
"body": Object {
241-
"event_category_field": undefined,
242-
"fields": Array [
243-
Object {
244-
"field": "*",
245-
"include_unmapped": true,
246-
},
247-
Object {
248-
"field": "@timestamp",
249-
"format": "strict_date_optional_time",
250-
},
251-
],
252-
"filter": Object {
253-
"bool": Object {
254-
"filter": Array [
255-
Object {
256-
"range": Object {
257-
"@timestamp": Object {
258-
"format": "strict_date_optional_time",
259-
"gte": "now-5m",
260-
"lte": "now",
261-
},
241+
Object {
242+
"allow_no_indices": true,
243+
"body": Object {
244+
"allow_partial_search_results": true,
245+
"event_category_field": undefined,
246+
"fields": Array [
247+
Object {
248+
"field": "*",
249+
"include_unmapped": true,
250+
},
251+
Object {
252+
"field": "@timestamp",
253+
"format": "strict_date_optional_time",
254+
},
255+
],
256+
"filter": Object {
257+
"bool": Object {
258+
"filter": Array [
259+
Object {
260+
"range": Object {
261+
"@timestamp": Object {
262+
"format": "strict_date_optional_time",
263+
"gte": "now-5m",
264+
"lte": "now",
262265
},
263266
},
264-
Object {
265-
"bool": Object {
266-
"filter": Array [],
267-
"must": Array [],
268-
"must_not": Array [
269-
Object {
270-
"bool": Object {
271-
"should": Array [
272-
Object {
273-
"bool": Object {
274-
"filter": Array [
275-
Object {
276-
"nested": Object {
277-
"path": "some.parentField",
278-
"query": Object {
279-
"bool": Object {
280-
"minimum_should_match": 1,
281-
"should": Array [
282-
Object {
283-
"match_phrase": Object {
284-
"some.parentField.nested.field": "some value",
285-
},
267+
},
268+
Object {
269+
"bool": Object {
270+
"filter": Array [],
271+
"must": Array [],
272+
"must_not": Array [
273+
Object {
274+
"bool": Object {
275+
"should": Array [
276+
Object {
277+
"bool": Object {
278+
"filter": Array [
279+
Object {
280+
"nested": Object {
281+
"path": "some.parentField",
282+
"query": Object {
283+
"bool": Object {
284+
"minimum_should_match": 1,
285+
"should": Array [
286+
Object {
287+
"match_phrase": Object {
288+
"some.parentField.nested.field": "some value",
286289
},
287-
],
288-
},
290+
},
291+
],
289292
},
290-
"score_mode": "none",
291293
},
294+
"score_mode": "none",
292295
},
293-
Object {
294-
"bool": Object {
295-
"minimum_should_match": 1,
296-
"should": Array [
297-
Object {
298-
"match_phrase": Object {
299-
"some.not.nested.field": "some value",
300-
},
296+
},
297+
Object {
298+
"bool": Object {
299+
"minimum_should_match": 1,
300+
"should": Array [
301+
Object {
302+
"match_phrase": Object {
303+
"some.not.nested.field": "some value",
301304
},
302-
],
303-
},
305+
},
306+
],
304307
},
305-
],
306-
},
308+
},
309+
],
307310
},
308-
],
309-
},
311+
},
312+
],
310313
},
311-
],
312-
"should": Array [],
313-
},
314+
},
315+
],
316+
"should": Array [],
314317
},
315-
],
316-
},
318+
},
319+
],
317320
},
318-
"query": "process where true",
319-
"runtime_mappings": undefined,
320-
"size": 100,
321-
"timestamp_field": undefined,
322321
},
323-
"index": Array [
324-
"testindex1",
325-
"testindex2",
326-
],
327-
}
328-
`);
322+
"query": "process where true",
323+
"runtime_mappings": undefined,
324+
"size": 100,
325+
"timestamp_field": undefined,
326+
},
327+
"index": Array [
328+
"testindex1",
329+
"testindex2",
330+
],
331+
}
332+
`);
329333
});
330334

331335
test('should build a request with filters', () => {
@@ -415,6 +419,7 @@ describe('buildEqlSearchRequest', () => {
415419
],
416420
},
417421
},
422+
allow_partial_search_results: true,
418423
fields: [
419424
{
420425
field: '*',

‎x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_eql_search_request.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,12 @@ export const buildEqlSearchRequest = ({
8787
filter: requestFilter,
8888
},
8989
},
90+
// the allow_partial_search_results query parameter will supersede
91+
// the corresponding xpack settings on cluster
92+
// @ts-expect-error unknown property allow_partial_search_results
93+
// TODO: remove this ts-expect when 8.18 elasticsearch client is released.
94+
// issue: https://github.com/elastic/kibana/issues/208760
95+
allow_partial_search_results: true,
9096
runtime_mappings: runtimeMappings,
9197
timestamp_field: timestampField,
9298
event_category_field: eventCategoryOverride,

‎x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ import type {
1515
} from '@kbn/alerting-plugin/server';
1616
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
1717
import type { Filter } from '@kbn/es-query';
18+
import isEmpty from 'lodash/isEmpty';
19+
1820
import { buildEqlSearchRequest } from './build_eql_search_request';
1921
import { createEnrichEventsFunction } from '../utils/enrichments';
2022

@@ -55,6 +57,8 @@ import type { RulePreviewLoggedRequest } from '../../../../../common/api/detecti
5557
import { logEqlRequest } from '../utils/logged_requests';
5658
import * as i18n from '../translations';
5759
import { alertSuppressionTypeGuard } from '../utils/get_is_alert_suppression_active';
60+
import { isEqlSequenceQuery } from '../../../../../common/detection_engine/utils';
61+
import { logShardFailures } from '../utils/log_shard_failure';
5862

5963
interface EqlExecutorParams {
6064
inputIndex: string[];
@@ -120,6 +124,8 @@ export const eqlExecutor = async ({
120124
uiSettingsClient: services.uiSettingsClient,
121125
});
122126

127+
const isSequenceQuery = isEqlSequenceQuery(ruleParams.query);
128+
123129
const request = buildEqlSearchRequest({
124130
query: ruleParams.query,
125131
index: inputIndex,
@@ -165,6 +171,16 @@ export const eqlExecutor = async ({
165171

166172
let newSignals: Array<WrappedFieldsLatest<BaseFieldsLatest>> | undefined;
167173

174+
// @ts-expect-error shard_failures exists in
175+
// elasticsearch response v9
176+
// needs to be spec needs to be backported
177+
// https://github.com/elastic/elasticsearch-specification/pull/3372#issuecomment-2621835599
178+
// TODO: remove ts-expect-error when ES lib version is updated
179+
const shardFailures = response.shard_failures;
180+
if (!isEmpty(shardFailures)) {
181+
logShardFailures(isSequenceQuery, shardFailures, result, ruleExecutionLogger);
182+
}
183+
168184
const { events, sequences } = response.hits;
169185

170186
if (events) {

‎x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/translations.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,27 @@ export const EQL_SEARCH_REQUEST_DESCRIPTION = i18n.translate(
2828
}
2929
);
3030

31+
export const EQL_SHARD_FAILURE_MESSAGE = (
32+
isEqlSequenceQuery: boolean,
33+
shardFailuresMessage: string
34+
) =>
35+
isEqlSequenceQuery
36+
? i18n.translate(
37+
'xpack.securitySolution.detectionEngine.eqlSequenceRuleType.eqlShardFailures',
38+
{
39+
defaultMessage: `The EQL query failed to run successfully due to unavailable shards: {shardFailures}`,
40+
values: {
41+
shardFailures: shardFailuresMessage,
42+
},
43+
}
44+
)
45+
: i18n.translate('xpack.securitySolution.detectionEngine.eqlEventRuleType.eqlShardFailures', {
46+
defaultMessage: `The EQL event query was only executed on the available shards. The query failed to run successfully on the following shards: {shardFailures}`,
47+
values: {
48+
shardFailures: shardFailuresMessage,
49+
},
50+
});
51+
3152
export const FIND_THRESHOLD_BUCKETS_DESCRIPTION = (afterBucket?: string) =>
3253
afterBucket
3354
? i18n.translate(

‎x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/log_shard_failure.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import type { ShardFailure } from '@elastic/elasticsearch/lib/api/types';
9+
import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring';
10+
import type { SearchAfterAndBulkCreateReturnType } from '../types';
11+
import * as i18n from '../translations';
12+
13+
export const logShardFailures = (
14+
isSequenceQuery: boolean,
15+
shardFailures: ShardFailure[],
16+
result: SearchAfterAndBulkCreateReturnType,
17+
ruleExecutionLogger: IRuleExecutionLogForExecutors
18+
) => {
19+
const shardFailureMessage = i18n.EQL_SHARD_FAILURE_MESSAGE(
20+
isSequenceQuery,
21+
JSON.stringify(shardFailures)
22+
);
23+
ruleExecutionLogger.error(shardFailureMessage);
24+
if (isSequenceQuery) {
25+
result.errors.push(shardFailureMessage);
26+
} else {
27+
result.warningMessages.push(shardFailureMessage);
28+
}
29+
};

‎x-pack/test/security_solution_api_integration/es_archive/serverless/packetbeat/default/data.json.gz‎

76.7 KB
Binary file not shown.

0 commit comments

Comments
 (0)