Skip to content

Commit 0d2930b

Browse files
jgowdyelastickibanamachinestratoula
authored
ES|QL pattern formatting (#222871)
Adds a recommended query for the `CATEGORIZE` function in ES|QL. Adds keyword highlighting for the patterns and the ability to open a new Discover tab to filter for docs which match the selected pattern. https://github.com/user-attachments/assets/9ed8c5b0-7e92-4cc8-88dd-cb7749b5ffd3 --------- Co-authored-by: kibanamachine <[email protected]> Co-authored-by: Stratoula Kalafateli <[email protected]>
1 parent 831004d commit 0d2930b

File tree

49 files changed

+1112
-61
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+1112
-61
lines changed

.github/CODEOWNERS

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -410,6 +410,7 @@ src/platform/packages/shared/home/sample_data_card @elastic/appex-sharedux
410410
src/platform/packages/shared/home/sample_data_tab @elastic/appex-sharedux
411411
src/platform/packages/shared/home/sample_data_types @elastic/appex-sharedux
412412
src/platform/packages/shared/kbn-actions-types @elastic/response-ops
413+
src/platform/packages/shared/kbn-aiops-utils @elastic/ml-ui
413414
src/platform/packages/shared/kbn-alerting-types @elastic/response-ops
414415
src/platform/packages/shared/kbn-alerts-as-data-utils @elastic/response-ops
415416
src/platform/packages/shared/kbn-alerts-ui-shared @elastic/response-ops
@@ -1220,6 +1221,7 @@ x-pack/test_serverless/api_integration/test_suites/common/platform_security @ela
12201221
/x-pack/test_serverless/functional/test_suites/common/examples/search_examples @elastic/kibana-data-discovery
12211222
/x-pack/test_serverless/functional/test_suites/common/examples/unified_field_list_examples @elastic/kibana-data-discovery
12221223
/x-pack/test_serverless/functional/test_suites/common/management/data_views @elastic/kibana-data-discovery
1224+
src/platform/plugins/shared/discover/public/context_awareness/profile_providers/common/patterns @elastic/ml-ui
12231225
src/platform/plugins/shared/discover/public/context_awareness/profile_providers/security @elastic/kibana-data-discovery @elastic/security-threat-hunting-investigations
12241226
# TODO: this deprecation_logs folder should be owned by kibana management team after 9.0
12251227
src/platform/plugins/shared/discover/public/context_awareness/profile_providers/common/deprecation_logs @elastic/kibana-data-discovery @elastic/kibana-core
@@ -2836,11 +2838,11 @@ src/platform/testfunctional/page_objects/solution_navigation.ts @elastic/appex-s
28362838
/x-pack/test/api_integration/deployment_agnostic/apis/intercepts/*.ts @elastic/appex-sharedux
28372839

28382840
# OpenAPI spec files
2839-
oas_docs/linters @elastic/core-docs @elastic/experience-docs
2840-
oas_docs/overlays @elastic/core-docs @elastic/experience-docs
2841-
oas_docs/kibana.info.serverless.yaml @elastic/core-docs @elastic/experience-docs
2842-
oas_docs/kibana.info.yaml @elastic/core-docs @elastic/experience-docs
2843-
oas_docs/output @elastic/core-docs @elastic/experience-docs
2841+
oas_docs/linters @elastic/core-docs @elastic/experience-docs
2842+
oas_docs/overlays @elastic/core-docs @elastic/experience-docs
2843+
oas_docs/kibana.info.serverless.yaml @elastic/core-docs @elastic/experience-docs
2844+
oas_docs/kibana.info.yaml @elastic/core-docs @elastic/experience-docs
2845+
oas_docs/output @elastic/core-docs @elastic/experience-docs
28442846

28452847
# Documentation settings files
28462848
docs/settings-gen @elastic/platform-docs @elastic/experience-docs

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,7 @@
183183
"@kbn/aiops-log-rate-analysis": "link:x-pack/platform/packages/shared/ml/aiops_log_rate_analysis",
184184
"@kbn/aiops-plugin": "link:x-pack/platform/plugins/shared/aiops",
185185
"@kbn/aiops-test-utils": "link:x-pack/platform/packages/private/ml/aiops_test_utils",
186+
"@kbn/aiops-utils": "link:src/platform/packages/shared/kbn-aiops-utils",
186187
"@kbn/alerting-api-integration-test-plugin": "link:x-pack/platform/test/alerting_api_integration/common/plugins/alerts",
187188
"@kbn/alerting-comparators": "link:x-pack/platform/packages/shared/kbn-alerting-comparators",
188189
"@kbn/alerting-example-plugin": "link:x-pack/examples/alerting_example",
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# @kbn/aiops-utils
2+
3+
AIOps utils for plugins which are not in x-pack
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
export {
11+
getCategorizationDataViewField,
12+
getCategorizationField,
13+
} from './src/get_categorization_field';
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
module.exports = {
11+
preset: '@kbn/test',
12+
rootDir: '../../../../..',
13+
roots: ['<rootDir>/src/platform/packages/shared/kbn-aiops-utils'],
14+
};
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"type": "shared-common",
3+
"id": "@kbn/aiops-utils",
4+
"owner": "@elastic/ml-ui",
5+
"group": "platform",
6+
"visibility": "shared"
7+
}
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"name": "@kbn/aiops-utils",
3+
"private": true,
4+
"version": "1.0.0",
5+
"license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0"
6+
}
Lines changed: 226 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,226 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
import { createStubDataView } from '@kbn/data-views-plugin/common/stubs';
11+
import { getCategorizationField } from './get_categorization_field';
12+
import { getCategorizationDataViewField } from './get_categorization_field';
13+
14+
describe('get_categorization_field utils', () => {
15+
describe('getCategorizationField', () => {
16+
it('returns "message" if present', () => {
17+
const fields = ['foo', 'bar', 'message', 'baz'];
18+
expect(getCategorizationField(fields)).toBe('message');
19+
});
20+
21+
it('returns "error.message" if present and "message" is not', () => {
22+
const fields = ['foo', 'error.message', 'baz'];
23+
expect(getCategorizationField(fields)).toBe('error.message');
24+
});
25+
26+
it('returns "event.original" if present and others are not', () => {
27+
const fields = ['event.original', 'foo', 'bar'];
28+
expect(getCategorizationField(fields)).toBe('event.original');
29+
});
30+
31+
it('returns first field if none of the priority fields are present', () => {
32+
const fields = ['foo', 'bar', 'baz'];
33+
expect(getCategorizationField(fields)).toBe('foo');
34+
});
35+
36+
it('returns first field, skipping meta data fields, if none of the priority fields are present', () => {
37+
const fields = ['_id', 'foo', 'bar', 'baz'];
38+
expect(getCategorizationField(fields)).toBe('foo');
39+
});
40+
41+
it('returns the first matching field according to priority', () => {
42+
const fields = ['event.original', 'error.message', 'message'];
43+
expect(getCategorizationField(fields)).toBe('message');
44+
});
45+
46+
it('handles empty array', () => {
47+
expect(getCategorizationField([])).toBeUndefined();
48+
});
49+
});
50+
51+
describe('getCategorizationDataViewField', () => {
52+
it('returns messageField as "message" if present', () => {
53+
const dataView = createStubDataView({
54+
spec: {
55+
id: 'test',
56+
fields: {
57+
foo: {
58+
searchable: false,
59+
aggregatable: false,
60+
name: 'foo',
61+
type: 'type',
62+
esTypes: ['keyword'],
63+
},
64+
message: {
65+
searchable: true,
66+
aggregatable: true,
67+
name: 'message',
68+
type: 'text',
69+
esTypes: ['text'],
70+
},
71+
'error.message': {
72+
searchable: true,
73+
aggregatable: true,
74+
name: 'error.message',
75+
type: 'text',
76+
esTypes: ['text'],
77+
},
78+
},
79+
},
80+
});
81+
const result = getCategorizationDataViewField(dataView);
82+
expect(result.messageField?.name).toBe('message');
83+
expect(result.dataViewFields.map((f) => f.name)).toContain('message');
84+
expect(result.dataViewFields.length).toBe(2);
85+
});
86+
it('returns messageField as "error.message" if "message" is not present', () => {
87+
const dataView = createStubDataView({
88+
spec: {
89+
id: 'test2',
90+
fields: {
91+
foo: {
92+
searchable: false,
93+
aggregatable: false,
94+
name: 'foo',
95+
type: 'type',
96+
esTypes: ['keyword'],
97+
},
98+
'error.message': {
99+
searchable: true,
100+
aggregatable: true,
101+
name: 'error.message',
102+
type: 'text',
103+
esTypes: ['text'],
104+
},
105+
bar: {
106+
searchable: true,
107+
aggregatable: true,
108+
name: 'bar',
109+
type: 'text',
110+
esTypes: ['text'],
111+
},
112+
},
113+
},
114+
});
115+
const result = getCategorizationDataViewField(dataView);
116+
expect(result.messageField?.name).toBe('error.message');
117+
expect(result.dataViewFields.map((f) => f.name)).toContain('error.message');
118+
expect(result.dataViewFields.length).toBe(2);
119+
});
120+
121+
it('handles empty fields array', () => {
122+
const dataView = createStubDataView({
123+
spec: {
124+
id: 'test6',
125+
fields: {},
126+
},
127+
});
128+
const result = getCategorizationDataViewField(dataView);
129+
expect(result.messageField).toBeNull();
130+
expect(result.dataViewFields.length).toBe(0);
131+
});
132+
133+
it('returns the first matching field according to priority', () => {
134+
const dataView = createStubDataView({
135+
spec: {
136+
id: 'test7',
137+
fields: {
138+
'event.original': {
139+
searchable: true,
140+
aggregatable: true,
141+
name: 'event.original',
142+
type: 'text',
143+
esTypes: ['text'],
144+
},
145+
'error.message': {
146+
searchable: true,
147+
aggregatable: true,
148+
name: 'error.message',
149+
type: 'text',
150+
esTypes: ['text'],
151+
},
152+
message: {
153+
searchable: true,
154+
aggregatable: true,
155+
name: 'message',
156+
type: 'text',
157+
esTypes: ['text'],
158+
},
159+
},
160+
},
161+
});
162+
const result = getCategorizationDataViewField(dataView);
163+
expect(result.messageField?.name).toBe('message');
164+
expect(result.dataViewFields.map((f) => f.name)).toEqual(
165+
expect.arrayContaining(['message', 'error.message', 'event.original'])
166+
);
167+
expect(result.dataViewFields.length).toBe(3);
168+
});
169+
it('returns the first field if no priority fields are present', () => {
170+
const dataView = createStubDataView({
171+
spec: {
172+
id: 'test8',
173+
fields: {
174+
foo: {
175+
searchable: true,
176+
aggregatable: true,
177+
name: 'foo',
178+
type: 'text',
179+
esTypes: ['text'],
180+
},
181+
bar: {
182+
searchable: true,
183+
aggregatable: true,
184+
name: 'bar',
185+
type: 'text',
186+
esTypes: ['text'],
187+
},
188+
},
189+
},
190+
});
191+
const result = getCategorizationDataViewField(dataView);
192+
expect(result.messageField?.name).toBe('foo');
193+
expect(result.dataViewFields.map((f) => f.name)).toEqual(
194+
expect.arrayContaining(['foo', 'bar'])
195+
);
196+
expect(result.dataViewFields.length).toBe(2);
197+
});
198+
it('returns null messageField if no text fields are present', () => {
199+
const dataView = createStubDataView({
200+
spec: {
201+
id: 'test9',
202+
fields: {
203+
foo: {
204+
searchable: true,
205+
aggregatable: true,
206+
name: 'foo',
207+
type: 'keyword',
208+
esTypes: ['keyword'],
209+
},
210+
bar: {
211+
searchable: true,
212+
aggregatable: true,
213+
name: 'bar',
214+
type: 'keyword',
215+
esTypes: ['keyword'],
216+
},
217+
},
218+
},
219+
});
220+
const result = getCategorizationDataViewField(dataView);
221+
expect(result.messageField).toBeNull();
222+
expect(result.dataViewFields.map((f) => f.name)).toEqual(expect.arrayContaining([]));
223+
expect(result.dataViewFields.length).toBe(0);
224+
});
225+
});
226+
});
Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
import type { DataView, DataViewField } from '@kbn/data-views-plugin/public';
11+
import { ES_FIELD_TYPES } from '@kbn/field-types';
12+
13+
const FIELD_PRIORITY = ['message', 'error.message', 'event.original'];
14+
15+
const METADATA_FIELDS = [
16+
'_version',
17+
'_id',
18+
'_index',
19+
'_source',
20+
'_ignored',
21+
'_index_mode',
22+
'_score',
23+
];
24+
25+
/**
26+
* This function returns the categorization field from the list of fields.
27+
* It checks for the presence of 'message', 'error.message', or 'event.original' in that order.
28+
* If none of these fields are present, it returns the first field from the list,
29+
* Assumes text fields have been passed in the `fields` array.
30+
*
31+
* @param fields, the list of fields to check
32+
* @returns string | undefined, the categorization field if found, otherwise undefined
33+
*/
34+
export function getCategorizationField(fields: string[]): string | undefined {
35+
const fieldSet = new Set(fields);
36+
for (const field of FIELD_PRIORITY) {
37+
if (fieldSet.has(field)) {
38+
return field;
39+
}
40+
}
41+
42+
// Filter out metadata fields
43+
const filteredFields = fields.filter((field) => !METADATA_FIELDS.includes(field));
44+
return filteredFields[0] ?? undefined;
45+
}
46+
47+
/**
48+
* This function returns the categorization field from the DataView.
49+
* It checks for the presence of 'message', 'error.message', or 'event.original' in that order.
50+
* If none of these fields are present, it returns the first text field from the DataView.
51+
*
52+
* @param dataView, the DataView to check
53+
* @returns an object containing the message field DataViewField and dataViewFields
54+
*/
55+
export function getCategorizationDataViewField(dataView: DataView): {
56+
messageField: DataViewField | null;
57+
dataViewFields: DataViewField[];
58+
} {
59+
const dataViewFields = dataView.fields.filter((f) => f.esTypes?.includes(ES_FIELD_TYPES.TEXT));
60+
const categorizationFieldName = getCategorizationField(dataViewFields.map((f) => f.name));
61+
if (categorizationFieldName) {
62+
return {
63+
messageField: dataView.fields.getByName(categorizationFieldName) ?? null,
64+
dataViewFields,
65+
};
66+
}
67+
68+
return {
69+
messageField: null,
70+
dataViewFields,
71+
};
72+
}

0 commit comments

Comments
 (0)