Skip to content

Commit 6f9b007

Browse files
kibanamachineJacek Kolezynskimaximpn
authored
[8.19] [Security Solution] Add support for arrays in the build_ebt_data_views script (elastic#234905) (elastic#235312)
# Backport This will backport the following commits from `main` to `8.19`: - [[Security Solution] Add support for arrays in the build_ebt_data_views script (elastic#234905)](elastic#234905) <!--- Backport version: 9.6.6 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport) <!--BACKPORT [{"author":{"name":"Jacek Kolezynski","email":"[email protected]"},"sourceCommit":{"committedDate":"2025-09-17T07:58:32Z","message":"[Security Solution] Add support for arrays in the build_ebt_data_views script (elastic#234905)\n\n**Partially resolves: elastic#140369**\n\n## Summary\n\nThis PR is a follow up for the elastic#234571, where I am introducing telemetry\nevent with array of primitive string values.\nIn order to display these values in Kibana Lens, a runtime mapping needs\nto be done in the `security-solution-ebt-kibana-server` data view.\nBefore that, it was done manually. I am introducing support for arrays,\nso that manual intervention is not needed for them.\n\n\n### Checklist\n\n- [x] The PR description includes the appropriate Release Notes section,\nand the correct `release_note:*` label is applied per the\n[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)\n\n---------\n\nCo-authored-by: Maxim Palenov <[email protected]>","sha":"f0e38a357dfbfe535ca77438d7afabde9c9b9cf2","branchLabelMapping":{"^v9.2.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Team:Detections and Resp","Team: SecuritySolution","Team:Detection Rule Management","Feature:Prebuilt Detection Rules","backport:version","v9.2.0","v8.18.8","v8.19.5","v9.0.8","v9.1.5"],"title":"[Security Solution] Add support for arrays in the build_ebt_data_views script","number":234905,"url":"https://github.com/elastic/kibana/pull/234905","mergeCommit":{"message":"[Security Solution] Add support for arrays in the build_ebt_data_views script (elastic#234905)\n\n**Partially resolves: elastic#140369**\n\n## Summary\n\nThis PR is a follow up for the elastic#234571, where I am introducing telemetry\nevent with array of primitive string values.\nIn order to display these values in Kibana Lens, a runtime mapping needs\nto be done in the `security-solution-ebt-kibana-server` data view.\nBefore that, it was done manually. I am introducing support for arrays,\nso that manual intervention is not needed for them.\n\n\n### Checklist\n\n- [x] The PR description includes the appropriate Release Notes section,\nand the correct `release_note:*` label is applied per the\n[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)\n\n---------\n\nCo-authored-by: Maxim Palenov <[email protected]>","sha":"f0e38a357dfbfe535ca77438d7afabde9c9b9cf2"}},"sourceBranch":"main","suggestedTargetBranches":["8.18","8.19","9.0","9.1"],"targetPullRequestStates":[{"branch":"main","label":"v9.2.0","branchLabelMappingKey":"^v9.2.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/234905","number":234905,"mergeCommit":{"message":"[Security Solution] Add support for arrays in the build_ebt_data_views script (elastic#234905)\n\n**Partially resolves: elastic#140369**\n\n## Summary\n\nThis PR is a follow up for the elastic#234571, where I am introducing telemetry\nevent with array of primitive string values.\nIn order to display these values in Kibana Lens, a runtime mapping needs\nto be done in the `security-solution-ebt-kibana-server` data view.\nBefore that, it was done manually. I am introducing support for arrays,\nso that manual intervention is not needed for them.\n\n\n### Checklist\n\n- [x] The PR description includes the appropriate Release Notes section,\nand the correct `release_note:*` label is applied per the\n[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)\n\n---------\n\nCo-authored-by: Maxim Palenov <[email protected]>","sha":"f0e38a357dfbfe535ca77438d7afabde9c9b9cf2"}},{"branch":"8.18","label":"v8.18.8","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"8.19","label":"v8.19.5","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"9.0","label":"v9.0.8","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"9.1","label":"v9.1.5","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}] BACKPORT--> Co-authored-by: Jacek Kolezynski <[email protected]> Co-authored-by: Maxim Palenov <[email protected]>
1 parent 11bb3a9 commit 6f9b007

File tree

2 files changed

+447
-29
lines changed

2 files changed

+447
-29
lines changed
Lines changed: 377 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,377 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
/* eslint-disable @typescript-eslint/no-explicit-any */
8+
9+
import axios from 'axios';
10+
import { flattenSchema, upsertRuntimeFields } from './build_ebt_data_view';
11+
12+
jest.mock('axios', () => ({
13+
__esModule: true,
14+
default: {
15+
put: jest.fn(),
16+
},
17+
}));
18+
19+
describe('upsertRuntimeFields', () => {
20+
const url = 'http://fake_url';
21+
const headers = {
22+
Authorization: 'ApiKey abc',
23+
'kbn-xsrf': 'xxx',
24+
'Content-Type': 'application/json',
25+
};
26+
27+
beforeEach(() => {
28+
jest.resetAllMocks();
29+
(axios.put as jest.Mock).mockResolvedValue({});
30+
});
31+
32+
test('sends one PUT per string field with correct payload and headers', async () => {
33+
const fields = {
34+
a: 'keyword',
35+
'nested.b': 'long',
36+
'deep.x.y': 'date',
37+
};
38+
39+
await upsertRuntimeFields(fields, url, headers);
40+
41+
expect(axios.put).toHaveBeenCalledTimes(3);
42+
43+
const calls = (axios.put as jest.Mock).mock.calls.map(([callUrl, payload, opts]) => ({
44+
callUrl,
45+
name: payload.name,
46+
type: payload.runtimeField?.type,
47+
opts,
48+
}));
49+
50+
const names = new Set(calls.map((c) => c.name));
51+
const types = new Set(calls.map((c) => c.type));
52+
const urls = new Set(calls.map((c) => c.callUrl));
53+
const allHeadersOk = calls.every(
54+
(c) => JSON.stringify(c.opts?.headers) === JSON.stringify(headers)
55+
);
56+
57+
expect(names).toEqual(new Set(['properties.a', 'properties.nested.b', 'properties.deep.x.y']));
58+
expect(types).toEqual(new Set(['keyword', 'long', 'date']));
59+
expect(urls).toEqual(new Set([url]));
60+
expect(allHeadersOk).toBe(true);
61+
});
62+
63+
test('ignores non-string field values', async () => {
64+
const fields = {
65+
ok: 'ip',
66+
skipNull: null,
67+
skipObj: { t: 'keyword' },
68+
skipNum: 123,
69+
};
70+
71+
await upsertRuntimeFields(fields as any, url, headers);
72+
73+
expect(axios.put).toHaveBeenCalledTimes(1);
74+
const [callUrl, payload, opts] = (axios.put as jest.Mock).mock.calls[0];
75+
76+
expect(callUrl).toBe(url);
77+
expect(payload).toEqual({
78+
name: 'properties.ok',
79+
runtimeField: { type: 'ip' },
80+
});
81+
expect(opts).toEqual({ headers });
82+
});
83+
84+
test('handles dotted field names correctly', async () => {
85+
const fields = {
86+
'one.two.three': 'double',
87+
};
88+
89+
await upsertRuntimeFields(fields, url, headers);
90+
91+
const [, payload] = (axios.put as jest.Mock).mock.calls[0];
92+
expect(payload.name).toBe('properties.one.two.three');
93+
expect(payload.runtimeField.type).toBe('double');
94+
});
95+
96+
describe('flattenSchema ', () => {
97+
test('flattens root primitive fields', () => {
98+
const schema = {
99+
a: { type: 'keyword' },
100+
b: { type: 'long' },
101+
c: { type: 'date' },
102+
d: { type: 'ip' },
103+
e: { type: 'double' },
104+
f: { type: 'boolean' },
105+
g: { type: 'text' },
106+
h: { type: 'lookup' },
107+
i: { type: 'geo_point' },
108+
j: { type: 'composite' },
109+
};
110+
111+
const out = flattenSchema(schema);
112+
113+
expect(out).toEqual({
114+
a: 'keyword',
115+
b: 'long',
116+
c: 'date',
117+
d: 'ip',
118+
e: 'double',
119+
f: 'boolean',
120+
g: 'text',
121+
h: 'lookup',
122+
i: 'geo_point',
123+
j: 'composite',
124+
});
125+
});
126+
127+
test('flattens nested objects via properties', () => {
128+
const schema = {
129+
parent: {
130+
properties: {
131+
child: { type: 'ip' },
132+
inner: {
133+
properties: {
134+
leaf: { type: 'double' },
135+
flag: { type: 'boolean' },
136+
},
137+
},
138+
},
139+
},
140+
};
141+
142+
const out = flattenSchema(schema);
143+
144+
expect(out).toEqual({
145+
'parent.child': 'ip',
146+
'parent.inner.leaf': 'double',
147+
'parent.inner.flag': 'boolean',
148+
});
149+
});
150+
151+
test('handles object node without "properties" but with nested shape', () => {
152+
const schema = {
153+
abc: {
154+
foo: { type: 'keyword' },
155+
bar: {
156+
baz: { type: 'long' },
157+
},
158+
},
159+
};
160+
161+
const out = flattenSchema(schema);
162+
163+
expect(out).toEqual({
164+
'abc.foo': 'keyword',
165+
'abc.bar.baz': 'long',
166+
});
167+
});
168+
169+
test('when node has type and properties, type takes precedence and children are not expanded', () => {
170+
const schema = {
171+
node: {
172+
type: 'keyword',
173+
properties: {
174+
x: { type: 'long' },
175+
},
176+
},
177+
};
178+
179+
const out = flattenSchema(schema);
180+
181+
expect(out).toEqual({
182+
node: 'keyword',
183+
});
184+
});
185+
186+
test('deeply nested mixed primitives and objects', () => {
187+
const schema = {
188+
root: {
189+
properties: {
190+
a: { type: 'keyword' },
191+
obj: {
192+
properties: {
193+
b: { type: 'long' },
194+
c: { type: 'text' },
195+
d: {
196+
properties: {
197+
e: { type: 'date' },
198+
},
199+
},
200+
},
201+
},
202+
},
203+
},
204+
lone: { type: 'ip' },
205+
};
206+
207+
const out = flattenSchema(schema);
208+
209+
expect(out).toEqual({
210+
'root.a': 'keyword',
211+
'root.obj.b': 'long',
212+
'root.obj.c': 'text',
213+
'root.obj.d.e': 'date',
214+
lone: 'ip',
215+
});
216+
});
217+
218+
test('ignores non-object or null schema nodes', () => {
219+
const schema = {
220+
a: null,
221+
b: 42,
222+
c: 'str',
223+
d: undefined,
224+
};
225+
226+
const out = flattenSchema(schema);
227+
228+
expect(out).toEqual({});
229+
});
230+
231+
test('array of primitive types maps to item type', () => {
232+
const schema = {
233+
tags: { type: 'array', items: { type: 'keyword' } },
234+
counts: { type: 'array', items: { type: 'long' } },
235+
dates: { type: 'array', items: { type: 'date' } },
236+
};
237+
238+
const out = flattenSchema(schema);
239+
240+
expect(out).toEqual({
241+
tags: 'keyword',
242+
counts: 'long',
243+
dates: 'date',
244+
});
245+
});
246+
247+
test('array of objects flattens item properties with parent prefix', () => {
248+
const schema = {
249+
rules: {
250+
type: 'array',
251+
items: {
252+
properties: {
253+
name: { type: 'keyword' },
254+
score: { type: 'integer' },
255+
meta: {
256+
properties: {
257+
enabled: { type: 'boolean' },
258+
},
259+
},
260+
},
261+
},
262+
},
263+
};
264+
265+
const out = flattenSchema(schema);
266+
267+
expect(out).toEqual({
268+
'rules.name': 'keyword',
269+
'rules.score': 'integer',
270+
'rules.meta.enabled': 'boolean',
271+
});
272+
});
273+
274+
test('array without items is emitted as array', () => {
275+
const schema = {
276+
unknown: { type: 'array' },
277+
};
278+
279+
const out = flattenSchema(schema);
280+
281+
expect(out).toEqual({
282+
unknown: 'array',
283+
});
284+
});
285+
286+
test('array with unknown item shape is emitted as array', () => {
287+
const schema = {
288+
misc: { type: 'array', items: {} },
289+
};
290+
291+
const out = flattenSchema(schema);
292+
293+
expect(out).toEqual({
294+
misc: 'array',
295+
});
296+
});
297+
298+
test('nested arrays: array of objects containing array of primitives', () => {
299+
const schema = {
300+
groups: {
301+
type: 'array',
302+
items: {
303+
properties: {
304+
ids: { type: 'array', items: { type: 'long' } },
305+
labels: { type: 'array', items: { type: 'keyword' } },
306+
},
307+
},
308+
},
309+
};
310+
311+
const out = flattenSchema(schema);
312+
313+
expect(out).toEqual({
314+
'groups.ids': 'long',
315+
'groups.labels': 'keyword',
316+
});
317+
});
318+
319+
test('deeply nested arrays in objects and objects in arrays', () => {
320+
const schema = {
321+
container: {
322+
properties: {
323+
matrix: { type: 'array', items: { type: 'array', items: { type: 'double' } } },
324+
wrappers: {
325+
type: 'array',
326+
items: {
327+
properties: {
328+
item: {
329+
properties: {
330+
values: { type: 'array', items: { type: 'ip' } },
331+
},
332+
},
333+
},
334+
},
335+
},
336+
},
337+
},
338+
};
339+
340+
const out = flattenSchema(schema);
341+
342+
expect(out).toEqual({
343+
'container.matrix': 'array',
344+
'container.wrappers.item.values': 'ip',
345+
});
346+
});
347+
348+
test('mixed: object properties and arrays coexist', () => {
349+
const schema = {
350+
user: {
351+
properties: {
352+
name: { type: 'keyword' },
353+
roles: { type: 'array', items: { type: 'keyword' } },
354+
sessions: {
355+
type: 'array',
356+
items: {
357+
properties: {
358+
started_at: { type: 'date' },
359+
device: { type: 'text' },
360+
},
361+
},
362+
},
363+
},
364+
},
365+
};
366+
367+
const out = flattenSchema(schema);
368+
369+
expect(out).toEqual({
370+
'user.name': 'keyword',
371+
'user.roles': 'keyword',
372+
'user.sessions.started_at': 'date',
373+
'user.sessions.device': 'text',
374+
});
375+
});
376+
});
377+
});

0 commit comments

Comments
 (0)