Skip to content

Commit 74e605d

Browse files
benakansarajustinkambic
benakansara
and
justinkambic
authored
Add new logs dataset and example configs (#240334)
Adds a simple dataset for generating success and error log messages. The dataset and configs are intended to generate data on which we can generate active/recovered/flapping alerts using ES|QL rule. ### Run example configs `node x-pack/scripts/data_forge.js --config x-pack/platform/packages/shared/kbn-data-forge/example_config/esql_poc/esql_active_recovered_alert.yaml` `node x-pack/scripts/data_forge.js --config x-pack/platform/packages/shared/kbn-data-forge/example_config/esql_poc/esql_flapping_alert.yaml` Co-authored-by: Justin Kambic <[email protected]>
1 parent 6b8f782 commit 74e605d

File tree

8 files changed

+362
-2
lines changed

8 files changed

+362
-2
lines changed

‎x-pack/platform/packages/shared/kbn-data-forge/example_config/success_and_error_log_messages/esql_active_recovered_alert.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
---
2+
elasticsearch:
3+
installKibanaUser: false
4+
5+
kibana:
6+
installAssets: true
7+
8+
indexing:
9+
eventsPerCycle: 1
10+
dataset: "database_logs"
11+
12+
schedule:
13+
- template: "bad"
14+
start: "now-2m"
15+
end: "now+2m"
16+
eventsPerCycle: 15
17+
randomness: 0.1
18+
- template: "good"
19+
start: "now+2m"
20+
end: "now+5m"
21+
randomness: 0.1
22+
- template: "bad"
23+
start: "now+5m"
24+
end: "now+8m"
25+
eventsPerCycle: 5
26+
randomness: 0.1
27+
- template: "good"
28+
start: "now+8m"
29+
end: "now+10m"
30+
randomness: 0.1
31+

‎x-pack/platform/packages/shared/kbn-data-forge/example_config/success_and_error_log_messages/esql_flapping_alert.yaml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
---
2+
elasticsearch:
3+
installKibanaUser: false
4+
5+
kibana:
6+
installAssets: true
7+
8+
indexing:
9+
eventsPerCycle: 1
10+
dataset: "database_logs"
11+
12+
schedule:
13+
- template: "bad"
14+
start: "now-2m"
15+
end: "now+2m"
16+
eventsPerCycle: 15
17+
randomness: 0.1
18+
- template: "good"
19+
start: "now+2m"
20+
end: "now+5m"
21+
randomness: 0.1
22+
- template: "bad"
23+
start: "now+5m"
24+
end: "now+8m"
25+
eventsPerCycle: 5
26+
randomness: 0.1
27+
- template: "good"
28+
start: "now+8m"
29+
end: "now+10m"
30+
randomness: 0.1
31+
- template: "bad"
32+
start: "now+10m"
33+
end: "now+14m"
34+
eventsPerCycle: 15
35+
randomness: 0.1
36+
- template: "good"
37+
start: "now+14m"
38+
end: "now+17m"
39+
randomness: 0.1
40+
- template: "bad"
41+
start: "now+17m"
42+
end: "now+20m"
43+
eventsPerCycle: 10
44+
randomness: 0.1
45+
- template: "good"
46+
start: "now+20m"
47+
end: "now+25m"
48+
randomness: 0.1
49+
- template: "bad"
50+
start: "now+25m"
51+
end: "now+30m"
52+
eventsPerCycle: 20
53+
randomness: 0.1
54+

‎x-pack/platform/packages/shared/kbn-data-forge/src/constants.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ export const FAKE_HOSTS = 'fake_hosts';
99
export const FAKE_LOGS = 'fake_logs';
1010
export const FAKE_STACK = 'fake_stack';
1111
export const SERVICE_LOGS = 'service.logs';
12+
export const DATABASE_LOGS = 'database_logs';
1213

1314
export const INDEX_PREFIX = 'kbn-data-forge';
1415

‎x-pack/platform/packages/shared/kbn-data-forge/src/data_sources/database_logs/ecs/index.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import template from '../template.json';
9+
import type { IndexTemplateDef } from '../../../types';
10+
11+
export const indexTemplate: IndexTemplateDef = {
12+
name: 'logs-database_logs@template',
13+
template,
14+
components: [],
15+
};

‎x-pack/platform/packages/shared/kbn-data-forge/src/data_sources/database_logs/index.ts‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { random, sample } from 'lodash';
9+
import type { GeneratorFunction } from '../../types';
10+
export { indexTemplate } from './ecs';
11+
12+
const createGroupIndex = (index: number) => Math.floor(index / 1000) * 1000;
13+
14+
const DATABASE_ERROR_MESSAGES = [
15+
'Connection timeout: database error occurred while connecting to primary server',
16+
'Query failed: database error in SELECT statement execution',
17+
'Transaction rollback: database error detected, rolling back changes',
18+
'Deadlock detected: database error in concurrent transaction processing',
19+
'Connection pool exhausted: database error - no available connections',
20+
'Constraint violation: database error in data integrity check',
21+
'Replication lag: database error in master-slave synchronization',
22+
'Index corruption: database error detected in table indexes',
23+
'Lock timeout: database error waiting for resource lock',
24+
'Authentication failed: database error in user credential verification',
25+
];
26+
27+
const SUCCESS_MESSAGES = [
28+
'Database connection established successfully',
29+
'Query executed successfully: SELECT * FROM users WHERE active = true',
30+
'Transaction committed successfully',
31+
'Data backup completed successfully',
32+
'Index optimization completed successfully',
33+
'Replication sync completed successfully',
34+
'Database health check passed',
35+
'Connection pool initialized successfully',
36+
'User authentication successful',
37+
'Cache refresh completed successfully',
38+
];
39+
40+
const DATABASE_HOSTS = [
41+
'db-primary-01',
42+
'db-primary-02',
43+
'db-replica-01',
44+
'db-replica-02',
45+
'db-analytics-01',
46+
'db-cache-01',
47+
'db-warehouse-01',
48+
];
49+
50+
export const generateEvent: GeneratorFunction = (config, schedule, index, timestamp) => {
51+
const groupIndex = createGroupIndex(index);
52+
const latency = random(50, 2000);
53+
const interval = schedule.interval ?? config.indexing.interval;
54+
const isError = schedule.template === 'error' || schedule.template === 'bad';
55+
const hostName = sample(DATABASE_HOSTS) || 'db-primary-01';
56+
57+
return [
58+
{
59+
namespace: 'database_logs',
60+
'@timestamp': timestamp.toISOString(),
61+
event: {
62+
module: 'database',
63+
dataset: 'database.logs',
64+
duration: latency,
65+
code: isError ? random(400, 599) : 200,
66+
category: isError ? 'database_error' : 'database_success',
67+
type: isError ? 'error' : 'info',
68+
},
69+
log: {
70+
level: isError ? 'error' : 'info',
71+
logger: 'database_logs',
72+
},
73+
host: {
74+
name: hostName,
75+
},
76+
service: {
77+
name: 'database-service',
78+
type: 'database',
79+
},
80+
labels: {
81+
groupId: `group-${groupIndex}`,
82+
eventId: `event-${index}`,
83+
scenario: schedule.template,
84+
database_type: sample(['postgresql', 'mysql', 'mongodb', 'elasticsearch']) || 'postgresql',
85+
},
86+
metricset: {
87+
period: interval,
88+
},
89+
message: isError
90+
? sample(DATABASE_ERROR_MESSAGES) || 'Unknown database error occurred'
91+
: sample(SUCCESS_MESSAGES) || 'Database operation completed successfully',
92+
database: {
93+
query_time: latency,
94+
connection_id: `conn_${random(1000, 9999)}`,
95+
query_type: sample(['SELECT', 'INSERT', 'UPDATE', 'DELETE']) || 'SELECT',
96+
},
97+
},
98+
];
99+
};

‎x-pack/platform/packages/shared/kbn-data-forge/src/data_sources/database_logs/template.json‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
{
2+
"_meta": {
3+
"description": "Database logs composable template for ES|QL testing",
4+
"version": "1.0.0"
5+
},
6+
"index_patterns": [
7+
"kbn-data-forge-database_logs.database_logs-*"
8+
],
9+
"priority": 1,
10+
"template": {
11+
"mappings": {
12+
"_meta": {
13+
"version": "1.0.0"
14+
},
15+
"date_detection": false,
16+
"dynamic_templates": [
17+
{
18+
"labels": {
19+
"path_match": "labels.*",
20+
"mapping": {
21+
"type": "keyword"
22+
},
23+
"match_mapping_type": "string"
24+
}
25+
},
26+
{
27+
"strings_as_keyword": {
28+
"mapping": {
29+
"ignore_above": 1024,
30+
"type": "keyword"
31+
},
32+
"match_mapping_type": "string"
33+
}
34+
}
35+
],
36+
"properties": {
37+
"@timestamp": {
38+
"type": "date"
39+
},
40+
"metricset": {
41+
"properties": {
42+
"period": {
43+
"type": "long"
44+
}
45+
}
46+
},
47+
"host": {
48+
"properties": {
49+
"name": {
50+
"type": "keyword",
51+
"ignore_above": 256
52+
}
53+
}
54+
},
55+
"service": {
56+
"properties": {
57+
"name": {
58+
"type": "keyword",
59+
"ignore_above": 256
60+
},
61+
"type": {
62+
"type": "keyword",
63+
"ignore_above": 256
64+
}
65+
}
66+
},
67+
"event": {
68+
"properties": {
69+
"dataset": {
70+
"type": "keyword",
71+
"ignore_above": 256
72+
},
73+
"module": {
74+
"type": "keyword",
75+
"ignore_above": 256
76+
},
77+
"category": {
78+
"type": "keyword",
79+
"ignore_above": 256
80+
},
81+
"type": {
82+
"type": "keyword",
83+
"ignore_above": 256
84+
},
85+
"code": {
86+
"type": "long"
87+
},
88+
"duration": {
89+
"type": "long"
90+
}
91+
}
92+
},
93+
"log": {
94+
"properties": {
95+
"level": {
96+
"type": "keyword",
97+
"ignore_above": 256
98+
},
99+
"logger": {
100+
"type": "keyword",
101+
"ignore_above": 256
102+
}
103+
}
104+
},
105+
"message": {
106+
"type": "text",
107+
"fields": {
108+
"keyword": {
109+
"type": "keyword",
110+
"ignore_above": 256
111+
}
112+
}
113+
},
114+
"database": {
115+
"properties": {
116+
"query_time": {
117+
"type": "long"
118+
},
119+
"connection_id": {
120+
"type": "keyword",
121+
"ignore_above": 256
122+
},
123+
"query_type": {
124+
"type": "keyword",
125+
"ignore_above": 256
126+
}
127+
}
128+
}
129+
}
130+
},
131+
"settings": {
132+
"index": {
133+
"mapping": {
134+
"total_fields": {
135+
"limit": "2000"
136+
}
137+
},
138+
"number_of_shards": "1",
139+
"number_of_replicas": "0",
140+
"query": {
141+
"default_field": [
142+
"message",
143+
"labels.*",
144+
"event.*",
145+
"host.name",
146+
"service.*"
147+
]
148+
}
149+
}
150+
},
151+
"aliases": {
152+
"logs-database_logs": {}
153+
}
154+
}
155+
}

0 commit comments

Comments
 (0)