Skip to content

Commit 028ad08

Browse files
harnish-crestharnish-crest
authored
[PHP-FPM] Add Integration Package with Process Data Stream (elastic#4652)
* Add process data stream * Update changelog.yml * minor changes in manifest.yml
1 parent 14194e9 commit 028ad08

File tree

17 files changed

+1398
-3
lines changed

17 files changed

+1398
-3
lines changed

‎packages/php_fpm/_dev/build/docs/README.md‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ PHP-FPM (FastCGI Process Manager) is a web tool used to speed up the performance
88

99
The PHP-FPM integration collects metrics data.
1010

11-
Metrics give you insight into the statistics of the PHP-FPM. Metrics data streams collected by the PHP-FPM integration include [pool](https://www.php.net/manual/en/fpm.status.php#:~:text=Basic%20information%20%2D%20Always%20displayed%20on%20the%20status%20page) so that the user could monitor and troubleshoot the performance of the PHP-FPM instances.
11+
Metrics give you insight into the statistics of the PHP-FPM. Metrics data streams collected by the PHP-FPM integration include [pool](https://www.php.net/manual/en/fpm.status.php#:~:text=Basic%20information%20%2D%20Always%20displayed%20on%20the%20status%20page) and [process](https://www.php.net/manual/en/fpm.status.php#:~:text=Per%2Dprocess%20information%20%2D%20only%20displayed%20in%20full%20output%20mode) so that the user can monitor and troubleshoot the performance of the PHP-FPM instances.
1212

1313
Note:
1414
- Users can monitor and see the metrics inside the ingested documents for PHP-FPM in the logs-* index pattern from `Discover`.
@@ -40,3 +40,11 @@ This is the `pool` data stream. `pool` data stream collects metrics related to t
4040
{{event "pool"}}
4141

4242
{{fields "pool"}}
43+
44+
### Process
45+
46+
This is the `process` data stream. `process` data stream collects metrics like request duration, content length, process state, etc.
47+
48+
{{event "process"}}
49+
50+
{{fields "process"}}

‎packages/php_fpm/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "0.2.0"
3+
changes:
4+
- description: PHP-FPM integration package with "Process" data stream.
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/4652
27
- version: "0.1.0"
38
changes:
49
- description: PHP-FPM integration package with "Pool" data stream.

‎packages/php_fpm/data_stream/process/_dev/test/pipeline/test-common-config.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
fields:
2+
tags:
3+
- preserve_original_event

‎packages/php_fpm/data_stream/process/_dev/test/pipeline/test-process-metrics.log‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"pool":"www","process manager":"dynamic","start time":1667299161,"start since":515234,"accepted conn":16647,"listen queue":0,"max listen queue":0,"listen queue len":0,"idle processes":1,"active processes":1,"total processes":2,"max active processes":1,"max children reached":0,"slow requests":0, "processes":{"pid":183251,"state":"Running","start time":1667299161,"start since":515234,"requests":8324,"request duration":208,"request method":"GET","request uri":"/status?json&full","content length":0,"user":"-","script":"-","last request cpu":0.00,"last request memory":0}}

‎packages/php_fpm/data_stream/process/_dev/test/pipeline/test-process-metrics.log-expected.json‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
{
2+
"expected": [
3+
{
4+
"ecs": {
5+
"version": "8.4.0"
6+
},
7+
"event": {
8+
"category": [
9+
"web",
10+
"configuration",
11+
"process"
12+
],
13+
"kind": "event",
14+
"module": "php_fpm",
15+
"original": "{\"pool\":\"www\",\"process manager\":\"dynamic\",\"start time\":1667299161,\"start since\":515234,\"accepted conn\":16647,\"listen queue\":0,\"max listen queue\":0,\"listen queue len\":0,\"idle processes\":1,\"active processes\":1,\"total processes\":2,\"max active processes\":1,\"max children reached\":0,\"slow requests\":0, \"processes\":{\"pid\":183251,\"state\":\"Running\",\"start time\":1667299161,\"start since\":515234,\"requests\":8324,\"request duration\":208,\"request method\":\"GET\",\"request uri\":\"/status?json\u0026full\",\"content length\":0,\"user\":\"-\",\"script\":\"-\",\"last request cpu\":0.00,\"last request memory\":0}}",
16+
"type": [
17+
"info"
18+
]
19+
},
20+
"http": {
21+
"request": {
22+
"body": {
23+
"bytes": 0
24+
},
25+
"method": "GET"
26+
}
27+
},
28+
"php_fpm": {
29+
"process": {
30+
"pool": {
31+
"name": "www"
32+
},
33+
"request": {
34+
"count": 8324,
35+
"duration": 208,
36+
"last": {
37+
"cpu": {
38+
"pct": 0.0
39+
},
40+
"memory": 0
41+
}
42+
},
43+
"script": "-",
44+
"start_since": 515234,
45+
"start_time": 1667299161,
46+
"state": "Running"
47+
}
48+
},
49+
"process": {
50+
"pid": 183251
51+
},
52+
"tags": [
53+
"preserve_original_event"
54+
],
55+
"url": {
56+
"original": "/status?json\u0026full"
57+
},
58+
"user": {
59+
"name": "-"
60+
}
61+
}
62+
]
63+
}

‎packages/php_fpm/data_stream/process/_dev/test/system/test-default-config.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
vars:
2+
hostname:
3+
- http://{{Hostname}}
4+
status_path:
5+
- /status

‎packages/php_fpm/data_stream/process/agent/stream/httpjson.yml.hbs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
config_version: 2
2+
interval: {{period}}
3+
{{#if ssl}}
4+
request.ssl: {{ssl}}
5+
{{/if}}
6+
request.method: GET
7+
request.url: {{hostname}}{{status_path}}?json&full
8+
response.split:
9+
keep_parent: true
10+
target: body.processes
11+
tags:
12+
{{#if preserve_original_event}}
13+
- preserve_original_event
14+
{{/if}}
15+
{{#each tags as |tag|}}
16+
- {{tag}}
17+
{{/each}}
18+
{{#contains "forwarded" tags}}
19+
publisher_pipeline.disable_host: true
20+
{{/contains}}
21+
{{#if processors}}
22+
processors:
23+
{{processors}}
24+
{{/if}}

‎packages/php_fpm/data_stream/process/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
---
2+
description: Pipeline for parsing PHP-FPM Process metrics.
3+
processors:
4+
- set:
5+
field: ecs.version
6+
value: 8.4.0
7+
ignore_failure: true
8+
ignore_empty_value: true
9+
- set:
10+
field: event.type
11+
value: ["info"]
12+
ignore_failure: true
13+
ignore_empty_value: true
14+
- set:
15+
field: event.kind
16+
value: event
17+
ignore_failure: true
18+
ignore_empty_value: true
19+
- set:
20+
field: event.category
21+
value: ["web","configuration","process"]
22+
ignore_failure: true
23+
ignore_empty_value: true
24+
- set:
25+
field: event.module
26+
value: php_fpm
27+
ignore_failure: true
28+
ignore_empty_value: true
29+
- rename:
30+
field: message
31+
target_field: event.original
32+
ignore_missing: true
33+
ignore_failure: true
34+
- json:
35+
field: event.original
36+
target_field: json
37+
ignore_failure: true
38+
- rename:
39+
field: json.processes.state
40+
target_field: php_fpm.process.state
41+
ignore_missing: true
42+
ignore_failure: true
43+
- rename:
44+
field: json.processes.requests
45+
target_field: php_fpm.process.request.count
46+
ignore_missing: true
47+
ignore_failure: true
48+
- rename:
49+
field: json.processes.request duration
50+
target_field: php_fpm.process.request.duration
51+
ignore_missing: true
52+
ignore_failure: true
53+
- rename:
54+
field: json.processes.script
55+
target_field: php_fpm.process.script
56+
ignore_missing: true
57+
ignore_failure: true
58+
- rename:
59+
field: json.pool
60+
target_field: php_fpm.process.pool.name
61+
ignore_missing: true
62+
ignore_failure: true
63+
- rename:
64+
field: json.processes.last request cpu
65+
target_field: php_fpm.process.request.last.cpu.pct
66+
ignore_missing: true
67+
ignore_failure: true
68+
- rename:
69+
field: json.processes.last request memory
70+
target_field: php_fpm.process.request.last.memory
71+
ignore_missing: true
72+
ignore_failure: true
73+
- rename:
74+
field: json.processes.pid
75+
target_field: process.pid
76+
ignore_missing: true
77+
ignore_failure: true
78+
- rename:
79+
field: json.processes.request uri
80+
target_field: url.original
81+
ignore_missing: true
82+
ignore_failure: true
83+
- rename:
84+
field: json.processes.user
85+
target_field: user.name
86+
ignore_missing: true
87+
ignore_failure: true
88+
- rename:
89+
field: json.processes.request method
90+
target_field: http.request.method
91+
ignore_missing: true
92+
ignore_failure: true
93+
- rename:
94+
field: json.processes.content length
95+
target_field: http.request.body.bytes
96+
ignore_missing: true
97+
ignore_failure: true
98+
- rename:
99+
field: json.processes.start since
100+
target_field: php_fpm.process.start_since
101+
ignore_missing: true
102+
ignore_failure: true
103+
- rename:
104+
field: json.processes.start time
105+
target_field: php_fpm.process.start_time
106+
ignore_missing: true
107+
ignore_failure: true
108+
- remove:
109+
field: event.original
110+
if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
111+
ignore_missing: true
112+
ignore_failure: true
113+
- script:
114+
description: Drops null/empty values recursively.
115+
lang: painless
116+
source: |
117+
boolean drop(Object o) {
118+
if (o == null || o == "" || o == "(null)") {
119+
return true;
120+
} else if (o instanceof Map) {
121+
((Map) o).values().removeIf(v -> drop(v));
122+
return (((Map) o).size() == 0);
123+
} else if (o instanceof List) {
124+
((List) o).removeIf(v -> drop(v));
125+
return (((List) o).length == 0);
126+
}
127+
return false;
128+
}
129+
drop(ctx);
130+
- remove:
131+
field:
132+
- json
133+
ignore_missing: true
134+
ignore_failure: true
135+
on_failure:
136+
- set:
137+
field: error.message
138+
value: "{{{_ingest.on_failure_message}}}"

‎packages/php_fpm/data_stream/process/fields/base-fields.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
- name: data_stream.type
2+
type: constant_keyword
3+
description: Data stream type.
4+
- name: data_stream.dataset
5+
type: constant_keyword
6+
description: Data stream dataset.
7+
- name: data_stream.namespace
8+
type: constant_keyword
9+
description: Data stream namespace.
10+
- name: input.type
11+
type: keyword
12+
description: Type of Filebeat input.
13+
- name: '@timestamp'
14+
type: date
15+
description: Event timestamp.

‎packages/php_fpm/data_stream/process/fields/ecs.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
- external: ecs
2+
name: ecs.version
3+
- external: ecs
4+
name: error.message
5+
- external: ecs
6+
name: event.category
7+
- external: ecs
8+
name: event.dataset
9+
- external: ecs
10+
name: event.ingested
11+
- external: ecs
12+
name: event.kind
13+
- external: ecs
14+
name: event.module
15+
- external: ecs
16+
name: event.type
17+
- external: ecs
18+
name: http.request.body.bytes
19+
- external: ecs
20+
name: http.request.method
21+
- external: ecs
22+
name: process.pid
23+
- external: ecs
24+
name: url.original
25+
- external: ecs
26+
name: user.name
27+
- external: ecs
28+
name: tags

0 commit comments

Comments
 (0)