Skip to content

Commit d150dd6

Browse files
siegfriedwebersiegfriedweber
committed
test: Fix metrics in smoke test
1 parent 021d14a commit d150dd6

File tree

1 file changed

+312
-0
lines changed

1 file changed

+312
-0
lines changed

‎tests/templates/kuttl/smoke/30-install-hdfs.yaml.j2‎

Lines changed: 312 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,26 @@ spec:
2828
envOverrides:
2929
COMMON_VAR: role-value # overridden by role group below
3030
ROLE_VAR: role-value # only defined here at role level
31+
HDFS_NAMENODE_OPTS: >
32+
-Djava.security.properties=/stackable/config/namenode/security.properties
33+
-javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar=8183:/stackable/jmx-exporter-config/namenode.yaml
34+
-Xmx838860k
35+
podOverrides:
36+
spec:
37+
initContainers:
38+
- name: format-namenodes
39+
volumeMounts:
40+
- name: jmx-exporter-config
41+
mountPath: /stackable/jmx-exporter-config
42+
containers:
43+
- name: namenode
44+
volumeMounts:
45+
- name: jmx-exporter-config
46+
mountPath: /stackable/jmx-exporter-config
47+
volumes:
48+
- name: jmx-exporter-config
49+
configMap:
50+
name: jmx-exporter-config
3151
config:
3252
listenerClass: {{ test_scenario['values']['listener-class'] }}
3353
logging:
@@ -42,6 +62,21 @@ spec:
4262
envOverrides:
4363
COMMON_VAR: role-value # overridden by role group below
4464
ROLE_VAR: role-value # only defined here at role level
65+
HDFS_DATANODE_OPTS: >
66+
-Djava.security.properties=/stackable/config/datanode/security.properties
67+
-javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar=8082:/stackable/jmx-exporter-config/datanode.yaml
68+
-Xmx419430k
69+
podOverrides:
70+
spec:
71+
containers:
72+
- name: datanode
73+
volumeMounts:
74+
- name: jmx-exporter-config
75+
mountPath: /stackable/jmx-exporter-config
76+
volumes:
77+
- name: jmx-exporter-config
78+
configMap:
79+
name: jmx-exporter-config
4580
config:
4681
listenerClass: {{ test_scenario['values']['listener-class'] }}
4782
logging:
@@ -71,6 +106,21 @@ spec:
71106
envOverrides:
72107
COMMON_VAR: role-value # overridden by role group below
73108
ROLE_VAR: role-value # only defined here at role level
109+
HDFS_JOURNALNODE_OPTS: >
110+
-Djava.security.properties=/stackable/config/journalnode/security.properties
111+
-javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar=8081:/stackable/jmx-exporter-config/journalnode.yaml
112+
-Xmx419430k
113+
podOverrides:
114+
spec:
115+
containers:
116+
- name: journalnode
117+
volumeMounts:
118+
- name: jmx-exporter-config
119+
mountPath: /stackable/jmx-exporter-config
120+
volumes:
121+
- name: jmx-exporter-config
122+
configMap:
123+
name: jmx-exporter-config
74124
config:
75125
logging:
76126
enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
@@ -93,3 +143,265 @@ spec:
93143
# https://github.com/stackabletech/hdfs-operator/issues/514
94144
- name: dashed-port
95145
containerPort: 1234
146+
---
147+
apiVersion: v1
148+
kind: ConfigMap
149+
metadata:
150+
name: jmx-exporter-config
151+
data:
152+
namenode.yaml: |-
153+
---
154+
startDelaySeconds: 10
155+
ssl: false
156+
lowercaseOutputName: true
157+
lowercaseOutputLabelNames: true
158+
whitelistObjectNames:
159+
- 'Hadoop:service=NameNode,name=*'
160+
- 'Hadoop:service=NameNode,name=MetricsSystem,sub=*'
161+
blacklistObjectNames:
162+
- 'Hadoop:service=NameNode,name=RetryCache.NameNodeRetryCache'
163+
- 'Hadoop:service=NameNode,name=RpcActivity*'
164+
- 'Hadoop:service=NameNode,name=RpcDetailedActivity*'
165+
- 'Hadoop:service=NameNode,name=UgiMetrics'
166+
rules:
167+
# MetricsSystem
168+
- pattern: 'Hadoop<service=(.*), name=MetricsSystem, sub=(.*)><>(.*): (\d+)'
169+
attrNameSnakeCase: true
170+
name: hadoop_$1_$3
171+
value: $4
172+
labels:
173+
service: HDFS
174+
role: $1
175+
kind: 'MetricsSystem'
176+
sub: $2
177+
type: GAUGE
178+
# Total raw capacity in bytes, e.g. Hadoop:name=NameNodeInfo,attribute=Total
179+
- pattern: 'Hadoop<service=(.*), name=(.*)><>(total): (\d+)'
180+
attrNameSnakeCase: true
181+
name: hadoop_$1_$3
182+
value: $4
183+
labels:
184+
service: HDFS
185+
role: $1
186+
kind: $2
187+
type: COUNTER
188+
# Generic counter, e.g. Hadoop:name=FSNamesystem,attribute=FilesTotal
189+
- pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_total): (\d+)'
190+
attrNameSnakeCase: true
191+
name: hadoop_$1_$3
192+
value: $4
193+
labels:
194+
service: HDFS
195+
role: $1
196+
kind: $2
197+
type: COUNTER
198+
# Metrics suffixed with _created, e.g. Hadoop:name=NameNodeActivity,attribute=FilesCreated
199+
# The suffix _created is reserved for timestamps, therefore an underscore is appended.
200+
- pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_created): (.*)'
201+
attrNameSnakeCase: true
202+
name: hadoop_$1_$3_
203+
value: $4
204+
labels:
205+
service: HDFS
206+
role: $1
207+
kind: $2
208+
type: GAUGE
209+
# Metrics suffixed with _info, e.g. Hadoop:name=JvmMetrics,attribute=LogInfo
210+
# The suffix _info is reserved for static information, therefore an underscore is appended.
211+
- pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_info): (.*)'
212+
attrNameSnakeCase: true
213+
name: hadoop_$1_$3_
214+
value: $4
215+
labels:
216+
service: HDFS
217+
role: $1
218+
kind: $2
219+
type: GAUGE
220+
# All other Hadoop metrics
221+
- pattern: 'Hadoop<service=(.*), name=(.*)><>(.*): (.*)'
222+
attrNameSnakeCase: true
223+
name: hadoop_$1_$3
224+
value: $4
225+
labels:
226+
service: HDFS
227+
role: $1
228+
kind: $2
229+
type: GAUGE
230+
datanode.yaml: |-
231+
---
232+
startDelaySeconds: 10
233+
ssl: false
234+
lowercaseOutputName: true
235+
lowercaseOutputLabelNames: true
236+
whitelistObjectNames:
237+
- 'Hadoop:service=DataNode,name=*'
238+
- 'Hadoop:service=DataNode,name=MetricsSystem,sub=*'
239+
blacklistObjectNames:
240+
- 'Hadoop:service=DataNode,name=RpcActivity*'
241+
- 'Hadoop:service=DataNode,name=RpcDetailedActivity*'
242+
- 'Hadoop:service=DataNode,name=UgiMetrics'
243+
rules:
244+
# MetricsSystem
245+
- pattern: 'Hadoop<service=(.*), name=MetricsSystem, sub=(.*)><>(.*): (\d+)'
246+
attrNameSnakeCase: true
247+
name: hadoop_$1_$3
248+
value: $4
249+
labels:
250+
service: HDFS
251+
role: $1
252+
kind: 'MetricsSystem'
253+
sub: $2
254+
type: GAUGE
255+
# FSDatasetState (also extracts the FSDataset ID)
256+
- pattern: 'Hadoop<service=(.*), name=FSDatasetState-(.*)><>(.*): (\d+)'
257+
attrNameSnakeCase: true
258+
name: hadoop_$1_$3
259+
value: $4
260+
labels:
261+
service: HDFS
262+
role: $1
263+
fsdatasetid: $2
264+
kind: 'FSDatasetState'
265+
type: GAUGE
266+
# DataNodeActivity (also extracts hostname and port)
267+
- pattern: 'Hadoop<service=(.*), name=DataNodeActivity-(.*)-(\d+)><>(.*): (\d+)'
268+
attrNameSnakeCase: true
269+
name: hadoop_$1_$4
270+
value: $5
271+
labels:
272+
service: HDFS
273+
role: $1
274+
host: $2
275+
port: $3
276+
kind: 'DataNodeActivity'
277+
type: GAUGE
278+
# Total raw capacity in bytes, e.g. Hadoop:name=NameNodeInfo,attribute=Total
279+
- pattern: 'Hadoop<service=(.*), name=(.*)><>(total): (\d+)'
280+
attrNameSnakeCase: true
281+
name: hadoop_$1_$3
282+
value: $4
283+
labels:
284+
service: HDFS
285+
role: $1
286+
kind: $2
287+
type: COUNTER
288+
# Generic counter, e.g. Hadoop:name=FSNamesystem,attribute=FilesTotal
289+
- pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_total): (\d+)'
290+
attrNameSnakeCase: true
291+
name: hadoop_$1_$3
292+
value: $4
293+
labels:
294+
service: HDFS
295+
role: $1
296+
kind: $2
297+
type: COUNTER
298+
# Metrics suffixed with _created, e.g. Hadoop:name=NameNodeActivity,attribute=FilesCreated
299+
# The suffix _created is reserved for timestamps, therefore an underscore is appended.
300+
- pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_created): (.*)'
301+
attrNameSnakeCase: true
302+
name: hadoop_$1_$3_
303+
value: $4
304+
labels:
305+
service: HDFS
306+
role: $1
307+
kind: $2
308+
type: GAUGE
309+
# Metrics suffixed with _info, e.g. Hadoop:name=JvmMetrics,attribute=LogInfo
310+
# The suffix _info is reserved for static information, therefore an underscore is appended.
311+
- pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_info): (.*)'
312+
attrNameSnakeCase: true
313+
name: hadoop_$1_$3_
314+
value: $4
315+
labels:
316+
service: HDFS
317+
role: $1
318+
kind: $2
319+
type: GAUGE
320+
# All other Hadoop metrics
321+
- pattern: 'Hadoop<service=(.*), name=(.*)><>(.*): (.*)'
322+
attrNameSnakeCase: true
323+
name: hadoop_$1_$3
324+
value: $4
325+
labels:
326+
service: HDFS
327+
role: $1
328+
kind: $2
329+
type: GAUGE
330+
journalnode.yaml: |-
331+
---
332+
startDelaySeconds: 10
333+
ssl: false
334+
lowercaseOutputName: true
335+
lowercaseOutputLabelNames: true
336+
whitelistObjectNames:
337+
- 'Hadoop:service=JournalNode,name=*'
338+
- 'Hadoop:service=JournalNode,name=MetricsSystem,sub=*'
339+
blacklistObjectNames:
340+
- 'Hadoop:service=JournalNode,name=RetryCache.JournalNodeRetryCache'
341+
- 'Hadoop:service=JournalNode,name=RpcActivity*'
342+
- 'Hadoop:service=JournalNode,name=RpcDetailedActivity*'
343+
- 'Hadoop:service=JournalNode,name=UgiMetrics'
344+
rules:
345+
# MetricsSystem
346+
- pattern: 'Hadoop<service=(.*), name=MetricsSystem, sub=(.*)><>(.*): (\d+)'
347+
attrNameSnakeCase: true
348+
name: hadoop_$1_$3
349+
value: $4
350+
labels:
351+
service: HDFS
352+
role: $1
353+
kind: 'MetricsSystem'
354+
sub: $2
355+
type: GAUGE
356+
# Total raw capacity in bytes, e.g. Hadoop:name=NameNodeInfo,attribute=Total
357+
- pattern: 'Hadoop<service=(.*), name=(.*)><>(total): (\d+)'
358+
attrNameSnakeCase: true
359+
name: hadoop_$1_$3
360+
value: $4
361+
labels:
362+
service: HDFS
363+
role: $1
364+
kind: $2
365+
type: COUNTER
366+
# Generic counter, e.g. Hadoop:name=FSNamesystem,attribute=FilesTotal
367+
- pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_total): (\d+)'
368+
attrNameSnakeCase: true
369+
name: hadoop_$1_$3
370+
value: $4
371+
labels:
372+
service: HDFS
373+
role: $1
374+
kind: $2
375+
type: COUNTER
376+
# Metrics suffixed with _created, e.g. Hadoop:name=NameNodeActivity,attribute=FilesCreated
377+
# The suffix _created is reserved for timestamps, therefore an underscore is appended.
378+
- pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_created): (.*)'
379+
attrNameSnakeCase: true
380+
name: hadoop_$1_$3_
381+
value: $4
382+
labels:
383+
service: HDFS
384+
role: $1
385+
kind: $2
386+
type: GAUGE
387+
# Metrics suffixed with _info, e.g. Hadoop:name=JvmMetrics,attribute=LogInfo
388+
# The suffix _info is reserved for static information, therefore an underscore is appended.
389+
- pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_info): (.*)'
390+
attrNameSnakeCase: true
391+
name: hadoop_$1_$3_
392+
value: $4
393+
labels:
394+
service: HDFS
395+
role: $1
396+
kind: $2
397+
type: GAUGE
398+
# All other Hadoop metrics
399+
- pattern: 'Hadoop<service=(.*), name=(.*)><>(.*): (.*)'
400+
attrNameSnakeCase: true
401+
name: hadoop_$1_$3
402+
value: $4
403+
labels:
404+
service: HDFS
405+
role: $1
406+
kind: $2
407+
type: GAUGE

0 commit comments

Comments
 (0)