Skip to content

Commit d491715

Browse files
arnaudfroidmontarnaudfroidmont
authored
Merge pull request #211 from oci-hpc/2.11.0_sam_17sep
Healthchecks and dashboard
2 parents 1e55551 + 82bdef8 commit d491715

File tree

3 files changed

+319
-61
lines changed

3 files changed

+319
-61
lines changed

‎playbooks/roles/grafana/files/main.jsonnet‎

Lines changed: 101 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,61 @@ local g = import './g.libsonnet';
22
local variables = import './variables.libsonnet';
33
local row = g.panel.row;
44

5+
local critical_status_ts = [
6+
{ expr: 'count_values("hostname", oca_version) by(version)', legend_format: '{{version}}', title: 'Hosts with OCA version', unit: 'none' },
7+
{ expr: 'count_values("hostname", gpu_count) by (instance_shape)', legend_format: '{{instance_shape}}', title: 'Shapes with matching GPU count', unit: 'none' },
8+
{ expr: 'count_values("hostname", up{instance=~".*9100"})', legend_format: '{{hostname}}', title: 'Hosts up', unit: 'none' },
9+
{ expr: 'check_bus_issue_count{hostname=~"$hostname", oci_name=~"$oci_name"}', legend_format: '{{ hostname }}', title: 'Devices fallen off bus error count', unit: 'none' },
10+
{ expr: 'DCGM_FI_DEV_XID_ERRORS{Hostname=~"$hostname", gpu=~"$gpu", oci_name=~"$oci_name"}', legend_format: '{{ Hostname }}:{{ gpu }}', title: 'Value of the last XID error encountered', unit: 'none' },
11+
];
12+
13+
local critical_status_stl = [
14+
{ expr1: 'rdma_device_status{hostname=~"$hostname", oci_name=~"$oci_name"}==0', expr2: 'rdma_device_status{hostname=~"$hostname", oci_name=~"$oci_name"}==1', legend_format: '{{hostname}}:{{rdma_device}}', title: 'RDMA Device Status', unit: 'none', colors: {'0': { text: 'down', color: 'red' },'1': { text: 'up', color: 'green' },} },
15+
{ expr1: 'gpu_row_remap_error_check{hostname=~"$hostname", oci_name=~"$oci_name"}==0', expr2: 'gpu_row_remap_error_check{hostname=~"$hostname", oci_name=~"$oci_name"}==1', legend_format: '{{hostname}}', title: 'GPU Row Remap Error Check', unit: 'none', colors: {'0': { text: 'passed', color: 'green' },'1': { text: 'failed', color: 'red' },} },
16+
{ expr1: 'gpu_ecc_error_check{hostname=~"$hostname", oci_name=~"$oci_name"}==0', expr2: 'gpu_ecc_error_check{hostname=~"$hostname", oci_name=~"$oci_name"}==1', legend_format: '{{hostname}}', title: 'GPU ECC Error Check', unit: 'none', colors: {'0': { text: 'failed', color: 'red' },'1': { text: 'passed', color: 'green' },} },
17+
{ expr1: 'xid_error_check{hostname=~"$hostname", oci_name=~"$oci_name"}==0', expr2: 'xid_error_check{hostname=~"$hostname", oci_name=~"$oci_name"}==1', legend_format: '{{hostname}}', title: 'Xid Error Check', unit: 'none', colors: {'0': { text: 'passed', color: 'green' },'1': { text: 'failed', color: 'red' },} },
18+
];
19+
20+
local health_status = [
21+
{ expr1: 'ib_link_state{hostname=~"$hostname", oci_name=~"$oci_name"}==1 or vector(0)', expr2: 'rdma_device_status{hostname=~"$hostname", oci_name=~"$oci_name"} > 1', legend_format: '{{hostname}}:{{rdma_device}}', title: 'RDMA Link State (h/w metric)', unit: 'none', colors: {'1': { text: 'down', color: 'red' },} },
22+
{ expr1: 'rdma_link_noflap{hostname=~"$hostname", oci_name=~"$oci_name"}==0', expr2: 'rdma_link_noflap{hostname=~"$hostname", oci_name=~"$oci_name"}==1', legend_format: '{{hostname}}:{{rdma_device}}', title: 'RDMA Link flapping', unit: 'none', colors: {'0': { text: 'down', color: 'red' },'1': { text: 'up', color: 'green' },} },
23+
{ expr1: 'rttcc_status{hostname=~"$hostname", oci_name=~"$oci_name"}==0', expr2: 'rttcc_status{hostname=~"$hostname", oci_name=~"$oci_name"}==1', legend_format: '{{hostname}}:{{rdma_device}}', title: 'RTTCC Status', unit: 'none', colors: {'0': { text: 'disabled', color: 'green' },'1': { text: 'enabled', color: 'red' },} },
24+
{ expr1: 'gpu_count{hostname=~"$hostname", oci_name=~"$oci_name"}==0', expr2: 'gpu_count{hostname=~"$hostname", oci_name=~"$oci_name"}==1', legend_format: '{{hostname}}:{{instance_shape}}', title: 'GPU Count', unit: 'none', colors: {'0': { text: 'down', color: 'red' },'1': { text: 'up', color: 'green' },} },
25+
{ expr1: 'oca_version{hostname=~"$hostname", oci_name=~"$oci_name"}==0', expr2: 'oca_version{hostname=~"$hostname", oci_name=~"$oci_name"}==1', legend_format: '{{hostname}}:{{version}}', title: 'OCA Version', unit: 'none', colors: {'0': { text: 'down', color: 'red' },'1': { text: 'up', color: 'green' },} },
26+
];
27+
28+
local cluster_metrics = [
29+
{ expr: 'avg by (cluster_name) (node_load1)', legend_format: '1m load average {{cluster_name}}', title: 'Cluster 1m load average', unit: 'percent' },
30+
{ expr: 'avg by (cluster_name) (node_load5)', legend_format: '5m load average {{cluster_name}}', title: 'Cluster 5m load average', unit: 'percent' },
31+
{ expr: 'avg by (cluster_name) (node_load15)', legend_format: '15m load average {{cluster_name}}', title: 'Cluster 15m load average', unit: 'percent' },
32+
];
33+
34+
local node_metrics = [
35+
{ expr: '(node_load1{hostname=~"$hostname",oci_name=~"$oci_name"})', legend_format: '{{oci_name}} {{hostname}}', title: 'Instance 1m load average', unit: 'percent' },
36+
{ expr: '(node_load5{hostname=~"$hostname",oci_name=~"$oci_name"})', legend_format: '{{oci_name}} {{hostname}}', title: 'Instance 5m load average', unit: 'percent' },
37+
{ expr: '(node_load15{hostname=~"$hostname",oci_name=~"$oci_name"})', legend_format: '{{oci_name}} {{hostname}}', title: 'Instance 15m load average', unit: 'percent' },
38+
{ expr: 'ceil((1 - (node_memory_MemAvailable_bytes{hostname=~"$hostname",oci_name=~"$oci_name"}/node_memory_MemTotal_bytes{hostname=~"$hostname",oci_name=~"$oci_name"}))*100)', legend_format: '{{oci_name}} {{hostname}}', title: 'Memory utilization', unit: 'percent' },
39+
{ expr: 'ceil((1 - (node_filesystem_avail_bytes{hostname=~"$hostname",oci_name=~"$oci_name",mountpoint=~"$mountpoint",device!~"rootfs"} / node_filesystem_size_bytes{hostname=~"$hostname",oci_name=~"$oci_name",mountpoint=~"$mountpoint",device!~"rootfs"}))*100)', legend_format: '{{oci_name}} {{hostname}} {{mountpoint}}', title: 'Storage utilization', unit: 'percent'},
40+
{ expr: 'irate(node_disk_reads_completed_total{hostname=~"$hostname",oci_name=~"$oci_name"}[5m])', legend_format: '{{oci_name}} {{hostname}} {{device}}', title: 'Disk reads completed iops', unit: 'iops'},
41+
{ expr: 'irate(node_disk_writes_completed_total{hostname=~"$hostname",oci_name=~"$oci_name"}[5m])', legend_format: '{{oci_name}} {{hostname}} {{device}}', title: 'Disk writes completed iops', unit: 'iops'},
42+
{ expr: 'irate(node_disk_read_bytes_total{hostname=~"$hostname",oci_name=~"$oci_name"}[5m])', legend_format: '{{oci_name}} {{hostname}} {{device}}', title: 'Disk read bytes', unit: 'Bps'},
43+
{ expr: 'irate(node_disk_written_bytes_total{hostname=~"$hostname",oci_name=~"$oci_name"}[5m])', legend_format: '{{oci_name}} {{hostname}} {{device}}', title: 'Disk write bytes', unit: 'Bps'},
44+
{ expr: 'irate(node_disk_io_time_seconds_total{hostname=~"$hostname",oci_name=~"$oci_name"}[5m])', legend_format: '{{oci_name}} {{hostname}} {{device}}', title: 'Time spent doing I/Os', unit: 'percentunit'},
45+
{ expr: 'rate(node_network_receive_bytes_total{hostname=~"$hostname",oci_name=~"$oci_name",device=~"$device"}[5m])', legend_format: "{{oci_name}} {{hostname}} {{device}}", title: 'Network Traffic Received', unit: 'Bps'},
46+
{ expr: 'rate(node_network_transmit_bytes_total{hostname=~"$hostname",oci_name=~"$oci_name",device=~"$device"}[5m])', legend_format: "{{oci_name}} {{hostname}} {{device}}", title: 'Network Traffic Sent', unit: 'Bps'}
47+
];
48+
49+
local nfs_metrics = [
50+
{ expr: 'rate(node_mountstats_nfs_total_read_bytes_total{hostname=~"$hostname", oci_name=~"$oci_name"}[$__range])', legend_format: '{{oci_name}} {{hostname}} {{export}}', title: 'Read Throughput', unit: 'Bps' },
51+
{ expr: 'rate(node_mountstats_nfs_total_write_bytes_total{hostname=~"$hostname", oci_name=~"$oci_name"}[$__range])', legend_format: '{{oci_name}} {{hostname}} {{export}}', title: 'Write Throughput', unit: 'Bps' },
52+
{ expr: 'sum by(oci_name, hostname) (rate(node_mountstats_nfs_operations_requests_total{hostname=~"$hostname", oci_name=~"$oci_name", operation!~"READ|WRITE"}[$__range]))', legend_format: '{{oci_name}} {{hostname}}', title: 'Metadata IOPS', unit: 'iops' },
53+
{ expr: 'sum by(oci_name, hostname) (rate(node_mountstats_nfs_operations_requests_total{hostname=~"$hostname", oci_name=~"$oci_name", operation=~"READ|WRITE"}[$__range]))', legend_format: '{{oci_name}} {{hostname}}', title: 'Read/Write IOPS', unit: 'iops' },
54+
{ expr: 'sum by(oci_name, hostname, export) (node_nfs_rpc_retransmissions_total{hostname=~"$hostname", oci_name=~"$oci_name"})', legend_format: '{{oci_name}} {{hostname}} {{export}}', title: 'NFS Retransmissions', unit: 'cps' },
55+
{ expr: 'avg by(oci_name, hostname, export) (rate(node_mountstats_nfs_operations_request_time_seconds_total{hostname=~"$hostname", oci_name=~"$oci_name"}[$__range]))', legend_format: '{{oci_name}} {{hostname}} {{export}}', title: 'NFS Request Time', unit: 's' },
56+
{ expr: 'avg by(oci_name, hostname, export) (rate(node_mountstats_nfs_operations_response_time_seconds_total{hostname=~"$hostname", oci_name=~"$oci_name"}[$__range]))', legend_format: '{{oci_name}} {{hostname}} {{export}}', title: 'NFS Response Time', unit: 's' },
57+
{ expr: 'avg by(oci_name, hostname, export) (rate(node_mountstats_nfs_operations_queue_time_seconds_total{hostname=~"$hostname", oci_name=~"$oci_name"}[$__range]))', legend_format: '{{oci_name}} {{hostname}} {{export}}', title: 'NFS Queue Time', unit: 's' },
58+
];
59+
560
local dcgm_metrics = [
661
{ name: 'DCGM_FI_DEV_SM_CLOCK', title: 'SM Clock', unit: 'hertz' },
762
{ name: 'DCGM_FI_DEV_MEM_CLOCK', title: 'Memory Clock', unit: 'hertz' },
@@ -30,6 +85,13 @@ local dcgm_errors = [
3085
{ name: 'DCGM_FI_DEV_ROW_REMAP_FAILURE', title: 'Whether remapping of rows has failed', unit: 'cps' },
3186
];
3287

88+
local nvlink_metrics = [
89+
{ name: 'nvlink_data_tx_kib', title: 'Total data transmitted', unit: 'KBs' },
90+
{ name: 'nvlink_data_rx_kib', title: 'Total data received', unit: 'KBs' },
91+
{ name: 'nvlink_raw_tx_kib', title: 'Total raw bytes transmitted', unit: 'KBs' },
92+
{ name: 'nvlink_raw_rx_kib', title: 'Total raw bytes received', unit: 'KBs' },
93+
];
94+
3395
local ib_port_metrics = [
3496
{ name: 'ib_port_xmit_data', title: 'Total number of data octets transmitted', unit: 'MiBs' },
3597
{ name: 'ib_port_rcv_data', title: 'Total number of data octets received', unit: 'MiBs' },
@@ -54,53 +116,6 @@ local roce2_errors = [
54116
{ name: 'rdma_roce_slow_restart', title: 'Number of times RoCE slow restart was used', unit: 'none' },
55117
];
56118

57-
local nvlink_metrics = [
58-
{ name: 'nvlink_data_tx_kib', title: 'Total data transmitted', unit: 'KBs' },
59-
{ name: 'nvlink_data_rx_kib', title: 'Total data received', unit: 'KBs' },
60-
{ name: 'nvlink_raw_tx_kib', title: 'Total raw bytes transmitted', unit: 'KBs' },
61-
{ name: 'nvlink_raw_rx_kib', title: 'Total raw bytes received', unit: 'KBs' },
62-
];
63-
64-
local cluster_metrics = [
65-
{ expr: 'avg by (cluster_name) (node_load1)', legend_format: '1m load average {{cluster_name}}', title: 'Cluster 1m load average', unit: 'percent' },
66-
{ expr: 'avg by (cluster_name) (node_load5)', legend_format: '5m load average {{cluster_name}}', title: 'Cluster 5m load average', unit: 'percent' },
67-
{ expr: 'avg by (cluster_name) (node_load15)', legend_format: '15m load average {{cluster_name}}', title: 'Cluster 15m load average', unit: 'percent' },
68-
];
69-
70-
local node_metrics = [
71-
{ expr: '(node_load1{hostname=~"$hostname",oci_name=~"$oci_name"})', legend_format: '{{oci_name}} {{hostname}}', title: 'Instance 1m load average', unit: 'percent' },
72-
{ expr: '(node_load5{hostname=~"$hostname",oci_name=~"$oci_name"})', legend_format: '{{oci_name}} {{hostname}}', title: 'Instance 5m load average', unit: 'percent' },
73-
{ expr: '(node_load15{hostname=~"$hostname",oci_name=~"$oci_name"})', legend_format: '{{oci_name}} {{hostname}}', title: 'Instance 15m load average', unit: 'percent' },
74-
{ expr: 'ceil((1 - (node_memory_MemAvailable_bytes{hostname=~"$hostname",oci_name=~"$oci_name"}/node_memory_MemTotal_bytes{hostname=~"$hostname",oci_name=~"$oci_name"}))*100)', legend_format: '{{oci_name}} {{hostname}}', title: 'Memory utilization', unit: 'percent' },
75-
{ expr: 'ceil((1 - (node_filesystem_avail_bytes{hostname=~"$hostname",oci_name=~"$oci_name",mountpoint=~"$mountpoint",device!~"rootfs"} / node_filesystem_size_bytes{hostname=~"$hostname",oci_name=~"$oci_name",mountpoint=~"$mountpoint",device!~"rootfs"}))*100)', legend_format: '{{oci_name}} {{hostname}} {{mountpoint}}', title: 'Storage utilization', unit: 'percent'},
76-
{ expr: 'irate(node_disk_reads_completed_total{hostname=~"$hostname",oci_name=~"$oci_name"}[5m])', legend_format: '{{oci_name}} {{hostname}} {{device}}', title: 'Disk reads completed iops', unit: 'iops'},
77-
{ expr: 'irate(node_disk_writes_completed_total{hostname=~"$hostname",oci_name=~"$oci_name"}[5m])', legend_format: '{{oci_name}} {{hostname}} {{device}}', title: 'Disk writes completed iops', unit: 'iops'},
78-
{ expr: 'irate(node_disk_read_bytes_total{hostname=~"$hostname",oci_name=~"$oci_name"}[5m])', legend_format: '{{oci_name}} {{hostname}} {{device}}', title: 'Disk read bytes', unit: 'Bps'},
79-
{ expr: 'irate(node_disk_written_bytes_total{hostname=~"$hostname",oci_name=~"$oci_name"}[5m])', legend_format: '{{oci_name}} {{hostname}} {{device}}', title: 'Disk write bytes', unit: 'Bps'},
80-
{ expr: 'irate(node_disk_io_time_seconds_total{hostname=~"$hostname",oci_name=~"$oci_name"}[5m])', legend_format: '{{oci_name}} {{hostname}} {{device}}', title: 'Time spent doing I/Os', unit: 'percentunit'},
81-
{ expr: 'rate(node_network_receive_bytes_total{hostname=~"$hostname",oci_name=~"$oci_name",device=~"$device"}[5m])', legend_format: "{{oci_name}} {{hostname}} {{device}}", title: 'Network Traffic Received', unit: 'Bps'},
82-
{ expr: 'rate(node_network_transmit_bytes_total{hostname=~"$hostname",oci_name=~"$oci_name",device=~"$device"}[5m])', legend_format: "{{oci_name}} {{hostname}} {{device}}", title: 'Network Traffic Sent', unit: 'Bps'}
83-
];
84-
85-
local health_status = [
86-
{ expr1: 'rdma_device_status{hostname=~"$hostname", oci_name=~"$oci_name"}==0', expr2: 'rdma_device_status{hostname=~"$hostname", oci_name=~"$oci_name"}==1', legend_format: '{{hostname}}:{{rdma_device}}', title: 'RDMA Device Status', unit: 'none' },
87-
{ expr1: 'rdma_link_noflap{hostname=~"$hostname", oci_name=~"$oci_name"}==0', expr2: 'rdma_link_noflap{hostname=~"$hostname", oci_name=~"$oci_name"}==1', legend_format: '{{hostname}}:{{rdma_device}}', title: 'RDMA Link flapping', unit: 'none' },
88-
{ expr1: 'rttcc_status{hostname=~"$hostname", oci_name=~"$oci_name"}==0', expr2: 'rttcc_status{hostname=~"$hostname", oci_name=~"$oci_name"}==1', legend_format: '{{hostname}}:{{rdma_device}}', title: 'RTTCC Status', unit: 'none' },
89-
{ expr1: 'gpu_count{hostname=~"$hostname", oci_name=~"$oci_name"}==0', expr2: 'gpu_count{hostname=~"$hostname", oci_name=~"$oci_name"}==1', legend_format: '{{hostname}}:{{instance_shape}}', title: 'GPU Status', unit: 'none' },
90-
{ expr1: 'oca_version{hostname=~"$hostname", oci_name=~"$oci_name"}==0', expr2: 'oca_version{hostname=~"$hostname", oci_name=~"$oci_name"}==1', legend_format: '{{hostname}}:{{version}}', title: 'OCA Version', unit: 'none' },
91-
];
92-
93-
local nfs_metrics = [
94-
{ expr: 'rate(node_mountstats_nfs_total_read_bytes_total{hostname=~"$hostname", oci_name=~"$oci_name"}[$__range])', legend_format: '{{oci_name}} {{hostname}} {{export}}', title: 'Read Throughput', unit: 'Bps' },
95-
{ expr: 'rate(node_mountstats_nfs_total_write_bytes_total{hostname=~"$hostname", oci_name=~"$oci_name"}[$__range])', legend_format: '{{oci_name}} {{hostname}} {{export}}', title: 'Write Throughput', unit: 'Bps' },
96-
{ expr: 'sum by(oci_name, hostname) (rate(node_mountstats_nfs_operations_requests_total{hostname=~"$hostname", oci_name=~"$oci_name", operation!~"READ|WRITE"}[$__range]))', legend_format: '{{oci_name}} {{hostname}}', title: 'Metadata IOPS', unit: 'iops' },
97-
{ expr: 'sum by(oci_name, hostname) (rate(node_mountstats_nfs_operations_requests_total{hostname=~"$hostname", oci_name=~"$oci_name", operation=~"READ|WRITE"}[$__range]))', legend_format: '{{oci_name}} {{hostname}}', title: 'Read/Write IOPS', unit: 'iops' },
98-
{ expr: 'sum by(oci_name, hostname, export) (node_nfs_rpc_retransmissions_total{hostname=~"$hostname", oci_name=~"$oci_name"})', legend_format: '{{oci_name}} {{hostname}} {{export}}', title: 'NFS Retransmissions', unit: 'cps' },
99-
{ expr: 'avg by(oci_name, hostname, export) (rate(node_mountstats_nfs_operations_request_time_seconds_total{hostname=~"$hostname", oci_name=~"$oci_name"}[$__range]))', legend_format: '{{oci_name}} {{hostname}} {{export}}', title: 'NFS Request Time', unit: 's' },
100-
{ expr: 'avg by(oci_name, hostname, export) (rate(node_mountstats_nfs_operations_response_time_seconds_total{hostname=~"$hostname", oci_name=~"$oci_name"}[$__range]))', legend_format: '{{oci_name}} {{hostname}} {{export}}', title: 'NFS Response Time', unit: 's' },
101-
{ expr: 'avg by(oci_name, hostname, export) (rate(node_mountstats_nfs_operations_queue_time_seconds_total{hostname=~"$hostname", oci_name=~"$oci_name"}[$__range]))', legend_format: '{{oci_name}} {{hostname}} {{export}}', title: 'NFS Queue Time', unit: 's' },
102-
];
103-
104119
g.dashboard.new('Cluster Dashboard')
105120
+ g.dashboard.withUid('cluster-dashboard')
106121
+ g.dashboard.withDescription(|||
@@ -127,6 +142,44 @@ g.dashboard.new('Cluster Dashboard')
127142
])
128143
+ g.dashboard.withPanels(
129144
g.util.grid.makeGrid([
145+
row.new('Critical Status')
146+
+ row.withCollapsed(true)
147+
+ row.withPanels([
148+
g.panel.timeSeries.new(metric.title)
149+
+ g.panel.timeSeries.queryOptions.withTargets([
150+
g.query.prometheus.new(
151+
'$PROMETHEUS_DS',
152+
metric.expr,
153+
)
154+
+ g.query.prometheus.withLegendFormat(metric.legend_format)
155+
])
156+
+ g.panel.timeSeries.standardOptions.withUnit(metric.unit)
157+
+ g.panel.timeSeries.gridPos.withW(24)
158+
+ g.panel.timeSeries.gridPos.withH(8)
159+
for metric in critical_status_ts] +
160+
[g.panel.stateTimeline.new(metric.title)
161+
+ g.panel.stateTimeline.queryOptions.withTargets([
162+
g.query.prometheus.new(
163+
'$PROMETHEUS_DS',
164+
metric.expr1,
165+
)
166+
+ g.query.prometheus.withLegendFormat(metric.legend_format),
167+
g.query.prometheus.new(
168+
'$PROMETHEUS_DS',
169+
metric.expr2,
170+
)
171+
+ g.query.prometheus.withLegendFormat(metric.legend_format),
172+
])
173+
+ g.panel.stateTimeline.standardOptions.withUnit(metric.unit)
174+
+ g.panel.stateTimeline.options.withShowValue('never')
175+
+ g.panel.stateTimeline.gridPos.withW(24)
176+
+ g.panel.stateTimeline.gridPos.withH(8)
177+
+ g.panel.stateTimeline.standardOptions.withMappings(
178+
g.panel.stateTimeline.standardOptions.mapping.ValueMap.withType() +
179+
g.panel.stateTimeline.standardOptions.mapping.ValueMap.withOptions(metric.colors)
180+
)
181+
for metric in critical_status_stl]
182+
),
130183
row.new('Health Status')
131184
+ row.withCollapsed(true)
132185
+ row.withPanels([
@@ -149,12 +202,7 @@ g.dashboard.new('Cluster Dashboard')
149202
+ g.panel.stateTimeline.gridPos.withH(8)
150203
+ g.panel.stateTimeline.standardOptions.withMappings(
151204
g.panel.stateTimeline.standardOptions.mapping.ValueMap.withType() +
152-
g.panel.stateTimeline.standardOptions.mapping.ValueMap.withOptions(
153-
{
154-
'0': { text: 'down', color: 'red' },
155-
'1': { text: 'up', color: 'green' },
156-
}
157-
)
205+
g.panel.stateTimeline.standardOptions.mapping.ValueMap.withOptions(metric.colors)
158206
)
159207
for metric in health_status
160208
]),
@@ -288,4 +336,3 @@ g.dashboard.new('Cluster Dashboard')
288336
]),
289337
])
290338
)
291-

0 commit comments

Comments
 (0)