Skip to content

Commit 60f67d8

Browse files
roffemurali-reddy
roffe
authored and
murali-reddy
committed
Metrics updates (#292)
* - added protocol & port label to metrics - removed some redundant code * added example dashboard * added dashboard screenshot * updated dashboard json & screenshot * ammend bad dashboard export
1 parent edda2b1 commit 60f67d8

File tree

5 files changed

+980
-41
lines changed

5 files changed

+980
-41
lines changed

‎Documentation/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,4 +269,5 @@ kubectl annotate service my-service "kube-router.io/service.scheduler=dh"
269269
[Configuring BGP Peers](bgp.md)
270270

271271
## Metrics
272+
272273
[Configure metrics gathering](metrics.md)

‎Documentation/metrics.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,4 +63,10 @@ The following metrics is exposed by kube-router prefixed by `kube_router_`
6363
Outgoing bytes per second
6464

6565
To get a grouped list of CPS for each service a Prometheus query could look like this e.g:
66-
`sum(kube_router_service_cps) by (namespace, service_name)`
66+
`sum(kube_router_service_cps) by (namespace, service_name)`
67+
68+
## Grafana Dashboard
69+
70+
This repo contains a example [Grafana dashboard](https://grafana.com/) utilizing all the above exposed metrics from kube-router.
71+
[kube-router.json](https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/dashboard/kube-router.json)
72+
[dashboard.png](https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/dashboard/dashboard.png)

‎app/controllers/network_services_controller.go‎

Lines changed: 52 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -49,53 +49,53 @@ var (
4949
serviceTotalConn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
5050
Namespace: namespace,
5151
Name: "service_total_connections",
52-
Help: "Total conntection to service",
53-
}, []string{"namespace", "service_name", "service_vip"})
52+
Help: "Total incoming conntections made",
53+
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
5454
servicePacketsIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
5555
Namespace: namespace,
5656
Name: "service_packets_in",
5757
Help: "Total incoming packets",
58-
}, []string{"namespace", "service_name", "service_vip"})
58+
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
5959
servicePacketsOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{
6060
Namespace: namespace,
6161
Name: "service_packets_out",
6262
Help: "Total outoging packets",
63-
}, []string{"namespace", "service_name", "service_vip"})
63+
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
6464
serviceBytesIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
6565
Namespace: namespace,
6666
Name: "service_bytes_in",
6767
Help: "Total incoming bytes",
68-
}, []string{"namespace", "service_name", "service_vip"})
68+
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
6969
serviceBytesOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{
7070
Namespace: namespace,
7171
Name: "service_bytes_out",
72-
Help: "Total outoging bytes",
73-
}, []string{"namespace", "service_name", "service_vip"})
72+
Help: "Total outgoing bytes",
73+
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
7474
servicePpsIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
7575
Namespace: namespace,
7676
Name: "service_pps_in",
7777
Help: "Incoming packets per second",
78-
}, []string{"namespace", "service_name", "service_vip"})
78+
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
7979
servicePpsOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{
8080
Namespace: namespace,
8181
Name: "service_pps_out",
8282
Help: "Outoging packets per second",
83-
}, []string{"namespace", "service_name", "service_vip"})
83+
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
8484
serviceCPS = prometheus.NewGaugeVec(prometheus.GaugeOpts{
8585
Namespace: namespace,
8686
Name: "service_cps",
8787
Help: "Service connections per second",
88-
}, []string{"namespace", "service_name", "service_vip"})
88+
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
8989
serviceBpsIn = prometheus.NewGaugeVec(prometheus.GaugeOpts{
9090
Namespace: namespace,
9191
Name: "service_bps_in",
9292
Help: "Incoming bytes per second",
93-
}, []string{"namespace", "service_name", "service_vip"})
93+
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
9494
serviceBpsOut = prometheus.NewGaugeVec(prometheus.GaugeOpts{
9595
Namespace: namespace,
9696
Name: "service_bps_out",
9797
Help: "Outoging bytes per second",
98-
}, []string{"namespace", "service_name", "service_vip"})
98+
}, []string{"namespace", "service_name", "service_vip", "protocol", "port"})
9999
)
100100

101101
// NetworkServicesController enables local node as network service proxy through IPVS/LVS.
@@ -782,40 +782,52 @@ func (nsc *NetworkServicesController) publishMetrics(serviceInfoMap serviceInfoM
782782

783783
for _, svc := range serviceInfoMap {
784784
var protocol uint16
785-
if svc.protocol == "tcp" {
785+
var pushMetric bool
786+
var svcVip string
787+
788+
switch aProtocol := svc.protocol; aProtocol {
789+
case "tcp":
786790
protocol = syscall.IPPROTO_TCP
787-
} else {
791+
case "udp":
788792
protocol = syscall.IPPROTO_UDP
793+
default:
794+
protocol = syscall.IPPROTO_NONE
789795
}
796+
glog.Info("Publishing Prometheus metrics")
790797
for _, ipvsSvc := range ipvsSvcs {
791-
if strings.Compare(svc.clusterIP.String(), ipvsSvc.Address.String()) == 0 &&
792-
protocol == ipvsSvc.Protocol && uint16(svc.port) == ipvsSvc.Port {
793-
glog.Infof("Publishing prometheus metrics " + svc.clusterIP.String() + ":" + strconv.Itoa(svc.port))
794-
serviceBpsIn.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.BPSIn))
795-
serviceBpsOut.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.BPSOut))
796-
serviceBytesIn.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.BytesIn))
797-
serviceBytesOut.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.BytesOut))
798-
serviceCPS.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.CPS))
799-
servicePacketsIn.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.PacketsIn))
800-
servicePacketsOut.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.PacketsOut))
801-
servicePpsIn.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.PPSIn))
802-
servicePpsOut.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.PPSOut))
803-
serviceTotalConn.WithLabelValues(svc.namespace, svc.name, svc.clusterIP.String()).Set(float64(ipvsSvc.Stats.Connections))
804798

799+
switch svcAddress := ipvsSvc.Address.String(); svcAddress {
800+
case svc.clusterIP.String():
801+
if protocol == ipvsSvc.Protocol && uint16(svc.port) == ipvsSvc.Port {
802+
pushMetric = true
803+
svcVip = svc.clusterIP.String()
804+
} else {
805+
pushMetric = false
806+
}
807+
case nsc.nodeIP.String():
808+
if protocol == ipvsSvc.Protocol && uint16(svc.port) == ipvsSvc.Port {
809+
pushMetric = true
810+
svcVip = nsc.nodeIP.String()
811+
} else {
812+
pushMetric = false
813+
}
814+
default:
815+
svcVip = ""
816+
pushMetric = false
805817
}
806-
if strings.Compare(nsc.nodeIP.String(), ipvsSvc.Address.String()) == 0 &&
807-
protocol == ipvsSvc.Protocol && uint16(svc.port) == ipvsSvc.Port {
808-
glog.Infof("Publishing prometheus metrics " + nsc.nodeIP.String() + ":" + strconv.Itoa(svc.port))
809-
serviceBpsIn.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.BPSIn))
810-
serviceBpsOut.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.BPSOut))
811-
serviceBytesIn.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.BytesIn))
812-
serviceBytesOut.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.BytesOut))
813-
serviceCPS.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.CPS))
814-
servicePacketsIn.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.PacketsIn))
815-
servicePacketsOut.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.PacketsOut))
816-
servicePpsIn.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.PPSIn))
817-
servicePpsOut.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.PPSOut))
818-
serviceTotalConn.WithLabelValues(svc.namespace, svc.name, nsc.nodeIP.String()).Set(float64(ipvsSvc.Stats.Connections))
818+
819+
if pushMetric {
820+
glog.V(3).Infof("Publishing metrics for %s/%s (%s:%d/%s)", svc.namespace, svc.name, svcVip, svc.port, svc.protocol)
821+
serviceBpsIn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.BPSIn))
822+
serviceBpsOut.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.BPSOut))
823+
serviceBytesIn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.BytesIn))
824+
serviceBytesOut.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.BytesOut))
825+
serviceCPS.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.CPS))
826+
servicePacketsIn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.PacketsIn))
827+
servicePacketsOut.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.PacketsOut))
828+
servicePpsIn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.PPSIn))
829+
servicePpsOut.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.PPSOut))
830+
serviceTotalConn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.Connections))
819831
}
820832
}
821833
}

‎dashboard/dashboard.png‎

849 KB
Loading

0 commit comments

Comments
 (0)