Skip to content

Commit d3ccfb8

Browse files
maxbrunetmaxbrunet
committed
Turn AWS VPC CNI into a control plane add-on
1 parent a330e86 commit d3ccfb8

File tree

2 files changed

+111
-107
lines changed

2 files changed

+111
-107
lines changed

‎jsonnet/kube-prometheus/addons/aws-vpc-cni.libsonnet‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
{
2+
values+:: {
3+
awsVpcCni: {
4+
// `minimumWarmIPs` should be inferior or equal to `WARM_IP_TARGET`.
5+
//
6+
// References:
7+
// https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.9.0/docs/eni-and-ip-target.md
8+
// https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.9.0/pkg/ipamd/ipamd.go#L61-L71
9+
minimumWarmIPs: 10,
10+
minimumWarmIPsTime: '10m',
11+
},
12+
},
13+
kubernetesControlPlane+: {
14+
serviceAwsVpcCni: {
15+
apiVersion: 'v1',
16+
kind: 'Service',
17+
metadata: {
18+
name: 'aws-node',
19+
namespace: 'kube-system',
20+
labels: { 'app.kubernetes.io/name': 'aws-node' },
21+
},
22+
spec: {
23+
ports: [
24+
{
25+
name: 'cni-metrics-port',
26+
port: 61678,
27+
targetPort: 61678,
28+
},
29+
],
30+
selector: { 'app.kubernetes.io/name': 'aws-node' },
31+
clusterIP: 'None',
32+
},
33+
},
34+
35+
serviceMonitorAwsVpcCni: {
36+
apiVersion: 'monitoring.coreos.com/v1',
37+
kind: 'ServiceMonitor',
38+
metadata: {
39+
name: 'aws-node',
40+
namespace: $.values.common.namespace,
41+
labels: {
42+
'app.kubernetes.io/name': 'aws-node',
43+
},
44+
},
45+
spec: {
46+
jobLabel: 'app.kubernetes.io/name',
47+
selector: {
48+
matchLabels: {
49+
'app.kubernetes.io/name': 'aws-node',
50+
},
51+
},
52+
namespaceSelector: {
53+
matchNames: [
54+
'kube-system',
55+
],
56+
},
57+
endpoints: [
58+
{
59+
port: 'cni-metrics-port',
60+
interval: '30s',
61+
path: '/metrics',
62+
relabelings: [
63+
{
64+
action: 'replace',
65+
regex: '(.*)',
66+
replacement: '$1',
67+
sourceLabels: ['__meta_kubernetes_pod_node_name'],
68+
targetLabel: 'instance',
69+
},
70+
],
71+
},
72+
],
73+
},
74+
},
75+
76+
prometheusRuleAwsVpcCni: {
77+
apiVersion: 'monitoring.coreos.com/v1',
78+
kind: 'PrometheusRule',
79+
metadata: {
80+
labels: $.prometheus._config.commonLabels + $.prometheus._config.mixin.ruleLabels,
81+
name: 'aws-vpc-cni-rules',
82+
namespace: $.prometheus._config.namespace,
83+
},
84+
spec: {
85+
groups: [
86+
{
87+
name: 'aws-vpc-cni.rules',
88+
rules: [
89+
{
90+
expr: 'sum by(instance) (awscni_total_ip_addresses) - sum by(instance) (awscni_assigned_ip_addresses) < %s' % $.values.awsVpcCni.minimumWarmIPs,
91+
labels: {
92+
severity: 'critical',
93+
},
94+
annotations: {
95+
summary: 'AWS VPC CNI has a low warm IP pool',
96+
description: |||
97+
Instance {{ $labels.instance }} has only {{ $value }} warm IPs which is lower than set threshold of %s.
98+
It could mean the current subnet is out of available IP addresses or the CNI is unable to request them from the EC2 API.
99+
||| % $.values.awsVpcCni.minimumWarmIPs,
100+
},
101+
'for': $.values.awsVpcCni.minimumWarmIPsTime,
102+
alert: 'AwsVpcCniWarmIPsLow',
103+
},
104+
],
105+
},
106+
],
107+
},
108+
},
109+
},
110+
}

‎jsonnet/kube-prometheus/platforms/eks.libsonnet‎

Lines changed: 1 addition & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,5 @@
1+
(import '../addons/aws-vpc-cni.libsonnet') +
12
(import '../addons/managed-cluster.libsonnet') + {
2-
values+:: {
3-
awsVpcCni: {
4-
// `minimumWarmIPs` should be inferior or equal to `WARM_IP_TARGET`.
5-
//
6-
// References:
7-
// https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.9.0/docs/eni-and-ip-target.md
8-
// https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.9.0/pkg/ipamd/ipamd.go#L61-L71
9-
minimumWarmIPs: 10,
10-
minimumWarmIPsTime: '10m',
11-
},
12-
},
133
kubernetesControlPlane+: {
144
serviceMonitorCoreDNS+: {
155
spec+: {
@@ -22,101 +12,5 @@
2212
],
2313
},
2414
},
25-
26-
serviceAwsVpcCniMetrics: {
27-
apiVersion: 'v1',
28-
kind: 'Service',
29-
metadata: {
30-
name: 'aws-node',
31-
namespace: 'kube-system',
32-
labels: { 'app.kubernetes.io/name': 'aws-node' },
33-
},
34-
spec: {
35-
ports: [
36-
{
37-
name: 'cni-metrics-port',
38-
port: 61678,
39-
targetPort: 61678,
40-
},
41-
],
42-
selector: { 'app.kubernetes.io/name': 'aws-node' },
43-
clusterIP: 'None',
44-
},
45-
},
46-
47-
serviceMonitorAwsVpcCni: {
48-
apiVersion: 'monitoring.coreos.com/v1',
49-
kind: 'ServiceMonitor',
50-
metadata: {
51-
name: 'aws-node',
52-
namespace: $.values.common.namespace,
53-
labels: {
54-
'app.kubernetes.io/name': 'aws-node',
55-
},
56-
},
57-
spec: {
58-
jobLabel: 'app.kubernetes.io/name',
59-
selector: {
60-
matchLabels: {
61-
'app.kubernetes.io/name': 'aws-node',
62-
},
63-
},
64-
namespaceSelector: {
65-
matchNames: [
66-
'kube-system',
67-
],
68-
},
69-
endpoints: [
70-
{
71-
port: 'cni-metrics-port',
72-
interval: '30s',
73-
path: '/metrics',
74-
relabelings: [
75-
{
76-
action: 'replace',
77-
regex: '(.*)',
78-
replacement: '$1',
79-
sourceLabels: ['__meta_kubernetes_pod_node_name'],
80-
targetLabel: 'instance',
81-
},
82-
],
83-
},
84-
],
85-
},
86-
},
87-
88-
prometheusRuleAwsVpcCni: {
89-
apiVersion: 'monitoring.coreos.com/v1',
90-
kind: 'PrometheusRule',
91-
metadata: {
92-
labels: $.prometheus._config.commonLabels + $.prometheus._config.mixin.ruleLabels,
93-
name: 'aws-vpc-cni-rules',
94-
namespace: $.prometheus._config.namespace,
95-
},
96-
spec: {
97-
groups: [
98-
{
99-
name: 'kube-prometheus-aws-vpc-cni.rules',
100-
rules: [
101-
{
102-
expr: 'sum by(instance) (awscni_total_ip_addresses) - sum by(instance) (awscni_assigned_ip_addresses) < %s' % $.values.awsVpcCni.minimumWarmIPs,
103-
labels: {
104-
severity: 'critical',
105-
},
106-
annotations: {
107-
summary: 'AWS VPC CNI has a low warm IP pool',
108-
description: |||
109-
Instance {{ $labels.instance }} has only {{ $value }} warm IPs which is lower than set threshold of %s.
110-
It could mean the current subnet is out of available IP addresses or the CNI is unable to request them from the EC2 API.
111-
||| % $.values.awsVpcCni.minimumWarmIPs,
112-
},
113-
'for': $.values.awsVpcCni.minimumWarmIPsTime,
114-
alert: 'AwsVpcCniWarmIPsLow',
115-
},
116-
],
117-
},
118-
],
119-
},
120-
},
12115
},
12216
}

0 commit comments

Comments
 (0)