Skip to content

Commit 9058562

Browse files
mikkeloscarmikkeloscar
committed
Optional internal control plane LB
Signed-off-by: Mikkel Oscar Lyderik Larsen <[email protected]>
1 parent 7ccfa4e commit 9058562

File tree

4 files changed

+97
-0
lines changed

4 files changed

+97
-0
lines changed

‎cluster/cluster.yaml

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,78 @@ Resources:
5252
Value: owned
5353
ToPort: 10250
5454
Type: 'AWS::EC2::SecurityGroupIngress'
55+
{{- if or (eq .Cluster.ConfigItems.control_plane_load_balancer_internal "pre") (eq .Cluster.ConfigItems.control_plane_load_balancer_internal "serving") (eq .Cluster.ConfigItems.control_plane_load_balancer_internal "active") }}
56+
ControlPlaneInternalLB:
57+
Type: AWS::ElasticLoadBalancingV2::LoadBalancer
58+
Properties:
59+
Name: "{{.Cluster.LocalID}}-nlb-internal"
60+
LoadBalancerAttributes:
61+
- Key: load_balancing.cross_zone.enabled
62+
Value: true
63+
Scheme: internal
64+
Subnets:
65+
{{ with $values := .Values }}
66+
{{ range $az := $values.availability_zones }}
67+
- "{{ index $values.lb_subnets $az }}"
68+
{{ end }}
69+
{{ end }}
70+
Tags:
71+
- Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
72+
Value: owned
73+
- Key: "component"
74+
Value: "kube-apiserver"
75+
Type: network
76+
ControlPlaneInternalLBTargetGroup:
77+
Type: AWS::ElasticLoadBalancingV2::TargetGroup
78+
Properties:
79+
HealthCheckIntervalSeconds: 10
80+
HealthCheckPort: 8443
81+
HealthCheckProtocol: HTTPS
82+
HealthCheckPath: "/readyz"
83+
HealthyThresholdCount: 2
84+
UnhealthyThresholdCount: 2
85+
Name: "{{.Cluster.LocalID}}-nlb-internal"
86+
Port: 8443
87+
Protocol: TLS
88+
Tags:
89+
- Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
90+
Value: owned
91+
- Key: "component"
92+
Value: "kube-apiserver"
93+
VpcId: "{{.Cluster.ConfigItems.vpc_id}}"
94+
TargetGroupAttributes:
95+
- Key: deregistration_delay.timeout_seconds
96+
Value: 60
97+
- Key: preserve_client_ip.enabled
98+
Value: "false"
99+
ControlPlaneInternalLBListener:
100+
Type: AWS::ElasticLoadBalancingV2::Listener
101+
Properties:
102+
AlpnPolicy:
103+
- {{ if eq .Cluster.ConfigItems.experimental_nlb_alpn_h2_enabled "true" }}HTTP2Preferred{{else}}None{{end}}
104+
SslPolicy: "ELBSecurityPolicy-TLS-1-2-2017-01"
105+
Certificates:
106+
- CertificateArn: "{{.Values.load_balancer_certificate}}"
107+
DefaultActions:
108+
- Type: forward
109+
TargetGroupArn: !Ref ControlPlaneInternalLBTargetGroup
110+
LoadBalancerArn: !Ref ControlPlaneInternalLB
111+
Port: 443
112+
Protocol: TLS
113+
ControlPlaneInternalLBVersionDomain:
114+
Properties:
115+
AliasTarget:
116+
DNSName: !GetAtt
117+
- ControlPlaneInternalLB
118+
- DNSName
119+
HostedZoneId: !GetAtt
120+
- ControlPlaneInternalLB
121+
- CanonicalHostedZoneID
122+
HostedZoneName: "{{.Values.hosted_zone}}."
123+
Name: "{{.Cluster.LocalID}}-internal.{{.Values.hosted_zone}}."
124+
Type: A
125+
Type: 'AWS::Route53::RecordSet'
126+
{{- end }}
55127
MasterLoadBalancerNLB:
56128
Type: AWS::ElasticLoadBalancingV2::LoadBalancer
57129
Properties:
@@ -2503,6 +2575,12 @@ Outputs:
25032575
Export:
25042576
Name: '{{.Cluster.ID}}:master-load-balancer-nlb-target-group'
25052577
Value: !Ref MasterLoadBalancerNLBTargetGroup
2578+
{{- if or (eq .Cluster.ConfigItems.control_plane_load_balancer_internal "pre") (eq .Cluster.ConfigItems.control_plane_load_balancer_internal "serving") (eq .Cluster.ConfigItems.control_plane_load_balancer_internal "active") }}
2579+
ControlPlaneInternalLBTargetGroup:
2580+
Export:
2581+
Name: '{{.Cluster.ID}}:control-plane-internal-lb-target-group'
2582+
Value: !Ref ControlPlaneInternalLBTargetGroup
2583+
{{- end }}
25062584
MasterSecurityGroup:
25072585
Export:
25082586
Name: '{{.Cluster.ID}}:master-security-group'

‎cluster/config-defaults.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1136,6 +1136,18 @@ control_plane_asg_lifecycle_hook: "false"
11361136
# enable graceful shutdown on the control_plane nodes
11371137
control_plane_graceful_shutdown: "true"
11381138

1139+
# Optionally enable an internal load balancer for the control plane nodes
1140+
# additionally to the public load balancer.
1141+
#
1142+
# Possible values:
1143+
# none - Load Balancer is not created
1144+
# pre - Load Balancer is created but not attached to control plane nodes
1145+
# serving - Load Balancer is created and attached to control plane nodes.
1146+
# active - Load Balancer is being actively used by worker nodes.
1147+
#
1148+
# For rolling back it needs to be done in multiple stages: active -> serving -> pre -> none
1149+
control_plane_load_balancer_internal: "none"
1150+
11391151
# This allows setting custom sysctl settings. The config-item is intended to be
11401152
# used on node-pools rather being set globally.
11411153
#

‎cluster/node-pools/master-default/stack.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ Resources:
4444
{{ end }}
4545
TargetGroupARNs:
4646
- !ImportValue '{{ .Cluster.ID }}:master-load-balancer-nlb-target-group'
47+
{{- if or (eq .Cluster.ConfigItems.control_plane_load_balancer_internal "serving") (eq .Cluster.ConfigItems.control_plane_load_balancer_internal "active") }}
48+
- !ImportValue '{{ .Cluster.ID }}:control-plane-internal-lb-target-group'
49+
{{- end}}
4750
Type: 'AWS::AutoScaling::AutoScalingGroup'
4851
LaunchTemplate:
4952
Properties:

‎cluster/node-pools/worker-splitaz/userdata.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,11 @@ write_files:
3434
clusters:
3535
- name: local
3636
cluster:
37+
{{- if eq .Cluster.ConfigItems.control_plane_load_balancer_internal "active" }}
38+
server: "https://{{.Cluster.LocalID}}-internal.{{.Values.hosted_zone}}"
39+
{{- else }}
3740
server: {{ .Cluster.APIServerURL }}
41+
{{- end }}
3842
users:
3943
- name: kubelet
4044
user:

0 commit comments

Comments
 (0)