Add optional support for internel node subnets

Signed-off-by: Mikkel Oscar Lyderik Larsen <[email protected]>

Author: mikkeloscar

cluster/config-defaults.yaml

@@ -1155,6 +1155,10 @@ control_plane_graceful_shutdown: "true"
 # For rolling back it needs to be done in multiple stages: active -> serving -> pre -> none
 control_plane_load_balancer_internal: "none"
 
+# Optionally use internal subnets for running the nodes. This can be configured
+# a node pool level to only run a subset of nodes in the internal subnets.
+internal_node_subnets_enabled: "false"
+
 # This allows setting custom sysctl settings. The config-item is intended to be
 # used on node-pools rather being set globally.
 #

cluster/node-pools/master-default/stack.yaml

@@ -37,9 +37,13 @@ Resources:
         PropagateAtLaunch: true
         Value: "{{ .NodePool.ConfigItems.pod_max_pids }}"
       VPCZoneIdentifier:
-{{ with $values := .Values }}
-{{ range $az := $values.availability_zones }}
-        - "{{ index $values.subnets $az }}"
+{{ with $data := . }}
+{{ range $az := $data.Values.availability_zones }}
+        # {{ if eq $data.NodePool.ConfigItems.internal_node_subnets_enabled "true" }}
+        - "{{ index $data.Values.internal_node_subnets $az }}"
+        # {{ else }}
+        - "{{ index $data.Values.subnets $az }}"
+        # {{ end }}
 {{ end }}
 {{ end }}
       TargetGroupARNs:

cluster/node-pools/worker-combined/stack.yaml

@@ -117,9 +117,13 @@ Resources:
         PropagateAtLaunch: true
         Value: "{{ .NodePool.ConfigItems.pod_max_pids }}"
       VPCZoneIdentifier:
-{{ with $values := .Values }}
-{{ range $az := $values.availability_zones }}
-        - "{{ index $values.subnets $az }}"
+{{ with $data := . }}
+{{ range $az := $data.Values.availability_zones }}
+        # {{ if eq $data.NodePool.ConfigItems.internal_node_subnets_enabled "true" }}
+        - "{{ index $data.Values.internal_node_subnets $az }}"
+        # {{ else }}
+        - "{{ index $data.Values.subnets $az }}"
+        # {{ end }}
 {{ end }}
 {{ end }}
     Type: 'AWS::AutoScaling::AutoScalingGroup'

cluster/node-pools/worker-karpenter/provisioners.yaml

@@ -17,7 +17,11 @@ spec:
     httpTokens: optional
   subnetSelectorTerms:
     - tags:
+        # {{ if eq .NodePool.ConfigItems.internal_node_subnets_enabled "true" }}
+        kubernetes.io/role/internal-node: "enabled"
+        # {{ else }}
         kubernetes.io/role/karpenter: "enabled"
+        # {{ end }}
   securityGroupSelectorTerms:
     - tags:
         karpenter.sh/discovery: "{{ .Cluster.ID }}/WorkerNodeSecurityGroup"

cluster/node-pools/worker-splitaz/stack.yaml

@@ -123,7 +123,11 @@ Resources:
         PropagateAtLaunch: true
         Value: "{{ $data.NodePool.ConfigItems.pod_max_pids }}"
       VPCZoneIdentifier:
+        # {{ if eq $data.NodePool.ConfigItems.internal_node_subnets_enabled "true" }}
+        - "{{ index $data.Values.internal_node_subnets $az }}"
+        # {{ else }}
         - "{{ index $data.Values.subnets $az }}"
+        # {{ end }}
     Type: 'AWS::AutoScaling::AutoScalingGroup'
 {{ end }}
 {{ end }}