Skip to content

Commit 48836ab

Browse files
jimmidysonjimmidyson
authored
feat: CIS benchmark improvements (#1137)
Pulling in most of the changes from #1132. This commit does not include Kubelet CSR Approver because that requires more invasive changes (e.g. deploying the `kubelet-csr-approver` service and reconfiguring kubelet after initial startup.
1 parent 02fde71 commit 48836ab

18 files changed

+445
-39
lines changed

‎charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/aws-cluster-class.yaml‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,24 +84,73 @@ spec:
8484
clusterConfiguration:
8585
apiServer:
8686
extraArgs:
87+
admission-control-config-file: /etc/kubernetes/admission.yaml
88+
enable-admission-plugins: DenyServiceExternalIPs,EventRateLimit,NodeRestriction
8789
profiling: "false"
90+
service-account-lookup: "true"
91+
extraVolumes:
92+
- hostPath: /etc/kubernetes/admission.yaml
93+
mountPath: /etc/kubernetes/admission.yaml
94+
name: admission-config
95+
pathType: File
96+
readOnly: true
97+
- hostPath: /etc/kubernetes/eventratelimit-config.yaml
98+
mountPath: /etc/kubernetes/eventratelimit-config.yaml
99+
name: eventratelimit-config
100+
pathType: File
101+
readOnly: true
88102
controllerManager:
89103
extraArgs:
90104
cloud-provider: external
91105
profiling: "false"
106+
terminated-pod-gc-threshold: "10000"
92107
scheduler:
93108
extraArgs:
94109
profiling: "false"
110+
files:
111+
- content: |
112+
apiVersion: apiserver.config.k8s.io/v1
113+
kind: AdmissionConfiguration
114+
plugins:
115+
- name: EventRateLimit
116+
path: /etc/kubernetes/eventratelimit-config.yaml
117+
path: /etc/kubernetes/admission.yaml
118+
permissions: "0600"
119+
- content: |
120+
apiVersion: eventratelimit.admission.k8s.io/v1alpha1
121+
kind: Configuration
122+
limits:
123+
- type: Server
124+
qps: 10000
125+
burst: 40000
126+
path: /etc/kubernetes/eventratelimit-config.yaml
127+
permissions: "0600"
95128
initConfiguration:
96129
nodeRegistration:
97130
kubeletExtraArgs:
98131
cloud-provider: external
132+
event-qps: "5"
133+
make-iptables-util-chains: "true"
134+
pod-max-pids: "4096"
135+
read-only-port: "0"
136+
streaming-connection-idle-timeout: 5m
137+
tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
99138
name: '{{ ds.meta_data.local_hostname }}'
100139
joinConfiguration:
101140
nodeRegistration:
102141
kubeletExtraArgs:
103142
cloud-provider: external
143+
event-qps: "5"
144+
make-iptables-util-chains: "true"
145+
pod-max-pids: "4096"
146+
read-only-port: "0"
147+
streaming-connection-idle-timeout: 5m
148+
tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
104149
name: '{{ ds.meta_data.local_hostname }}'
150+
postKubeadmCommands:
151+
- chmod 600 "$(systemctl show -P FragmentPath kubelet.service)"
152+
- chmod 600 $(systemctl show -P DropInPaths kubelet.service)
153+
- chmod 600 /var/lib/kubelet/config.yaml
105154
---
106155
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
107156
kind: AWSMachineTemplate
@@ -140,4 +189,13 @@ spec:
140189
nodeRegistration:
141190
kubeletExtraArgs:
142191
cloud-provider: external
192+
event-qps: "5"
193+
pod-max-pids: "4096"
194+
read-only-port: "0"
195+
streaming-connection-idle-timeout: 5m
196+
tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
143197
name: '{{ ds.meta_data.local_hostname }}'
198+
postKubeadmCommands:
199+
- chmod 600 "$(systemctl show -P FragmentPath kubelet.service)"
200+
- chmod 600 $(systemctl show -P DropInPaths kubelet.service)
201+
- chmod 600 /var/lib/kubelet/config.yaml

‎charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/docker-cluster-class.yaml‎

Lines changed: 64 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -83,17 +83,68 @@ spec:
8383
clusterConfiguration:
8484
apiServer:
8585
extraArgs:
86+
admission-control-config-file: /etc/kubernetes/admission.yaml
87+
enable-admission-plugins: DenyServiceExternalIPs,EventRateLimit,NodeRestriction
8688
profiling: "false"
89+
service-account-lookup: "true"
90+
extraVolumes:
91+
- hostPath: /etc/kubernetes/admission.yaml
92+
mountPath: /etc/kubernetes/admission.yaml
93+
name: admission-config
94+
pathType: File
95+
readOnly: true
96+
- hostPath: /etc/kubernetes/eventratelimit-config.yaml
97+
mountPath: /etc/kubernetes/eventratelimit-config.yaml
98+
name: eventratelimit-config
99+
pathType: File
100+
readOnly: true
87101
controllerManager:
88102
extraArgs:
89103
profiling: "false"
104+
terminated-pod-gc-threshold: "10000"
90105
scheduler:
91106
extraArgs:
92107
profiling: "false"
108+
files:
109+
- content: |
110+
apiVersion: apiserver.config.k8s.io/v1
111+
kind: AdmissionConfiguration
112+
plugins:
113+
- name: EventRateLimit
114+
path: /etc/kubernetes/eventratelimit-config.yaml
115+
path: /etc/kubernetes/admission.yaml
116+
permissions: "0600"
117+
- content: |
118+
apiVersion: eventratelimit.admission.k8s.io/v1alpha1
119+
kind: Configuration
120+
limits:
121+
- type: Server
122+
qps: 10000
123+
burst: 40000
124+
path: /etc/kubernetes/eventratelimit-config.yaml
125+
permissions: "0600"
93126
initConfiguration:
94-
nodeRegistration: {}
127+
nodeRegistration:
128+
kubeletExtraArgs:
129+
event-qps: "5"
130+
make-iptables-util-chains: "true"
131+
pod-max-pids: "4096"
132+
read-only-port: "0"
133+
streaming-connection-idle-timeout: 5m
134+
tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
95135
joinConfiguration:
96-
nodeRegistration: {}
136+
nodeRegistration:
137+
kubeletExtraArgs:
138+
event-qps: "5"
139+
make-iptables-util-chains: "true"
140+
pod-max-pids: "4096"
141+
read-only-port: "0"
142+
streaming-connection-idle-timeout: 5m
143+
tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
144+
postKubeadmCommands:
145+
- chmod 600 "$(systemctl show -P FragmentPath kubelet.service)"
146+
- chmod 600 $(systemctl show -P DropInPaths kubelet.service)
147+
- chmod 600 /var/lib/kubelet/config.yaml
97148
---
98149
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
99150
kind: DockerMachineTemplate
@@ -131,4 +182,14 @@ spec:
131182
template:
132183
spec:
133184
joinConfiguration:
134-
nodeRegistration: {}
185+
nodeRegistration:
186+
kubeletExtraArgs:
187+
event-qps: "5"
188+
pod-max-pids: "4096"
189+
read-only-port: "0"
190+
streaming-connection-idle-timeout: 5m
191+
tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
192+
postKubeadmCommands:
193+
- chmod 600 "$(systemctl show -P FragmentPath kubelet.service)"
194+
- chmod 600 $(systemctl show -P DropInPaths kubelet.service)
195+
- chmod 600 /var/lib/kubelet/config.yaml

‎charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/nutanix-cluster-class.yaml‎

Lines changed: 55 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,17 @@ spec:
1111
nodeRegistration:
1212
kubeletExtraArgs:
1313
cloud-provider: external
14+
event-qps: "5"
1415
eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10%
15-
tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
16+
pod-max-pids: "4096"
17+
read-only-port: "0"
18+
streaming-connection-idle-timeout: 5m
19+
tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
1620
postKubeadmCommands:
1721
- echo "after kubeadm call" > /var/log/postkubeadm.log
22+
- chmod 600 "$(systemctl show -P FragmentPath kubelet.service)"
23+
- chmod 600 $(systemctl show -P DropInPaths kubelet.service)
24+
- chmod 600 /var/lib/kubelet/config.yaml
1825
preKubeadmCommands:
1926
- echo "before kubeadm call" > /var/log/prekubeadm.log
2027
- hostnamectl set-hostname "{{ ds.meta_data.hostname }}"
@@ -123,12 +130,27 @@ spec:
123130
clusterConfiguration:
124131
apiServer:
125132
extraArgs:
133+
admission-control-config-file: /etc/kubernetes/admission.yaml
134+
enable-admission-plugins: DenyServiceExternalIPs,EventRateLimit,NodeRestriction
126135
profiling: "false"
136+
service-account-lookup: "true"
127137
tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
138+
extraVolumes:
139+
- hostPath: /etc/kubernetes/admission.yaml
140+
mountPath: /etc/kubernetes/admission.yaml
141+
name: admission-config
142+
pathType: File
143+
readOnly: true
144+
- hostPath: /etc/kubernetes/eventratelimit-config.yaml
145+
mountPath: /etc/kubernetes/eventratelimit-config.yaml
146+
name: eventratelimit-config
147+
pathType: File
148+
readOnly: true
128149
controllerManager:
129150
extraArgs:
130151
cloud-provider: external
131152
profiling: "false"
153+
terminated-pod-gc-threshold: "10000"
132154
tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
133155
scheduler:
134156
extraArgs:
@@ -199,21 +221,49 @@ spec:
199221
path: /etc/kubernetes/admin.conf
200222
name: kubeconfig
201223
path: /etc/kubernetes/manifests/kube-vip.yaml
224+
- content: |
225+
apiVersion: apiserver.config.k8s.io/v1
226+
kind: AdmissionConfiguration
227+
plugins:
228+
- name: EventRateLimit
229+
path: /etc/kubernetes/eventratelimit-config.yaml
230+
path: /etc/kubernetes/admission.yaml
231+
permissions: "0600"
232+
- content: |
233+
apiVersion: eventratelimit.admission.k8s.io/v1alpha1
234+
kind: Configuration
235+
limits:
236+
- type: Server
237+
qps: 10000
238+
burst: 40000
239+
path: /etc/kubernetes/eventratelimit-config.yaml
240+
permissions: "0600"
202241
initConfiguration:
203242
nodeRegistration:
204243
kubeletExtraArgs:
205244
cloud-provider: external
245+
event-qps: "5"
206246
eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10%
207-
tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
247+
make-iptables-util-chains: "true"
248+
pod-max-pids: "4096"
249+
read-only-port: "0"
250+
streaming-connection-idle-timeout: 5m
251+
tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
208252
joinConfiguration:
209253
nodeRegistration:
210254
kubeletExtraArgs:
211255
cloud-provider: external
256+
event-qps: "5"
212257
eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10%
213-
tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
258+
make-iptables-util-chains: "true"
259+
pod-max-pids: "4096"
260+
read-only-port: "0"
261+
streaming-connection-idle-timeout: 5m
262+
tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
214263
postKubeadmCommands:
215-
- echo export KUBECONFIG=/etc/kubernetes/admin.conf >> /root/.bashrc
216-
- echo "after kubeadm call" > /var/log/postkubeadm.log
264+
- chmod 600 "$(systemctl show -P FragmentPath kubelet.service)"
265+
- chmod 600 $(systemctl show -P DropInPaths kubelet.service)
266+
- chmod 600 /var/lib/kubelet/config.yaml
217267
preKubeadmCommands:
218268
- echo "before kubeadm call" > /var/log/prekubeadm.log
219269
- hostnamectl set-hostname "{{ ds.meta_data.hostname }}"

‎hack/examples/overlays/clusterclasses/aws/kustomization.yaml.tmpl‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,5 +68,29 @@ patches:
6868
# BEGIN CIS patches
6969
- target:
7070
kind: KubeadmControlPlaneTemplate
71-
path: ../../../patches/disable-kubeadmcontrolplane-profiling.yaml
71+
path: ../../../patches/cis-kubeadmcontrolplanetemplate.yaml
72+
- target:
73+
kind: KubeadmConfigTemplate
74+
path: ../../../patches/cis-kubeadmconfigtemplate.yaml
75+
- target:
76+
kind: KubeadmControlPlaneTemplate
77+
path: ../../../patches/initialize-kubeadmcontrolplanetemplate-postkubeadmcommands.yaml
78+
- target:
79+
kind: KubeadmControlPlaneTemplate
80+
path: ../../../patches/cis-kubeadmcontrolplanetemplate-kubelet-file-permissions.yaml
81+
- target:
82+
kind: KubeadmConfigTemplate
83+
path: ../../../patches/initialize-kubeadmconfigtemplate-postkubeadmcommands.yaml
84+
- target:
85+
kind: KubeadmConfigTemplate
86+
path: ../../../patches/cis-kubeadmconfigtemplate-kubelet-file-permissions.yaml
87+
- target:
88+
kind: KubeadmControlPlaneTemplate
89+
path: ../../../patches/initialize-files.yaml
90+
- target:
91+
kind: KubeadmControlPlaneTemplate
92+
path: ../../../patches/initialize-extravolumes.yaml
93+
- target:
94+
kind: KubeadmControlPlaneTemplate
95+
path: ../../../patches/cis-admissionconfiguration.yaml
7296
# END CIS patches

‎hack/examples/overlays/clusterclasses/docker/kustomization.yaml.tmpl‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,29 @@ patches:
2929
# BEGIN CIS patches
3030
- target:
3131
kind: KubeadmControlPlaneTemplate
32-
path: ../../../patches/disable-kubeadmcontrolplane-profiling.yaml
32+
path: ../../../patches/cis-kubeadmcontrolplanetemplate.yaml
33+
- target:
34+
kind: KubeadmConfigTemplate
35+
path: ../../../patches/cis-kubeadmconfigtemplate.yaml
36+
- target:
37+
kind: KubeadmControlPlaneTemplate
38+
path: ../../../patches/initialize-kubeadmcontrolplanetemplate-postkubeadmcommands.yaml
39+
- target:
40+
kind: KubeadmControlPlaneTemplate
41+
path: ../../../patches/cis-kubeadmcontrolplanetemplate-kubelet-file-permissions.yaml
42+
- target:
43+
kind: KubeadmConfigTemplate
44+
path: ../../../patches/initialize-kubeadmconfigtemplate-postkubeadmcommands.yaml
45+
- target:
46+
kind: KubeadmConfigTemplate
47+
path: ../../../patches/cis-kubeadmconfigtemplate-kubelet-file-permissions.yaml
48+
- target:
49+
kind: KubeadmControlPlaneTemplate
50+
path: ../../../patches/initialize-files.yaml
51+
- target:
52+
kind: KubeadmControlPlaneTemplate
53+
path: ../../../patches/initialize-extravolumes.yaml
54+
- target:
55+
kind: KubeadmControlPlaneTemplate
56+
path: ../../../patches/cis-admissionconfiguration.yaml
3357
# END CIS patches

‎hack/examples/overlays/clusterclasses/nutanix/kustomization.yaml.tmpl‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,23 @@ patches:
2929
# BEGIN CIS patches
3030
- target:
3131
kind: KubeadmControlPlaneTemplate
32-
path: ../../../patches/disable-kubeadmcontrolplane-profiling.yaml
32+
path: ../../../patches/cis-kubeadmcontrolplanetemplate.yaml
33+
- target:
34+
kind: KubeadmConfigTemplate
35+
path: ../../../patches/cis-kubeadmconfigtemplate.yaml
36+
- target:
37+
kind: KubeadmControlPlaneTemplate
38+
path: ../../../patches/initialize-kubeadmcontrolplanetemplate-postkubeadmcommands.yaml
39+
- target:
40+
kind: KubeadmControlPlaneTemplate
41+
path: ../../../patches/cis-kubeadmcontrolplanetemplate-kubelet-file-permissions.yaml
42+
- target:
43+
kind: KubeadmConfigTemplate
44+
path: ../../../patches/cis-kubeadmconfigtemplate-kubelet-file-permissions.yaml
45+
- target:
46+
kind: KubeadmControlPlaneTemplate
47+
path: ../../../patches/initialize-extravolumes.yaml
48+
- target:
49+
kind: KubeadmControlPlaneTemplate
50+
path: ../../../patches/cis-admissionconfiguration.yaml
3351
# END CIS patches

0 commit comments

Comments
 (0)