refactor: move deprecated Kubelet flags to KubeletConfiguration (#1259)

**What problem does this PR solve?**:
Moving deprecated Kubelet flags to use KubeletConfiguration mechanism
using CAPI's Patch API
https://cluster-api.sigs.k8s.io/tasks/bootstrap/kubeadm-bootstrap/kubelet-config.html?highlight=KubeletConfiguration#use-kubeadms-kubeletconfiguration-patch-target

Besides removing deprecated configuration on the Kubelet, this change
also allows us to enable additional configuration, ie
`maxParallelImagePulls` that is not available through flags.

This change is safe because there are no Handlers changes and these
templates are part of ClusterClasses that would be versioned during
upgrades.

**Which issue(s) this PR fixes**:
Fixes #

**How Has This Been Tested?**:
<!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
-->

**Special notes for your reviewer**:
<!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
-->

Author: dkoshkin

charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/aws-cluster-class.yaml

@@ -108,6 +108,33 @@ spec:
             extraArgs:
               profiling: "false"
         files:
+        - content: |
+            apiVersion: kubelet.config.k8s.io/v1beta1
+            kind: KubeletConfiguration
+            evictionHard:
+              nodefs.available: "10%"
+              nodefs.inodesFree: "5%"
+              imagefs.available: "15%"
+              memory.available: "100Mi"
+              imagefs.inodesFree: "10%"
+          path: /etc/kubernetes/patches/kubeletconfiguration0+strategic.json
+          permissions: "0600"
+        - content: |
+            apiVersion: kubelet.config.k8s.io/v1beta1
+            kind: KubeletConfiguration
+            # 4.2.4 Ensure that the --read-only-port argument is set to 0
+            readOnlyPort: 0
+            # 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0
+            # Recommendation: Set to 5m instead of 4h as per CIS guidelines
+            streamingConnectionIdleTimeout: "5m"
+            # 4.2.8 Ensure that the event-qps argument is set to a level which ensures appropriate event capture
+            eventRecordQPS: 5
+            # 4.2.12 Updated with recommended strong cipher suites
+            tlsCipherSuites: [TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384]
+            # 4.2.13 Ensure that a limit is set on pod PIDs
+            podPidsLimit: 4096
+          path: /etc/kubernetes/patches/kubeletconfiguration1+strategic.json
+          permissions: "0600"
         - content: |
             apiVersion: apiserver.config.k8s.io/v1
             kind: AdmissionConfiguration
@@ -129,24 +156,16 @@ spec:
           nodeRegistration:
             kubeletExtraArgs:
               cloud-provider: external
-              event-qps: "5"
-              make-iptables-util-chains: "true"
-              pod-max-pids: "4096"
-              read-only-port: "0"
-              streaming-connection-idle-timeout: 5m
-              tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
             name: '{{ ds.meta_data.local_hostname }}'
+          patches:
+            directory: /etc/kubernetes/patches
         joinConfiguration:
           nodeRegistration:
             kubeletExtraArgs:
               cloud-provider: external
-              event-qps: "5"
-              make-iptables-util-chains: "true"
-              pod-max-pids: "4096"
-              read-only-port: "0"
-              streaming-connection-idle-timeout: 5m
-              tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
             name: '{{ ds.meta_data.local_hostname }}'
+          patches:
+            directory: /etc/kubernetes/patches
         postKubeadmCommands:
         - chmod 600 "$(systemctl show -P FragmentPath kubelet.service)"
         - chmod 600 $(systemctl show -P DropInPaths kubelet.service)
@@ -185,16 +204,41 @@ metadata:
 spec:
   template:
     spec:
+      files:
+      - content: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          evictionHard:
+            nodefs.available: "10%"
+            nodefs.inodesFree: "5%"
+            imagefs.available: "15%"
+            memory.available: "100Mi"
+            imagefs.inodesFree: "10%"
+        path: /etc/kubernetes/patches/kubeletconfiguration0+strategic.json
+        permissions: "0600"
+      - content: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          # 4.2.4 Ensure that the --read-only-port argument is set to 0
+          readOnlyPort: 0
+          # 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0
+          # Recommendation: Set to 5m instead of 4h as per CIS guidelines
+          streamingConnectionIdleTimeout: "5m"
+          # 4.2.8 Ensure that the event-qps argument is set to a level which ensures appropriate event capture
+          eventRecordQPS: 5
+          # 4.2.12 Updated with recommended strong cipher suites
+          tlsCipherSuites: [TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384]
+          # 4.2.13 Ensure that a limit is set on pod PIDs
+          podPidsLimit: 4096
+        path: /etc/kubernetes/patches/kubeletconfiguration1+strategic.json
+        permissions: "0600"
       joinConfiguration:
         nodeRegistration:
           kubeletExtraArgs:
             cloud-provider: external
-            event-qps: "5"
-            pod-max-pids: "4096"
-            read-only-port: "0"
-            streaming-connection-idle-timeout: 5m
-            tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
           name: '{{ ds.meta_data.local_hostname }}'
+        patches:
+          directory: /etc/kubernetes/patches
       postKubeadmCommands:
       - chmod 600 "$(systemctl show -P FragmentPath kubelet.service)"
       - chmod 600 $(systemctl show -P DropInPaths kubelet.service)

charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/docker-cluster-class.yaml

@@ -106,6 +106,22 @@ spec:
             extraArgs:
               profiling: "false"
         files:
+        - content: |
+            apiVersion: kubelet.config.k8s.io/v1beta1
+            kind: KubeletConfiguration
+            # 4.2.4 Ensure that the --read-only-port argument is set to 0
+            readOnlyPort: 0
+            # 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0
+            # Recommendation: Set to 5m instead of 4h as per CIS guidelines
+            streamingConnectionIdleTimeout: "5m"
+            # 4.2.8 Ensure that the event-qps argument is set to a level which ensures appropriate event capture
+            eventRecordQPS: 5
+            # 4.2.12 Updated with recommended strong cipher suites
+            tlsCipherSuites: [TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384]
+            # 4.2.13 Ensure that a limit is set on pod PIDs
+            podPidsLimit: 4096
+          path: /etc/kubernetes/patches/kubeletconfiguration1+strategic.json
+          permissions: "0600"
         - content: |
             apiVersion: apiserver.config.k8s.io/v1
             kind: AdmissionConfiguration
@@ -124,23 +140,13 @@ spec:
           path: /etc/kubernetes/eventratelimit-config.yaml
           permissions: "0600"
         initConfiguration:
-          nodeRegistration:
-            kubeletExtraArgs:
-              event-qps: "5"
-              make-iptables-util-chains: "true"
-              pod-max-pids: "4096"
-              read-only-port: "0"
-              streaming-connection-idle-timeout: 5m
-              tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+          nodeRegistration: {}
+          patches:
+            directory: /etc/kubernetes/patches
         joinConfiguration:
-          nodeRegistration:
-            kubeletExtraArgs:
-              event-qps: "5"
-              make-iptables-util-chains: "true"
-              pod-max-pids: "4096"
-              read-only-port: "0"
-              streaming-connection-idle-timeout: 5m
-              tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+          nodeRegistration: {}
+          patches:
+            directory: /etc/kubernetes/patches
         postKubeadmCommands:
         - chmod 600 "$(systemctl show -P FragmentPath kubelet.service)"
         - chmod 600 $(systemctl show -P DropInPaths kubelet.service)
@@ -181,14 +187,27 @@ metadata:
 spec:
   template:
     spec:
+      files:
+      - content: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          # 4.2.4 Ensure that the --read-only-port argument is set to 0
+          readOnlyPort: 0
+          # 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0
+          # Recommendation: Set to 5m instead of 4h as per CIS guidelines
+          streamingConnectionIdleTimeout: "5m"
+          # 4.2.8 Ensure that the event-qps argument is set to a level which ensures appropriate event capture
+          eventRecordQPS: 5
+          # 4.2.12 Updated with recommended strong cipher suites
+          tlsCipherSuites: [TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384]
+          # 4.2.13 Ensure that a limit is set on pod PIDs
+          podPidsLimit: 4096
+        path: /etc/kubernetes/patches/kubeletconfiguration1+strategic.json
+        permissions: "0600"
       joinConfiguration:
-        nodeRegistration:
-          kubeletExtraArgs:
-            event-qps: "5"
-            pod-max-pids: "4096"
-            read-only-port: "0"
-            streaming-connection-idle-timeout: 5m
-            tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+        nodeRegistration: {}
+        patches:
+          directory: /etc/kubernetes/patches
       postKubeadmCommands:
       - chmod 600 "$(systemctl show -P FragmentPath kubelet.service)"
       - chmod 600 $(systemctl show -P DropInPaths kubelet.service)

charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/nutanix-cluster-class.yaml

@@ -7,16 +7,40 @@ metadata:
 spec:
   template:
     spec:
+      files:
+      - content: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          evictionHard:
+            nodefs.available: "10%"
+            nodefs.inodesFree: "5%"
+            imagefs.available: "15%"
+            memory.available: "100Mi"
+            imagefs.inodesFree: "10%"
+        path: /etc/kubernetes/patches/kubeletconfiguration0+strategic.json
+        permissions: "0600"
+      - content: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          # 4.2.4 Ensure that the --read-only-port argument is set to 0
+          readOnlyPort: 0
+          # 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0
+          # Recommendation: Set to 5m instead of 4h as per CIS guidelines
+          streamingConnectionIdleTimeout: "5m"
+          # 4.2.8 Ensure that the event-qps argument is set to a level which ensures appropriate event capture
+          eventRecordQPS: 5
+          # 4.2.12 Updated with recommended strong cipher suites
+          tlsCipherSuites: [TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384]
+          # 4.2.13 Ensure that a limit is set on pod PIDs
+          podPidsLimit: 4096
+        path: /etc/kubernetes/patches/kubeletconfiguration1+strategic.json
+        permissions: "0600"
       joinConfiguration:
         nodeRegistration:
           kubeletExtraArgs:
             cloud-provider: external
-            event-qps: "5"
-            eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10%
-            pod-max-pids: "4096"
-            read-only-port: "0"
-            streaming-connection-idle-timeout: 5m
-            tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+        patches:
+          directory: /etc/kubernetes/patches
       postKubeadmCommands:
       - echo "after kubeadm call" > /var/log/postkubeadm.log
       - chmod 600 "$(systemctl show -P FragmentPath kubelet.service)"
@@ -134,7 +158,6 @@ spec:
               enable-admission-plugins: DenyServiceExternalIPs,EventRateLimit,NodeRestriction
               profiling: "false"
               service-account-lookup: "true"
-              tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
             extraVolumes:
             - hostPath: /etc/kubernetes/admission.yaml
               mountPath: /etc/kubernetes/admission.yaml
@@ -221,6 +244,33 @@ spec:
                     path: /etc/kubernetes/admin.conf
                   name: kubeconfig
           path: /etc/kubernetes/manifests/kube-vip.yaml
+        - content: |
+            apiVersion: kubelet.config.k8s.io/v1beta1
+            kind: KubeletConfiguration
+            evictionHard:
+              nodefs.available: "10%"
+              nodefs.inodesFree: "5%"
+              imagefs.available: "15%"
+              memory.available: "100Mi"
+              imagefs.inodesFree: "10%"
+          path: /etc/kubernetes/patches/kubeletconfiguration0+strategic.json
+          permissions: "0600"
+        - content: |
+            apiVersion: kubelet.config.k8s.io/v1beta1
+            kind: KubeletConfiguration
+            # 4.2.4 Ensure that the --read-only-port argument is set to 0
+            readOnlyPort: 0
+            # 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0
+            # Recommendation: Set to 5m instead of 4h as per CIS guidelines
+            streamingConnectionIdleTimeout: "5m"
+            # 4.2.8 Ensure that the event-qps argument is set to a level which ensures appropriate event capture
+            eventRecordQPS: 5
+            # 4.2.12 Updated with recommended strong cipher suites
+            tlsCipherSuites: [TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384]
+            # 4.2.13 Ensure that a limit is set on pod PIDs
+            podPidsLimit: 4096
+          path: /etc/kubernetes/patches/kubeletconfiguration1+strategic.json
+          permissions: "0600"
         - content: |
             apiVersion: apiserver.config.k8s.io/v1
             kind: AdmissionConfiguration
@@ -242,24 +292,14 @@ spec:
           nodeRegistration:
             kubeletExtraArgs:
               cloud-provider: external
-              event-qps: "5"
-              eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10%
-              make-iptables-util-chains: "true"
-              pod-max-pids: "4096"
-              read-only-port: "0"
-              streaming-connection-idle-timeout: 5m
-              tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+          patches:
+            directory: /etc/kubernetes/patches
         joinConfiguration:
           nodeRegistration:
             kubeletExtraArgs:
               cloud-provider: external
-              event-qps: "5"
-              eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10%
-              make-iptables-util-chains: "true"
-              pod-max-pids: "4096"
-              read-only-port: "0"
-              streaming-connection-idle-timeout: 5m
-              tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+          patches:
+            directory: /etc/kubernetes/patches
         postKubeadmCommands:
         - chmod 600 "$(systemctl show -P FragmentPath kubelet.service)"
         - chmod 600 $(systemctl show -P DropInPaths kubelet.service)

hack/examples/bases/aws/cluster/kustomization.yaml.tmpl

@@ -26,7 +26,7 @@ patches:
       path: "/metadata/labels"
 - target:
     kind: Cluster
-  path: ../../../patches/initialize-variables.yaml
+  path: ../../../patches/initialize/variables.yaml
 - target:
     kind: Cluster
   path: ../../../patches/cluster-network.yaml

hack/examples/bases/docker/cluster/kustomization.yaml.tmpl

@@ -26,7 +26,7 @@ patches:
       value: "docker-quick-start"
 - target:
     kind: Cluster
-  path: ../../../patches/initialize-variables.yaml
+  path: ../../../patches/initialize/variables.yaml
 - target:
     kind: Cluster
   path: ../../../patches/cluster-network.yaml

hack/examples/bases/nutanix/cluster/kustomization.yaml.tmpl

@@ -20,7 +20,7 @@ labels:
 patches:
 - target:
     kind: Cluster
-  path: ../../../patches/initialize-variables.yaml
+  path: ../../../patches/initialize/variables.yaml
 - target:
     kind: Cluster
   path: ../../../patches/cluster-network.yaml