From 6771cdb4bd495b921bd44cffd098af78014b45d3 Mon Sep 17 00:00:00 2001 From: WinsonSou Date: Wed, 21 May 2025 19:11:39 +0530 Subject: [PATCH 1/4] Included hardening steps --- .../control-plane/README.md | 159 ++++++++++++++++ .../cis-mitigations-cp-patch.yaml | 106 +++++++++++ .../control-plane/kustomization.yaml | 11 ++ .../worker/README.md | 175 ++++++++++++++++++ .../worker/cis-mitigations-worker-patch.yaml | 28 +++ .../worker/kustomization.yaml | 11 ++ 6 files changed, 490 insertions(+) create mode 100644 examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/README.md create mode 100644 examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/cis-mitigations-cp-patch.yaml create mode 100644 examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/kustomization.yaml create mode 100644 examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/README.md create mode 100644 examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/cis-mitigations-worker-patch.yaml create mode 100644 examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/kustomization.yaml diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/README.md b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/README.md new file mode 100644 index 000000000..4d6c50283 --- /dev/null +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/README.md @@ -0,0 +1,159 @@ +# Hardened Kubernetes Control Plane Template + +## Required Component: kubelet-csr-approver + +To meet the following CIS security benchmarks: + +- 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate +- 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true + +You must install the Postfinance kubelet-csr-approver: + +```bash +helm repo add kubelet-csr-approver https://postfinance.github.io/kubelet-csr-approver +helm upgrade --install kubelet-csr-approver \ + kubelet-csr-approver/kubelet-csr-approver \ + -n kube-system \ + --create-namespace \ + --set maxExpirationSeconds=2592000 \ + --set leaderElection=true \ + --set bypassDnsResolution=true \ + --set rbac.create=true +``` + +**Note**: If you choose not to install the kubelet-csr-approver, you must omit the flags related to the CIS benchmarks mentioned above from your configuration. + +## Prerequisites + +Before applying the kustomization, you need to determine the available Nutanix-provided KubeadmControlPlaneTemplates (those with the "nkp-" prefix) and their versions: + +```bash +# List all available KubeadmControlPlaneTemplates +kubectl get kubeadmcontrolplanetemplates.controlplane.cluster.x-k8s.io + +# Example output: +# NAME AGE +# nkp-nutanix-v2.14.0 4d +# some-other-template 19h +``` + +**Important**: You must use the Nutanix-provided template (starting with "nkp-") for these hardening configurations to work correctly. + +Then export the original Nutanix KubeadmControlPlaneTemplate using the appropriate version: + +```bash +# Replace with your actual version (e.g., v2.14.0) +kubectl get kubeadmcontrolplanetemplates.controlplane.cluster.x-k8s.io nkp-nutanix- -o yaml > nkp-nutanix-.yaml +``` + +## Structure + +- `nkp-nutanix-.yaml` - The original KubeadmControlPlaneTemplate (generated by user, replace with your actual version) +- `cis-mitigations-cp-patch.yaml` - Patch file containing CIS hardening configurations +- `kustomization.yaml` - Kustomization file that applies the patch and renames the template + +**Note**: You need to edit BOTH of these files with your actual version: + +1. Edit the `cis-mitigations-cp-patch.yaml` file to replace `` with your actual version in the `metadata.name` field: + +```yaml +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlaneTemplate +metadata: + name: nkp-nutanix- # <-- Replace with your actual version +``` + +2. Edit the `kustomization.yaml` file to replace `` in the resources section: + +```yaml +resources: +- nkp-nutanix-.yaml # <-- Replace with your actual version +``` + +## Applying the Kustomization + +You can apply this kustomization directly with kubectl: + +```bash +#Ensure you are in a directory for the kustomization.yaml, cis-mitigations-cp-patch.yaml and the nkp-nutanix-.yaml file +kubectl apply -k . +``` + +## Verification + +The kustomization will: + +1. Take the user-generated KubeadmControlPlaneTemplate `nkp-nutanix-` +2. Apply the CIS mitigations patch that adds: + - API Server hardening (profiling disabled, service account lookup, admission plugins) + - Controller Manager settings (terminated pod GC threshold, profiling disabled) + - Scheduler settings (profiling disabled) + - Required configuration files for EventRateLimit + - Additional postKubeadmCommands for file permission hardening +3. Rename it to `nkp-nutanix--hardened` + +## CIS Mitigations Applied + +The following CIS mitigations are applied: + +### API Server + +- **1.2.15**: Disabled profiling for API server (`profiling: "false"`) +- **1.2.21**: Enabled service account lookup (`service-account-lookup: "true"`) +- **1.2.3, 1.2.9, 1.2.11, 1.2.14**: Added admission plugins: + - AlwaysPullImages: Enforces that images are always pulled prior to starting containers + - DenyServiceExternalIPs: Prevents services from using arbitrary external IPs + - EventRateLimit: Mitigates event flooding attacks + - NodeRestriction: Limits node access to specific APIs +- **1.2.9**: Configured EventRateLimit with appropriate admission control config file +- **1.2.5**: Set kubelet certificate authority (`kubelet-certificate-authority: /etc/kubernetes/pki/ca.crt`) + - **Note**: This requires kubelet-csr-approver to be installed. If not installed, this flag should be omitted. + +### Controller Manager + +- **1.3.1**: Set terminated pod GC threshold to 10000 for better garbage collection +- **1.3.2**: Disabled profiling (`profiling: "false"`) +- **1.3.6**: Enabled RotateKubeletServerCertificate feature gate (`feature-gates: RotateKubeletServerCertificate=true`) + - **Note**: This requires kubelet-csr-approver to be installed. If not installed, this flag should be omitted. + +### Scheduler + +- **1.4.1**: Disabled profiling (`profiling: "false"`) + +### Kubelet Configuration (Both Init and Join) + +- **1.2.5, 1.3.6**: Enabled kubelet server certificate rotation (`rotate-server-certificates: "true"`) + - **Note**: This requires kubelet-csr-approver to be installed. If not installed, this flag should be omitted. + +### EventRateLimit Configuration + +- **1.2.9**: Created admission configuration files with detailed rate limits: + - Server-wide: 5000 QPS with 20000 burst + - Namespace: 500 QPS with 2000 burst (1000 cache size) + - User: 100 QPS with 400 burst (2000 cache size) + - SourceAndObject: 50 QPS with 100 burst (5000 cache size) + +### File Permissions + +- **4.1.1**: Set appropriate file permissions (0600) for sensitive files including: + - kubelet.service + - kubelet config.yaml + - 10-kubeadm.conf + +## Applying to Cluster Class + +After generating the hardened template, you need to update the cluster class to use it: + +1. First, identify the cluster class name and version: + +```bash +kubectl get clusterclasses.cluster.x-k8s.io +``` + +2. Patch the cluster class to use the hardened control plane template (replace `` and cluster class name as needed): + +```bash +kubectl patch clusterclass nkp-nutanix- \ + --type merge \ + -p='{"spec":{"controlPlane":{"ref":{"name":"nkp-nutanix--hardened"}}}}' +``` diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/cis-mitigations-cp-patch.yaml b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/cis-mitigations-cp-patch.yaml new file mode 100644 index 000000000..4a8744859 --- /dev/null +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/cis-mitigations-cp-patch.yaml @@ -0,0 +1,106 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlaneTemplate +metadata: + name: nkp-nutanix- +spec: + template: + spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + #1.2.15 Ensure that the --profiling argument is set to false + profiling: "false" + #1.2.21 Ensure that the --service-account-lookup argument is set to true + service-account-lookup: "true" + #1.2.3 Ensure that the DenyServiceExternalIPs is set + #1.2.9 Ensure that the admission control plugin EventRateLimit is set + #1.2.11 Ensure that the admission control plugin AlwaysPullImages is set + #1.2.14 Ensure that the admission control plugin NodeRestriction is set + enable-admission-plugins: AlwaysPullImages,EventRateLimit,DenyServiceExternalIPs,NodeRestriction + #1.2.9 Ensure that the admission control plugin EventRateLimit is set + admission-control-config-file: /etc/kubernetes/admission/admissionConfiguration.yaml + #1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate + #This requires https://github.com/postfinance/kubelet-csr-approver to be installed. Install using the below helm command. + #helm upgrade --install kubelet-csr-approver kubelet-csr-approver/kubelet-csr-approver -n kube-system --create-namespace --set maxExpirationSeconds=2592000 --set leaderElection=true --set bypassDnsResolution=true --set rbac.create=true + #If kubelet-csr-approver is not installed, ensure the below flag is omitted using a # + kubelet-certificate-authority: /etc/kubernetes/pki/ca.crt + extraVolumes: + #1.2.9 Ensure that the admission control plugin EventRateLimit is set + - name: admission-config + hostPath: /etc/kubernetes/admission + mountPath: /etc/kubernetes/admission + readOnly: true + pathType: DirectoryOrCreate + controllerManager: + extraArgs: + #1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate + terminated-pod-gc-threshold: "10000" + #1.3.2 Ensure that the --profiling argument is set to false + profiling: "false" + #1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true + #This requires https://github.com/postfinance/kubelet-csr-approver to be installed. Install using the below helm command. + #helm upgrade --install kubelet-csr-approver kubelet-csr-approver/kubelet-csr-approver -n kube-system --create-namespace --set maxExpirationSeconds=2592000 --set leaderElection=true --set bypassDnsResolution=true --set rbac.create=true + #If kubelet-csr-approver is not installed, ensure the below flag is omitted using a # + feature-gates: RotateKubeletServerCertificate=true + scheduler: + extraArgs: + #1.4.1 Ensure that the --profiling argument is set to false + profiling: "false" + files: + #1.2.9 Ensure that the admission control plugin EventRateLimit is set + - content: | + apiVersion: apiserver.config.k8s.io/v1 + kind: AdmissionConfiguration + plugins: + - name: EventRateLimit + path: /etc/kubernetes/admission/eventRateLimit.yaml + path: /etc/kubernetes/admission/admissionConfiguration.yaml + permissions: "0600" + #1.2.9 Ensure that the admission control plugin EventRateLimit is set + - content: | + apiVersion: eventratelimit.admission.k8s.io/v1alpha1 + kind: Configuration + limits: + - type: Server + qps: 5000 + burst: 20000 + - type: Namespace + qps: 500 + burst: 2000 + cacheSize: 1000 + - type: User + qps: 100 + burst: 400 + cacheSize: 2000 + - type: SourceAndObject + qps: 50 + burst: 100 + cacheSize: 5000 + path: /etc/kubernetes/admission/eventRateLimit.yaml + permissions: "0600" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + #1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate + #1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true + #This requires https://github.com/postfinance/kubelet-csr-approver to be installed. Install using the below helm command. + #helm upgrade --install kubelet-csr-approver kubelet-csr-approver/kubelet-csr-approver -n kube-system --create-namespace --set maxExpirationSeconds=2592000 --set leaderElection=true --set bypassDnsResolution=true --set rbac.create=true + #If kubelet-csr-approver is not installed, ensure the below flag is omitted using a # + rotate-server-certificates: "true" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + #1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate + #1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true + #This requires https://github.com/postfinance/kubelet-csr-approver to be installed. Install using the below helm command. + #helm upgrade --install kubelet-csr-approver kubelet-csr-approver/kubelet-csr-approver -n kube-system --create-namespace --set maxExpirationSeconds=2592000 --set leaderElection=true --set bypassDnsResolution=true --set rbac.create=true + #If kubelet-csr-approver is not installed, ensure the below flag is omitted using a # + rotate-server-certificates: "true" + postKubeadmCommands: + - echo export KUBECONFIG=/etc/kubernetes/admin.conf >> /root/.bashrc + - echo "after kubeadm call" > /var/log/postkubeadm.log + #4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive + - chmod 600 "$(systemctl show -P FragmentPath kubelet.service)" + - chmod 600 /var/lib/kubelet/config.yaml + - chmod 600 /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/kustomization.yaml b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/kustomization.yaml new file mode 100644 index 000000000..efc0053c5 --- /dev/null +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- nkp-nutanix-.yaml + +patches: +- path: cis-mitigations-cp-patch.yaml + +namePrefix: "" +nameSuffix: "-hardened" diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/README.md b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/README.md new file mode 100644 index 000000000..109e978a7 --- /dev/null +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/README.md @@ -0,0 +1,175 @@ +# Hardened Kubernetes Worker Node Template + +This directory contains kustomization files to apply CIS security mitigations to the Kubernetes worker node template. + +## Prerequisites + +Before applying the kustomization, you need to determine the available Nutanix-provided KubeadmConfigTemplates (those with the "nkp-" prefix) and their versions: + +```bash +# List all available KubeadmConfigTemplates +kubectl get kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io + +# Example output: +# NAME AGE +# nkp-nutanix-worker-v2.14.0 4d +# wskn-nkp-mgmt-md-0-4phrm 19h +``` + +**Important**: You must use the Nutanix-provided template (starting with "nkp-") for these hardening configurations to work correctly. + +Then export the original Nutanix KubeadmConfigTemplate using the appropriate version: + +```bash +# Replace with your actual version (e.g., v2.14.0) +kubectl get kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io nkp-nutanix-worker- -o yaml > nkp-nutanix-worker-.yaml +``` + +## Structure + +- `nkp-nutanix-worker-.yaml` - The original KubeadmConfigTemplate (generated by user, replace with your actual version) +- `cis-mitigations-worker-patch.yaml` - Patch file containing CIS hardening configurations +- `kustomization.yaml` - Kustomization file that applies the patch and renames the template + +**Note**: You need to edit BOTH of these files with your actual version: + +1. Edit the `cis-mitigations-worker-patch.yaml` file to replace `` with your actual version in the `metadata.name` field: + +```yaml +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: nkp-nutanix-worker- # <-- Replace with your actual version +``` + +2. Edit the `kustomization.yaml` file to replace `` in the resources section: + +```yaml +resources: +- nkp-nutanix-worker-.yaml # <-- Replace with your actual version +``` + +## Applying the Kustomization + +You can apply this kustomization directly with kubectl: + +```bash +kubectl apply -k /path/to/hardened-clusters/kustomize/worker +``` + +Or you can build and then apply: + +```bash +# If you have kustomize installed separately +kustomize build /path/to/hardened-clusters/kustomize/worker | kubectl apply -f - + +# Or using kubectl's built-in kustomize +kubectl kustomize /path/to/hardened-clusters/kustomize/worker | kubectl apply -f - +``` + +## Verification + +The kustomization will: + +1. Take the user-generated KubeadmConfigTemplate `nkp-nutanix-worker-` +2. Apply the CIS mitigations patch that adds: + - Kubelet security settings (read-only port, connection timeout, iptables) + - Resource limit configurations + - Enhanced TLS cipher configuration + - Additional postKubeadmCommands for file permission hardening +3. Rename it to `nkp-nutanix-worker--hardened` + +## CIS Mitigations Applied + +The following CIS mitigations are applied: + +- **4.1.1**: Set kubelet service file permissions to 600 or more restrictive +- **4.2.4**: Set read-only-port argument to 0 +- **4.2.5**: Set streaming-connection-idle-timeout to non-zero value (5m as per CIS guidelines) +- **4.2.6**: Set make-iptables-util-chains argument to true +- **4.2.8**: Set event-qps argument to appropriate level (5) +- **4.2.12**: Enforce strong TLS cryptographic ciphers with an updated suite of recommended ciphers +- **4.2.13**: Set pod-max-pids limit to 4096 + +## Applying to Cluster Class + +After generating the hardened template, you need to update the cluster class to use it: + +1. First, identify the cluster class name and version: + +```bash +kubectl get clusterclasses.cluster.x-k8s.io +``` + +2. Patch the cluster class to use the hardened worker template (replace `` and cluster class name as needed): + +```bash +kubectl patch clusterclass nkp-nutanix- \ + --type json \ + -p='[{ + "op":"replace", + "path":"/spec/workers/machineDeployments/0/template/bootstrap/ref/name", + "value":"nkp-nutanix-worker--hardened" + }]' +``` + +## Handling Upgrades + +When upgrading to a new Nutanix Kubernetes Platform version, you'll need to create hardened templates for the new version. Follow these steps: + +1. Upgrade CAPI components first: + +```bash +nkp upgrade capi-components +``` + +2. Identify the new worker template version and export it: + +```bash +# List available templates after upgrade +kubectl get kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io + +# Export the new template (replace with the actual new version, e.g., v2.15.0) +kubectl get kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io nkp-nutanix-worker- -o yaml > nkp-nutanix-worker-.yaml +``` + +3. Create a new directory for the new version and copy the kustomization files there: + +```bash +mkdir -p /path/to/hardened-clusters/kustomize/worker- +cp /path/to/hardened-clusters/kustomize/worker/cis-mitigations-worker-patch.yaml /path/to/hardened-clusters/kustomize/worker-/ +cp /path/to/hardened-clusters/kustomize/worker/kustomization.yaml /path/to/hardened-clusters/kustomize/worker-/ +``` + +4. Update the files in the new directory to use the new version: + +```bash +# Edit the version in both files +sed -i 's///g' /path/to/hardened-clusters/kustomize/worker-/cis-mitigations-worker-patch.yaml +sed -i 's///g' /path/to/hardened-clusters/kustomize/worker-/kustomization.yaml +``` + +5. Apply the kustomization for the new version: + +```bash +kubectl apply -k /path/to/hardened-clusters/kustomize/worker- +``` + +6. Patch the new cluster class to use the hardened template: + +```bash +# Replace with the actual new version +kubectl patch clusterclass nkp-nutanix- \ + --type json \ + -p='[{ + "op":"replace", + "path":"/spec/workers/machineDeployments/0/template/bootstrap/ref/name", + "value":"nkp-nutanix-worker--hardened" + }]' +``` + +7. Upgrade the cluster using the new patched cluster class: + +```bash +nkp upgrade cluster nutanix --cluster-class nkp-nutanix- +``` diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/cis-mitigations-worker-patch.yaml b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/cis-mitigations-worker-patch.yaml new file mode 100644 index 000000000..7d809764f --- /dev/null +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/cis-mitigations-worker-patch.yaml @@ -0,0 +1,28 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: nkp-nutanix-worker- +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + # 4.2.4 Ensure that the --read-only-port argument is set to 0 + read-only-port: "0" + # 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 + # Recommendation: Set to 5m instead of 4h as per CIS guidelines + streaming-connection-idle-timeout: "5m" + # 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true + make-iptables-util-chains: "true" + # 4.2.8 Ensure that the event-qps argument is set to a level which ensures appropriate event capture + event-qps: "5" + # 4.2.13 Ensure that a limit is set on pod PIDs + pod-max-pids: "4096" + # 4.2.12 Updated with recommended strong cipher suites + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 + # 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive + postKubeadmCommands: + - echo "after kubeadm call" > /var/log/postkubeadm.log + - chmod 600 "$(systemctl show -P FragmentPath kubelet.service)" + - chmod 600 /var/lib/kubelet/config.yaml diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/kustomization.yaml b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/kustomization.yaml new file mode 100644 index 000000000..e4467b3a3 --- /dev/null +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- nkp-nutanix-worker-.yaml + +patches: +- path: cis-mitigations-worker-patch.yaml + +namePrefix: "" +nameSuffix: "-hardened" From b63e5cf59f0d6ab9178101c4ac8a5a5bdaf22a81 Mon Sep 17 00:00:00 2001 From: WinsonSou Date: Wed, 21 May 2025 19:24:53 +0530 Subject: [PATCH 2/4] removed upgrade stuff as caren will take care of it --- .../worker/README.md | 63 +------------------ 1 file changed, 1 insertion(+), 62 deletions(-) diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/README.md b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/README.md index 109e978a7..b199f66af 100644 --- a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/README.md +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/README.md @@ -111,65 +111,4 @@ kubectl patch clusterclass nkp-nutanix- \ "path":"/spec/workers/machineDeployments/0/template/bootstrap/ref/name", "value":"nkp-nutanix-worker--hardened" }]' -``` - -## Handling Upgrades - -When upgrading to a new Nutanix Kubernetes Platform version, you'll need to create hardened templates for the new version. Follow these steps: - -1. Upgrade CAPI components first: - -```bash -nkp upgrade capi-components -``` - -2. Identify the new worker template version and export it: - -```bash -# List available templates after upgrade -kubectl get kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io - -# Export the new template (replace with the actual new version, e.g., v2.15.0) -kubectl get kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io nkp-nutanix-worker- -o yaml > nkp-nutanix-worker-.yaml -``` - -3. Create a new directory for the new version and copy the kustomization files there: - -```bash -mkdir -p /path/to/hardened-clusters/kustomize/worker- -cp /path/to/hardened-clusters/kustomize/worker/cis-mitigations-worker-patch.yaml /path/to/hardened-clusters/kustomize/worker-/ -cp /path/to/hardened-clusters/kustomize/worker/kustomization.yaml /path/to/hardened-clusters/kustomize/worker-/ -``` - -4. Update the files in the new directory to use the new version: - -```bash -# Edit the version in both files -sed -i 's///g' /path/to/hardened-clusters/kustomize/worker-/cis-mitigations-worker-patch.yaml -sed -i 's///g' /path/to/hardened-clusters/kustomize/worker-/kustomization.yaml -``` - -5. Apply the kustomization for the new version: - -```bash -kubectl apply -k /path/to/hardened-clusters/kustomize/worker- -``` - -6. Patch the new cluster class to use the hardened template: - -```bash -# Replace with the actual new version -kubectl patch clusterclass nkp-nutanix- \ - --type json \ - -p='[{ - "op":"replace", - "path":"/spec/workers/machineDeployments/0/template/bootstrap/ref/name", - "value":"nkp-nutanix-worker--hardened" - }]' -``` - -7. Upgrade the cluster using the new patched cluster class: - -```bash -nkp upgrade cluster nutanix --cluster-class nkp-nutanix- -``` +``` \ No newline at end of file From de6d99a3193c5e636e521ef310bd1c4a9b9909fc Mon Sep 17 00:00:00 2001 From: WinsonSou Date: Sun, 25 May 2025 07:23:44 +0800 Subject: [PATCH 3/4] simplified hardening + support for 2.14.0 and 2.15.0 --- .../control-plane/README.md | 94 +++---------- .../cis-mitigations-cp-patch.yaml | 41 +----- .../control-plane/harden.sh | 41 ++++++ .../control-plane/kustomization.yaml | 106 ++++++++++++++ .../worker/README.md | 130 +++++++----------- .../worker/cis-mitigations-worker-patch.yaml | 11 +- .../worker/harden.sh | 48 +++++++ .../worker/kustomization.yaml | 16 +++ 8 files changed, 284 insertions(+), 203 deletions(-) create mode 100755 examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/harden.sh create mode 100755 examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/harden.sh diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/README.md b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/README.md index 4d6c50283..847ed92a6 100644 --- a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/README.md +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/README.md @@ -21,80 +21,41 @@ helm upgrade --install kubelet-csr-approver \ --set rbac.create=true ``` -**Note**: If you choose not to install the kubelet-csr-approver, you must omit the flags related to the CIS benchmarks mentioned above from your configuration. +**Note**: If you choose not to install the kubelet-csr-approver, you must omit the flags related to the CIS benchmarks mentioned above from your configuration from the `cis-mitigations-cp-patch.yaml` file. -## Prerequisites +## Directory Structure -Before applying the kustomization, you need to determine the available Nutanix-provided KubeadmControlPlaneTemplates (those with the "nkp-" prefix) and their versions: +This directory contains the following files: -```bash -# List all available KubeadmControlPlaneTemplates -kubectl get kubeadmcontrolplanetemplates.controlplane.cluster.x-k8s.io - -# Example output: -# NAME AGE -# nkp-nutanix-v2.14.0 4d -# some-other-template 19h -``` - -**Important**: You must use the Nutanix-provided template (starting with "nkp-") for these hardening configurations to work correctly. - -Then export the original Nutanix KubeadmControlPlaneTemplate using the appropriate version: - -```bash -# Replace with your actual version (e.g., v2.14.0) -kubectl get kubeadmcontrolplanetemplates.controlplane.cluster.x-k8s.io nkp-nutanix- -o yaml > nkp-nutanix-.yaml -``` - -## Structure - -- `nkp-nutanix-.yaml` - The original KubeadmControlPlaneTemplate (generated by user, replace with your actual version) +- `harden.sh` - Automated script to simplify the hardening process (recommended method) - `cis-mitigations-cp-patch.yaml` - Patch file containing CIS hardening configurations - `kustomization.yaml` - Kustomization file that applies the patch and renames the template +- `nkp-nutanix-.yaml` - The original KubeadmControlPlaneTemplate (generated during the hardening process via `./harden.sh`) -**Note**: You need to edit BOTH of these files with your actual version: - -1. Edit the `cis-mitigations-cp-patch.yaml` file to replace `` with your actual version in the `metadata.name` field: - -```yaml -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlaneTemplate -metadata: - name: nkp-nutanix- # <-- Replace with your actual version -``` - -2. Edit the `kustomization.yaml` file to replace `` in the resources section: - -```yaml -resources: -- nkp-nutanix-.yaml # <-- Replace with your actual version -``` +The `harden.sh` script automates the following tasks: +1. Lists available KubeadmControlPlaneTemplates +2. Prompts for the NKP version +3. Exports the original template +4. Updates all version placeholders in configuration files +5. Applies the kustomization to create the hardened template +6. Provides guidance on patching the ClusterClass to use the hardened template -## Applying the Kustomization +## Applying the Hardening -You can apply this kustomization directly with kubectl: +Simply run the hardening script and follow the prompts. Ensure that you have the `KUBECONFIG` environment variable set to the Management Cluster (or Self-Managed) before running it: ```bash -#Ensure you are in a directory for the kustomization.yaml, cis-mitigations-cp-patch.yaml and the nkp-nutanix-.yaml file -kubectl apply -k . +#export KUBECONFIG= +./harden.sh ``` -## Verification - -The kustomization will: +This script will guide you through the process, automatically generate the required files, and apply the kustomization. -1. Take the user-generated KubeadmControlPlaneTemplate `nkp-nutanix-` -2. Apply the CIS mitigations patch that adds: - - API Server hardening (profiling disabled, service account lookup, admission plugins) - - Controller Manager settings (terminated pod GC threshold, profiling disabled) - - Scheduler settings (profiling disabled) - - Required configuration files for EventRateLimit - - Additional postKubeadmCommands for file permission hardening -3. Rename it to `nkp-nutanix--hardened` +**Note**: For a fully hardened cluster, you should also apply hardening to the worker nodes by using the scripts in the `../worker` directory. ## CIS Mitigations Applied -The following CIS mitigations are applied: +The following CIS mitigations are applied to the Control Plane Nodes: ### API Server @@ -140,20 +101,3 @@ The following CIS mitigations are applied: - kubelet config.yaml - 10-kubeadm.conf -## Applying to Cluster Class - -After generating the hardened template, you need to update the cluster class to use it: - -1. First, identify the cluster class name and version: - -```bash -kubectl get clusterclasses.cluster.x-k8s.io -``` - -2. Patch the cluster class to use the hardened control plane template (replace `` and cluster class name as needed): - -```bash -kubectl patch clusterclass nkp-nutanix- \ - --type merge \ - -p='{"spec":{"controlPlane":{"ref":{"name":"nkp-nutanix--hardened"}}}}' -``` diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/cis-mitigations-cp-patch.yaml b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/cis-mitigations-cp-patch.yaml index 4a8744859..c04d29264 100644 --- a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/cis-mitigations-cp-patch.yaml +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/cis-mitigations-cp-patch.yaml @@ -47,38 +47,7 @@ spec: extraArgs: #1.4.1 Ensure that the --profiling argument is set to false profiling: "false" - files: - #1.2.9 Ensure that the admission control plugin EventRateLimit is set - - content: | - apiVersion: apiserver.config.k8s.io/v1 - kind: AdmissionConfiguration - plugins: - - name: EventRateLimit - path: /etc/kubernetes/admission/eventRateLimit.yaml - path: /etc/kubernetes/admission/admissionConfiguration.yaml - permissions: "0600" - #1.2.9 Ensure that the admission control plugin EventRateLimit is set - - content: | - apiVersion: eventratelimit.admission.k8s.io/v1alpha1 - kind: Configuration - limits: - - type: Server - qps: 5000 - burst: 20000 - - type: Namespace - qps: 500 - burst: 2000 - cacheSize: 1000 - - type: User - qps: 100 - burst: 400 - cacheSize: 2000 - - type: SourceAndObject - qps: 50 - burst: 100 - cacheSize: 5000 - path: /etc/kubernetes/admission/eventRateLimit.yaml - permissions: "0600" + initConfiguration: nodeRegistration: kubeletExtraArgs: @@ -97,10 +66,4 @@ spec: #helm upgrade --install kubelet-csr-approver kubelet-csr-approver/kubelet-csr-approver -n kube-system --create-namespace --set maxExpirationSeconds=2592000 --set leaderElection=true --set bypassDnsResolution=true --set rbac.create=true #If kubelet-csr-approver is not installed, ensure the below flag is omitted using a # rotate-server-certificates: "true" - postKubeadmCommands: - - echo export KUBECONFIG=/etc/kubernetes/admin.conf >> /root/.bashrc - - echo "after kubeadm call" > /var/log/postkubeadm.log - #4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive - - chmod 600 "$(systemctl show -P FragmentPath kubelet.service)" - - chmod 600 /var/lib/kubelet/config.yaml - - chmod 600 /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/harden.sh b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/harden.sh new file mode 100755 index 000000000..4f116a269 --- /dev/null +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/harden.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# List all the KubeadmControlPlaneTemplates to find the latest version +echo "Listing all KubeadmControlPlaneTemplates:" +kubectl get kubeadmcontrolplanetemplates.controlplane.cluster.x-k8s.io + +# Prompt for the NKP version +echo "Please enter the NKP version from the list above (e.g., for NKP version nkp-nutanix-v2.14.0, enter v2.14.0):" +read -p "> " VERSION + +# Validate version format +if [[ ! $VERSION =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Invalid version format. Please use the format v.. (e.g., v2.14.1)" + exit 1 +fi + +echo "Using NKP version: $VERSION" + +# Clone the Latest KubeadmControlplaneTemplate +echo "Cloning the template: nkp-nutanix-${VERSION}" +kubectl get kubeadmcontrolplanetemplates.controlplane.cluster.x-k8s.io nkp-nutanix-${VERSION} -o yaml > nkp-nutanix-${VERSION}.yaml + +# Replace with the actual version in all files +echo "Replacing with ${VERSION} in all files..." +sed -i "s//${VERSION}/g" kustomization.yaml +sed -i "s//${VERSION}/g" cis-mitigations-cp-patch.yaml + +echo "Replacement complete!" +echo "Files have been updated with version: ${VERSION}" + +echo "Applying kustomization to create hardened control plane template..." +kubectl apply -k . + +echo "You can now patch the ClusterClass to use the Hardened KubeadmControlPlaneTemplates" +echo "Here are the available ClusterClass" + +kubectl get clusterclasses.cluster.x-k8s.io + +echo "Run the below command after replacing the with the ClusterClass in use from the list above" +echo "kubectl patch clusterclass \\ + --type merge -p='{\"spec\":{\"controlPlane\":{\"ref\":{\"name\":\"nkp-nutanix-${VERSION}-hardened\"}}}}'" \ No newline at end of file diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/kustomization.yaml b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/kustomization.yaml index efc0053c5..d28c314bd 100644 --- a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/kustomization.yaml +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/kustomization.yaml @@ -6,6 +6,112 @@ resources: patches: - path: cis-mitigations-cp-patch.yaml +- target: + group: controlplane.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmControlPlaneTemplate + #target <2.15.0 + name: nkp-nutanix-v2.1[0-4].* + patch: | + #so that it works in v2.14.0. Make sure to remove after to 2.14.0 + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files + value: [] + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + value: + path: /etc/kubernetes/admission/admissionConfiguration.yaml + permissions: "0600" + content: | + apiVersion: apiserver.config.k8s.io/v1 + kind: AdmissionConfiguration + plugins: + - name: EventRateLimit + path: /etc/kubernetes/admission/eventRateLimit.yaml + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + value: + path: /etc/kubernetes/admission/eventRateLimit.yaml + permissions: "0600" + content: | + apiVersion: eventratelimit.admission.k8s.io/v1alpha1 + kind: Configuration + limits: + - type: Server + qps: 5000 + burst: 20000 + - type: Namespace + qps: 500 + burst: 2000 + cacheSize: 1000 + - type: User + qps: 100 + burst: 400 + cacheSize: 2000 + - type: SourceAndObject + qps: 50 + burst: 100 + cacheSize: 5000 + - op: add + path: /spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/- + value: 'chmod 600 "$(systemctl show -P FragmentPath kubelet.service)"' + - op: add + path: /spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/- + value: 'chmod 600 /var/lib/kubelet/config.yaml' + - op: add + path: /spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/- + value: 'chmod 600 /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf' +- target: + group: controlplane.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmControlPlaneTemplate + #target only NKP => v2.15.0 + name: nkp-nutanix-v2.1[5-9].* + patch: | + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + value: + path: /etc/kubernetes/admission/admissionConfiguration.yaml + permissions: "0600" + content: | + apiVersion: apiserver.config.k8s.io/v1 + kind: AdmissionConfiguration + plugins: + - name: EventRateLimit + path: /etc/kubernetes/admission/eventRateLimit.yaml + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + value: + path: /etc/kubernetes/admission/eventRateLimit.yaml + permissions: "0600" + content: | + apiVersion: eventratelimit.admission.k8s.io/v1alpha1 + kind: Configuration + limits: + - type: Server + qps: 5000 + burst: 20000 + - type: Namespace + qps: 500 + burst: 2000 + cacheSize: 1000 + - type: User + qps: 100 + burst: 400 + cacheSize: 2000 + - type: SourceAndObject + qps: 50 + burst: 100 + cacheSize: 5000 + - op: add + path: /spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/- + value: 'chmod 600 "$(systemctl show -P FragmentPath kubelet.service)"' + - op: add + path: /spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/- + value: 'chmod 600 /var/lib/kubelet/config.yaml' + - op: add + path: /spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/- + value: 'chmod 600 /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf' namePrefix: "" nameSuffix: "-hardened" diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/README.md b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/README.md index b199f66af..ca1379f62 100644 --- a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/README.md +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/README.md @@ -1,114 +1,76 @@ # Hardened Kubernetes Worker Node Template -This directory contains kustomization files to apply CIS security mitigations to the Kubernetes worker node template. +## Required Component: kubelet-csr-approver -## Prerequisites +To meet the following CIS security benchmarks: -Before applying the kustomization, you need to determine the available Nutanix-provided KubeadmConfigTemplates (those with the "nkp-" prefix) and their versions: +- 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate +- 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true +- Enable Kubelet Server Cert Rotation -```bash -# List all available KubeadmConfigTemplates -kubectl get kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io +You must install the Postfinance kubelet-csr-approver: -# Example output: -# NAME AGE -# nkp-nutanix-worker-v2.14.0 4d -# wskn-nkp-mgmt-md-0-4phrm 19h +```bash +helm repo add kubelet-csr-approver https://postfinance.github.io/kubelet-csr-approver +helm upgrade --install kubelet-csr-approver \ + kubelet-csr-approver/kubelet-csr-approver \ + -n kube-system \ + --create-namespace \ + --set maxExpirationSeconds=2592000 \ + --set leaderElection=true \ + --set bypassDnsResolution=true \ + --set rbac.create=true ``` -**Important**: You must use the Nutanix-provided template (starting with "nkp-") for these hardening configurations to work correctly. +**Note**: If you choose not to install the kubelet-csr-approver, you must omit the flags related to the CIS benchmarks mentioned above from your configuration from the `cis-mitigations-worker-patch.yaml` file. -Then export the original Nutanix KubeadmConfigTemplate using the appropriate version: +## Directory Structure -```bash -# Replace with your actual version (e.g., v2.14.0) -kubectl get kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io nkp-nutanix-worker- -o yaml > nkp-nutanix-worker-.yaml -``` +This directory contains the following files: -## Structure - -- `nkp-nutanix-worker-.yaml` - The original KubeadmConfigTemplate (generated by user, replace with your actual version) +- `harden.sh` - Automated script to simplify the hardening process (recommended method) - `cis-mitigations-worker-patch.yaml` - Patch file containing CIS hardening configurations - `kustomization.yaml` - Kustomization file that applies the patch and renames the template +- `nkp-nutanix-worker-.yaml` - The original KubeadmConfigTemplate (generated during the hardening process via `./harden.sh`) -**Note**: You need to edit BOTH of these files with your actual version: - -1. Edit the `cis-mitigations-worker-patch.yaml` file to replace `` with your actual version in the `metadata.name` field: +The `harden.sh` script automates the following tasks: +1. Lists available KubeadmConfigTemplates for workers +2. Prompts for the NKP version +3. Exports the original template +4. Updates all version placeholders in configuration files +5. Applies the kustomization to create the hardened template +6. Provides guidance on patching the ClusterClass to use the hardened template -```yaml -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: nkp-nutanix-worker- # <-- Replace with your actual version -``` +## Applying the Hardening -2. Edit the `kustomization.yaml` file to replace `` in the resources section: - -```yaml -resources: -- nkp-nutanix-worker-.yaml # <-- Replace with your actual version -``` - -## Applying the Kustomization - -You can apply this kustomization directly with kubectl: +Simply run the hardening script and follow the prompts. Ensure that you have the `KUBECONFIG` environment variable set to the Management Cluster (or Self-Managed) before running it: ```bash -kubectl apply -k /path/to/hardened-clusters/kustomize/worker +#export KUBECONFIG= +./harden.sh ``` -Or you can build and then apply: +This script will guide you through the process, automatically generate the required files, and apply the kustomization. -```bash -# If you have kustomize installed separately -kustomize build /path/to/hardened-clusters/kustomize/worker | kubectl apply -f - - -# Or using kubectl's built-in kustomize -kubectl kustomize /path/to/hardened-clusters/kustomize/worker | kubectl apply -f - -``` - -## Verification - -The kustomization will: - -1. Take the user-generated KubeadmConfigTemplate `nkp-nutanix-worker-` -2. Apply the CIS mitigations patch that adds: - - Kubelet security settings (read-only port, connection timeout, iptables) - - Resource limit configurations - - Enhanced TLS cipher configuration - - Additional postKubeadmCommands for file permission hardening -3. Rename it to `nkp-nutanix-worker--hardened` +**Note**: For a fully hardened cluster, you should also apply hardening to the control plane nodes by using the scripts in the `../control-plane` directory. ## CIS Mitigations Applied -The following CIS mitigations are applied: +The following CIS mitigations are applied to worker nodes: + +### Kubelet Configuration - **4.1.1**: Set kubelet service file permissions to 600 or more restrictive -- **4.2.4**: Set read-only-port argument to 0 -- **4.2.5**: Set streaming-connection-idle-timeout to non-zero value (5m as per CIS guidelines) -- **4.2.6**: Set make-iptables-util-chains argument to true -- **4.2.8**: Set event-qps argument to appropriate level (5) +- **4.2.4**: Disable read-only port (`read-only-port: 0`) +- **4.2.5**: Set streaming connection idle timeout (`streaming-connection-idle-timeout: 5m`) +- **4.2.6**: Enable make-iptables-util-chains (`make-iptables-util-chains: true`) +- **4.2.8**: Set appropriate event QPS (`event-qps: 5`) - **4.2.12**: Enforce strong TLS cryptographic ciphers with an updated suite of recommended ciphers - **4.2.13**: Set pod-max-pids limit to 4096 -## Applying to Cluster Class - -After generating the hardened template, you need to update the cluster class to use it: +### File Permissions -1. First, identify the cluster class name and version: - -```bash -kubectl get clusterclasses.cluster.x-k8s.io -``` - -2. Patch the cluster class to use the hardened worker template (replace `` and cluster class name as needed): - -```bash -kubectl patch clusterclass nkp-nutanix- \ - --type json \ - -p='[{ - "op":"replace", - "path":"/spec/workers/machineDeployments/0/template/bootstrap/ref/name", - "value":"nkp-nutanix-worker--hardened" - }]' -``` \ No newline at end of file +- **4.1.1**: Set appropriate file permissions (0600) for sensitive files including: + - kubelet.service + - kubelet config.yaml + - 10-kubeadm.conf diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/cis-mitigations-worker-patch.yaml b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/cis-mitigations-worker-patch.yaml index 7d809764f..39da55ceb 100644 --- a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/cis-mitigations-worker-patch.yaml +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/cis-mitigations-worker-patch.yaml @@ -21,8 +21,9 @@ spec: pod-max-pids: "4096" # 4.2.12 Updated with recommended strong cipher suites tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 - # 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive - postKubeadmCommands: - - echo "after kubeadm call" > /var/log/postkubeadm.log - - chmod 600 "$(systemctl show -P FragmentPath kubelet.service)" - - chmod 600 /var/lib/kubelet/config.yaml + #1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate + #1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true + #This requires https://github.com/postfinance/kubelet-csr-approver to be installed. Install using the below helm command. + #helm upgrade --install kubelet-csr-approver kubelet-csr-approver/kubelet-csr-approver -n kube-system --create-namespace --set maxExpirationSeconds=2592000 --set leaderElection=true --set bypassDnsResolution=true --set rbac.create=true + rotate-server-certificates: "true" + diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/harden.sh b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/harden.sh new file mode 100755 index 000000000..f40f46ae9 --- /dev/null +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/harden.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# List all the KubeadmConfigTemplates to find the latest version +echo "Listing all KubeadmConfigTemplates for workers:" +kubectl get kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io | grep worker + +# Prompt for the NKP version +echo "Please enter the NKP version from the list above (e.g., for NKP worker version nkp-nutanix-worker-v2.14.0, enter v2.14.0):" +read -p "> " VERSION + +# Validate version format +if [[ ! $VERSION =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Invalid version format. Please use the format v.. (e.g., v2.14.1)" + exit 1 +fi + +echo "Using NKP version: $VERSION" + +# Clone the latest KubeadmConfigTemplate for workers +echo "Cloning the worker template: nkp-nutanix-worker-${VERSION}" +kubectl get kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io nkp-nutanix-worker-${VERSION} -o yaml > nkp-nutanix-worker-${VERSION}.yaml + +# Replace with the actual version in all files +echo "Replacing with ${VERSION} in all files..." +sed -i "s//${VERSION}/g" kustomization.yaml +sed -i "s//${VERSION}/g" cis-mitigations-worker-patch.yaml + +echo "Replacement complete!" +echo "Files have been updated with version: ${VERSION}" + +# Apply the kustomization to create the hardened worker template +echo "Applying kustomization to create hardened worker template..." +kubectl apply -k . + +echo "You can now patch the ClusterClass to use the Hardened KubeadmConfigTemplate for workers" +echo "Here are the available ClusterClasses:" + +kubectl get clusterclasses.cluster.x-k8s.io + +echo "Run the below command after replacing the with the ClusterClass in use from the list above" + +echo "kubectl patch clusterclass \\ + --type json \\ + -p='[{ + \"op\":\"replace\", + \"path\":\"/spec/workers/machineDeployments/0/template/bootstrap/ref/name\", + \"value\":\"nkp-nutanix-worker-${VERSION}-hardened\" + }]'" diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/kustomization.yaml b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/kustomization.yaml index e4467b3a3..70127f0d1 100644 --- a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/kustomization.yaml +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/kustomization.yaml @@ -6,6 +6,22 @@ resources: patches: - path: cis-mitigations-worker-patch.yaml +- target: + group: bootstrap.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmConfigTemplate + name: nkp-nutanix-worker- + # 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive + patch: | + - op: add + path: /spec/template/spec/postKubeadmCommands/- + value: 'chmod 600 "$(systemctl show -P FragmentPath kubelet.service)"' + - op: add + path: /spec/template/spec/postKubeadmCommands/- + value: 'chmod 600 /var/lib/kubelet/config.yaml' + - op: add + path: /spec/template/spec/postKubeadmCommands/- + value: 'chmod 600 /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf' namePrefix: "" nameSuffix: "-hardened" From 17b6400982b9d730dc59bdff541cf069e4b798b6 Mon Sep 17 00:00:00 2001 From: WinsonSou Date: Sat, 7 Jun 2025 15:21:46 +0800 Subject: [PATCH 4/4] simplified hardening for workload + management cluster using ./harden.sh --- .../cis-mitigations-cp-patch.yaml | 1 + .../control-plane/harden.sh | 69 ++++++++++++----- .../control-plane/kustomization.yaml | 8 ++ .../worker/cis-mitigations-worker-patch.yaml | 1 + .../worker/harden.sh | 76 +++++++++++++------ .../worker/kustomization.yaml | 4 + 6 files changed, 115 insertions(+), 44 deletions(-) diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/cis-mitigations-cp-patch.yaml b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/cis-mitigations-cp-patch.yaml index c04d29264..242603016 100644 --- a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/cis-mitigations-cp-patch.yaml +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/cis-mitigations-cp-patch.yaml @@ -2,6 +2,7 @@ apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate metadata: name: nkp-nutanix- + namespace: spec: template: spec: diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/harden.sh b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/harden.sh index 4f116a269..c717b6655 100755 --- a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/harden.sh +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/harden.sh @@ -1,41 +1,72 @@ #!/bin/bash -# List all the KubeadmControlPlaneTemplates to find the latest version -echo "Listing all KubeadmControlPlaneTemplates:" -kubectl get kubeadmcontrolplanetemplates.controlplane.cluster.x-k8s.io +# Color codes +CYAN='\033[0;36m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' # No Color -# Prompt for the NKP version -echo "Please enter the NKP version from the list above (e.g., for NKP version nkp-nutanix-v2.14.0, enter v2.14.0):" +echo -e "${CYAN}Listing all clusters in all namespaces...${NC}" + +kubectl get clusters -A + +echo "" +echo -e "${YELLOW}Please enter the namespace of the cluster you wish to harden (press Enter to use 'default'):${NC}" +read -p "> " NAMESPACE +NAMESPACE=${NAMESPACE:-default} + +echo "" +echo -e "${CYAN}Using namespace: $NAMESPACE${NC}" + +echo "" +echo -e "${CYAN}Listing all the KubeadmControlPlaneTemplates in namespace $NAMESPACE...${NC}" +kubectl get kubeadmcontrolplanetemplates.controlplane.cluster.x-k8s.io -n $NAMESPACE + +echo "" +echo -e "${YELLOW}Please enter the NKP version from the list above (e.g., for NKP version nkp-nutanix-v2.14.0, enter v2.14.0):${NC}" read -p "> " VERSION +echo "" # Validate version format if [[ ! $VERSION =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "Invalid version format. Please use the format v.. (e.g., v2.14.1)" + echo -e "${RED}Invalid version format. Please use the format v.. (e.g., v2.14.1)${NC}" exit 1 fi -echo "Using NKP version: $VERSION" +echo -e "${CYAN}Using NKP version: $VERSION${NC}" +echo "" # Clone the Latest KubeadmControlplaneTemplate -echo "Cloning the template: nkp-nutanix-${VERSION}" -kubectl get kubeadmcontrolplanetemplates.controlplane.cluster.x-k8s.io nkp-nutanix-${VERSION} -o yaml > nkp-nutanix-${VERSION}.yaml +echo -e "${CYAN}Cloning the template: nkp-nutanix-${VERSION} from namespace $NAMESPACE...${NC}" +kubectl get kubeadmcontrolplanetemplates.controlplane.cluster.x-k8s.io nkp-nutanix-${VERSION} -n $NAMESPACE -o yaml > nkp-nutanix-${VERSION}.yaml +echo "" # Replace with the actual version in all files -echo "Replacing with ${VERSION} in all files..." +echo -e "${CYAN}Replacing with ${VERSION} in all files...${NC}" sed -i "s//${VERSION}/g" kustomization.yaml sed -i "s//${VERSION}/g" cis-mitigations-cp-patch.yaml -echo "Replacement complete!" -echo "Files have been updated with version: ${VERSION}" +# Replace with the actual namespace in all files +echo -e "${CYAN}Replacing with ${NAMESPACE} in all files...${NC}" +sed -i "s//${NAMESPACE}/g" kustomization.yaml +sed -i "s//${NAMESPACE}/g" cis-mitigations-cp-patch.yaml + +echo "" +echo -e "${GREEN}Replacement complete!${NC}" +echo -e "${GREEN}Files have been updated with version: ${VERSION} and namespace: ${NAMESPACE}${NC}" -echo "Applying kustomization to create hardened control plane template..." -kubectl apply -k . +echo "" +echo -e "${CYAN}Applying kustomization to create hardened control plane template...${NC}" +kubectl apply -n $NAMESPACE -k . -echo "You can now patch the ClusterClass to use the Hardened KubeadmControlPlaneTemplates" -echo "Here are the available ClusterClass" +echo "" +echo -e "${CYAN}You can now patch the ClusterClass to use the Hardened KubeadmControlPlaneTemplates${NC}" +echo -e "${CYAN}Here are the available ClusterClass in namespace $NAMESPACE${NC}" -kubectl get clusterclasses.cluster.x-k8s.io +kubectl get clusterclasses.cluster.x-k8s.io -n $NAMESPACE -echo "Run the below command after replacing the with the ClusterClass in use from the list above" -echo "kubectl patch clusterclass \\ +echo "" +echo -e "${YELLOW}Run the below command after replacing the with the ClusterClass in use from the list above${NC}" +echo -e "kubectl patch clusterclass -n $NAMESPACE \\ --type merge -p='{\"spec\":{\"controlPlane\":{\"ref\":{\"name\":\"nkp-nutanix-${VERSION}-hardened\"}}}}'" \ No newline at end of file diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/kustomization.yaml b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/kustomization.yaml index d28c314bd..c13275a9c 100644 --- a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/kustomization.yaml +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/control-plane/kustomization.yaml @@ -12,6 +12,7 @@ patches: kind: KubeadmControlPlaneTemplate #target <2.15.0 name: nkp-nutanix-v2.1[0-4].* + namespace: patch: | #so that it works in v2.14.0. Make sure to remove after to 2.14.0 - op: add @@ -58,6 +59,9 @@ patches: - op: add path: /spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/- value: 'chmod 600 /var/lib/kubelet/config.yaml' + - op: add + path: /spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/- + value: 'echo "serverTLSBootstrap: true" >> /var/lib/kubelet/config.yaml' - op: add path: /spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/- value: 'chmod 600 /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf' @@ -68,6 +72,7 @@ patches: kind: KubeadmControlPlaneTemplate #target only NKP => v2.15.0 name: nkp-nutanix-v2.1[5-9].* + namespace: patch: | - op: add path: /spec/template/spec/kubeadmConfigSpec/files/- @@ -110,6 +115,9 @@ patches: - op: add path: /spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/- value: 'chmod 600 /var/lib/kubelet/config.yaml' + - op: add + path: /spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/- + value: 'echo "serverTLSBootstrap: true" >> /var/lib/kubelet/config.yaml' - op: add path: /spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/- value: 'chmod 600 /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf' diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/cis-mitigations-worker-patch.yaml b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/cis-mitigations-worker-patch.yaml index 39da55ceb..7ee0ddd08 100644 --- a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/cis-mitigations-worker-patch.yaml +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/cis-mitigations-worker-patch.yaml @@ -2,6 +2,7 @@ apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: name: nkp-nutanix-worker- + namespace: spec: template: spec: diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/harden.sh b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/harden.sh index f40f46ae9..83d774b96 100755 --- a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/harden.sh +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/harden.sh @@ -1,48 +1,74 @@ #!/bin/bash -# List all the KubeadmConfigTemplates to find the latest version -echo "Listing all KubeadmConfigTemplates for workers:" -kubectl get kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io | grep worker +# Color codes +CYAN='\033[0;36m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' # No Color -# Prompt for the NKP version -echo "Please enter the NKP version from the list above (e.g., for NKP worker version nkp-nutanix-worker-v2.14.0, enter v2.14.0):" + +echo -e "${CYAN}Listing all clusters in all namespaces...${NC}" + +kubectl get clusters -A + +echo "" +echo -e "${YELLOW}Please enter the namespace of the cluster you wish to harden (press Enter to use 'default'):${NC}" +read -p "> " NAMESPACE +NAMESPACE=${NAMESPACE:-default} + +echo "" +echo -e "${CYAN}Using namespace: $NAMESPACE${NC}" + +echo "" +echo -e "${CYAN}Listing all the KubeadmConfigTemplates for workers in namespace $NAMESPACE...${NC}" +kubectl get kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io -n $NAMESPACE | grep worker + +echo "" +echo -e "${YELLOW}Please enter the NKP version from the list above (e.g., for NKP worker version nkp-nutanix-worker-v2.14.0, enter v2.14.0):${NC}" read -p "> " VERSION +echo "" # Validate version format if [[ ! $VERSION =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "Invalid version format. Please use the format v.. (e.g., v2.14.1)" + echo -e "${RED}Invalid version format. Please use the format v.. (e.g., v2.14.1)${NC}" exit 1 fi -echo "Using NKP version: $VERSION" +echo -e "${CYAN}Using NKP version: $VERSION${NC}" +echo "" # Clone the latest KubeadmConfigTemplate for workers -echo "Cloning the worker template: nkp-nutanix-worker-${VERSION}" -kubectl get kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io nkp-nutanix-worker-${VERSION} -o yaml > nkp-nutanix-worker-${VERSION}.yaml +echo -e "${CYAN}Cloning the worker template: nkp-nutanix-worker-${VERSION} from namespace $NAMESPACE...${NC}" +kubectl get kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io nkp-nutanix-worker-${VERSION} -n $NAMESPACE -o yaml > nkp-nutanix-worker-${VERSION}.yaml +echo "" # Replace with the actual version in all files -echo "Replacing with ${VERSION} in all files..." +echo -e "${CYAN}Replacing with ${VERSION} in all files...${NC}" sed -i "s//${VERSION}/g" kustomization.yaml sed -i "s//${VERSION}/g" cis-mitigations-worker-patch.yaml -echo "Replacement complete!" -echo "Files have been updated with version: ${VERSION}" +# Replace with the actual namespace in all files +echo -e "${CYAN}Replacing with ${NAMESPACE} in all files...${NC}" +sed -i "s//${NAMESPACE}/g" kustomization.yaml +sed -i "s//${NAMESPACE}/g" cis-mitigations-worker-patch.yaml -# Apply the kustomization to create the hardened worker template -echo "Applying kustomization to create hardened worker template..." -kubectl apply -k . +echo "" +echo -e "${GREEN}Replacement complete!${NC}" +echo -e "${GREEN}Files have been updated with version: ${VERSION} and namespace: ${NAMESPACE}${NC}" -echo "You can now patch the ClusterClass to use the Hardened KubeadmConfigTemplate for workers" -echo "Here are the available ClusterClasses:" +echo "" +echo -e "${CYAN}Applying kustomization to create hardened worker template...${NC}" +kubectl apply -n $NAMESPACE -k . -kubectl get clusterclasses.cluster.x-k8s.io +echo "" +echo -e "${CYAN}You can now patch the ClusterClass to use the Hardened KubeadmConfigTemplate for workers${NC}" +echo -e "${CYAN}Here are the available ClusterClasses in namespace $NAMESPACE:${NC}" -echo "Run the below command after replacing the with the ClusterClass in use from the list above" +kubectl get clusterclasses.cluster.x-k8s.io -n $NAMESPACE -echo "kubectl patch clusterclass \\ +echo "" +echo -e "${YELLOW}Run the below command after replacing the with the ClusterClass in use from the list above${NC}" +echo -e "kubectl patch clusterclass -n $NAMESPACE \\ --type json \\ - -p='[{ - \"op\":\"replace\", - \"path\":\"/spec/workers/machineDeployments/0/template/bootstrap/ref/name\", - \"value\":\"nkp-nutanix-worker-${VERSION}-hardened\" - }]'" + -p='[{\n \"op\":\"replace\",\n \"path\":\"/spec/workers/machineDeployments/0/template/bootstrap/ref/name\",\n \"value\":\"nkp-nutanix-worker-${VERSION}-hardened\"\n }]'" diff --git a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/kustomization.yaml b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/kustomization.yaml index 70127f0d1..98e9acf52 100644 --- a/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/kustomization.yaml +++ b/examples/capi-quick-start/nutanix-cluster-hardened-clusterclass/worker/kustomization.yaml @@ -11,6 +11,7 @@ patches: version: v1beta1 kind: KubeadmConfigTemplate name: nkp-nutanix-worker- + namespace: # 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive patch: | - op: add @@ -19,6 +20,9 @@ patches: - op: add path: /spec/template/spec/postKubeadmCommands/- value: 'chmod 600 /var/lib/kubelet/config.yaml' + - op: add + path: /spec/template/spec/postKubeadmCommands/- + value: 'echo "serverTLSBootstrap: true" >> /var/lib/kubelet/config.yaml' - op: add path: /spec/template/spec/postKubeadmCommands/- value: 'chmod 600 /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf'