cherry-pick: refactor: manually generate self-signed kubelet serving cert when opting out of certificate rotation (#5526)

Co-authored-by: Cameron Meissner <cameissner@microsoft.com>

Author: cameronmeissner

e2e/scenario_test.go

@@ -1039,15 +1039,17 @@ func Test_Ubuntu2204DisableKubeletServingCertificateRotationWithTags(t *testing.
 				vmss.Tags["aks-disable-kubelet-serving-certificate-rotation"] = to.Ptr("true")
 			},
 			LiveVMValidators: []*LiveVMValidator{
+				FileHasContentsValidator("/etc/default/kubelet", "\\-\\-tls-cert-file=/etc/kubernetes/certs/kubeletserver.crt"),
+				FileHasContentsValidator("/etc/default/kubelet", "\\-\\-tls-private-key-file=/etc/kubernetes/certs/kubeletserver.key"),
 				FileExcludesContentsValidator("/etc/default/kubelet", "\\-\\-rotate-server-certificates=true", "\\-\\-rotate-server-certificates=true"),
 				FileExcludesContentsValidator("/etc/default/kubelet", "kubernetes.azure.com/kubelet-serving-ca=cluster", "kubernetes.azure.com/kubelet-serving-ca=cluster"),
-				FileHasContentsValidator("/etc/default/kubelet", "\\-\\-rotate-server-certificates=false"),
+				DirectoryValidator("/etc/kubernetes/certs", []string{"kubeletserver.crt", "kubeletserver.key"}),
 			},
 		},
 	})
 }
 
-func Test_Ubuntu2204DisableKubeletServingCertificateRotationWithTags_CustomKubeletConfig(t *testing.T) {
+func Test_Ubuntu2204_DisableKubeletServingCertificateRotationWithTags_CustomKubeletConfig(t *testing.T) {
 	RunScenario(t, &Scenario{
 		Tags: Tags{
 			ServerTLSBootstrapping: true,
@@ -1080,10 +1082,12 @@ func Test_Ubuntu2204DisableKubeletServingCertificateRotationWithTags_CustomKubel
 				vmss.Tags["aks-disable-kubelet-serving-certificate-rotation"] = to.Ptr("true")
 			},
 			LiveVMValidators: []*LiveVMValidator{
+				FileHasContentsValidator("/etc/default/kubeletconfig.json", "\"tlsCertFile\": \"/etc/kubernetes/certs/kubeletserver.crt\""),
+				FileHasContentsValidator("/etc/default/kubeletconfig.json", "\"tlsPrivateKeyFile\": \"/etc/kubernetes/certs/kubeletserver.key\""),
 				FileExcludesContentsValidator("/etc/default/kubelet", "\\-\\-rotate-server-certificates=true", "\\-\\-rotate-server-certificates=true"),
 				FileExcludesContentsValidator("/etc/default/kubelet", "kubernetes.azure.com/kubelet-serving-ca=cluster", "kubernetes.azure.com/kubelet-serving-ca=cluster"),
 				FileExcludesContentsValidator("/etc/default/kubeletconfig.json", "\"serverTLSBootstrap\": true", "serverTLSBootstrap: true"),
-				FileHasContentsValidator("/etc/default/kubeletconfig.json", "\"serverTLSBootstrap\": false"),
+				DirectoryValidator("/etc/kubernetes/certs", []string{"kubeletserver.crt", "kubeletserver.key"}),
 			},
 		},
 	})
@@ -1109,9 +1113,11 @@ func Test_Ubuntu2204DisableKubeletServingCertificateRotationWithTags_AlreadyDisa
 				vmss.Tags["aks-disable-kubelet-serving-certificate-rotation"] = to.Ptr("true")
 			},
 			LiveVMValidators: []*LiveVMValidator{
+				FileHasContentsValidator("/etc/default/kubelet", "\\-\\-tls-cert-file=/etc/kubernetes/certs/kubeletserver.crt"),
+				FileHasContentsValidator("/etc/default/kubelet", "\\-\\-tls-private-key-file=/etc/kubernetes/certs/kubeletserver.key"),
 				FileExcludesContentsValidator("/etc/default/kubelet", "\\-\\-rotate-server-certificates=true", "\\-\\-rotate-server-certificates=true"),
 				FileExcludesContentsValidator("/etc/default/kubelet", "kubernetes.azure.com/kubelet-serving-ca=cluster", "kubernetes.azure.com/kubelet-serving-ca=cluster"),
-				FileExcludesContentsValidator("/etc/default/kubeletconfig.json", "\"serverTLSBootstrap\": true", "serverTLSBootstrap: true"),
+				DirectoryValidator("/etc/kubernetes/certs", []string{"kubeletserver.crt", "kubeletserver.key"}),
 			},
 		},
 	})
@@ -1145,9 +1151,12 @@ func Test_Ubuntu2204DisableKubeletServingCertificateRotationWithTags_AlreadyDisa
 				vmss.Tags["aks-disable-kubelet-serving-certificate-rotation"] = to.Ptr("true")
 			},
 			LiveVMValidators: []*LiveVMValidator{
+				FileHasContentsValidator("/etc/default/kubeletconfig.json", "\"tlsCertFile\": \"/etc/kubernetes/certs/kubeletserver.crt\""),
+				FileHasContentsValidator("/etc/default/kubeletconfig.json", "\"tlsPrivateKeyFile\": \"/etc/kubernetes/certs/kubeletserver.key\""),
 				FileExcludesContentsValidator("/etc/default/kubelet", "\\-\\-rotate-server-certificates=true", "\\-\\-rotate-server-certificates=true"),
 				FileExcludesContentsValidator("/etc/default/kubelet", "kubernetes.azure.com/kubelet-serving-ca=cluster", "kubernetes.azure.com/kubelet-serving-ca=cluster"),
 				FileExcludesContentsValidator("/etc/default/kubeletconfig.json", "\"serverTLSBootstrap\": true", "serverTLSBootstrap: true"),
+				DirectoryValidator("/etc/kubernetes/certs", []string{"kubeletserver.crt", "kubeletserver.key"}),
 			},
 		},
 	})

parts/linux/cloud-init/artifacts/cse_config.sh

@@ -164,15 +164,6 @@ EOF
   systemctl restart containerd
 }
 
-# this simply generates a self-signed certificate used for serving by the kubelet
-configureKubeletServerCert() {
-    KUBELET_SERVER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/kubeletserver.key"
-    KUBELET_SERVER_CERT_PATH="/etc/kubernetes/certs/kubeletserver.crt"
-
-    openssl genrsa -out $KUBELET_SERVER_PRIVATE_KEY_PATH 2048
-    openssl req -new -x509 -days 7300 -key $KUBELET_SERVER_PRIVATE_KEY_PATH -out $KUBELET_SERVER_CERT_PATH -subj "/CN=${NODE_NAME}" -addext "subjectAltName=DNS:${NODE_NAME}"
-}
-
 configureK8s() {
     mkdir -p "/etc/kubernetes/certs"
 
@@ -249,15 +240,6 @@ EOF
         sed -i "/cloudProviderBackoffJitter/d" /etc/kubernetes/azure.json
     fi
 
-    # generate a kubelet serving certificate if we aren't relying on TLS bootstrapping to generate one for us.
-    # NOTE: in the case where ENABLE_KUBELET_SERVING_CERTIFICATE_ROTATION is true but 
-    # the customer has disabled serving certificate rotation via nodepool tags,
-    # the self-signed serving certificate will be bootstrapped by the kubelet instead of this function
-    # TODO(cameissner): remove configureKubeletServerCert altogether
-    if [ "${ENABLE_KUBELET_SERVING_CERTIFICATE_ROTATION}" != "true" ]; then
-        configureKubeletServerCert
-    fi
-
     if [ "${IS_CUSTOM_CLOUD}" == "true" ]; then
         set +x
         AKS_CUSTOM_CLOUD_JSON_PATH="/etc/kubernetes/${TARGET_ENVIRONMENT}.json"
@@ -423,12 +405,25 @@ getPrimaryNicIP() {
     echo "$ip"
 }
 
-configureKubeletServingCertificateRotation() {
+generateSelfSignedKubeletServingCertificate() {
+    KUBELET_SERVER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/kubeletserver.key"
+    KUBELET_SERVER_CERT_PATH="/etc/kubernetes/certs/kubeletserver.crt"
+
+    openssl genrsa -out $KUBELET_SERVER_PRIVATE_KEY_PATH 2048
+    openssl req -new -x509 -days 7300 -key $KUBELET_SERVER_PRIVATE_KEY_PATH -out $KUBELET_SERVER_CERT_PATH -subj "/CN=${NODE_NAME}" -addext "subjectAltName=DNS:${NODE_NAME}"
+}
+
+configureKubeletServing() {
     if [ "${ENABLE_KUBELET_SERVING_CERTIFICATE_ROTATION}" != "true" ]; then
-        echo "kubelet serving certificate rotation is disabled, nothing to configure"
+        echo "kubelet serving certificate rotation is disabled, generating self-signed serving certificate with openssl"
+        generateSelfSignedKubeletServingCertificate
         return 0
     fi
 
+    KUBELET_SERVING_CERTIFICATE_ROTATION_LABEL="kubernetes.azure.com/kubelet-serving-ca=cluster"
+    KUBELET_SERVER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/kubeletserver.key"
+    KUBELET_SERVER_CERT_PATH="/etc/kubernetes/certs/kubeletserver.crt"
+
     # check if kubelet serving certificate rotation is disabled by customer-specified nodepool tags
     export -f should_disable_kubelet_serving_certificate_rotation
     DISABLE_KUBELET_SERVING_CERTIFICATE_ROTATION=$(retrycmd_if_failure_no_stats 10 1 10 bash -cx should_disable_kubelet_serving_certificate_rotation)
@@ -437,27 +432,43 @@ configureKubeletServingCertificateRotation() {
         exit $ERR_LOOKUP_DISABLE_KUBELET_SERVING_CERTIFICATE_ROTATION_TAG
     fi
 
-    KUBELET_SERVING_CERTIFICATE_ROTATION_LABEL="kubernetes.azure.com/kubelet-serving-ca=cluster"
-
-    if [ "${DISABLE_KUBELET_SERVING_CERTIFICATE_ROTATION,,}" == "true" ]; then
-        echo "kubelet serving certificate rotation is disabled by nodepool tags, reconfiguring kubelet flags and node labels"
+    if [ "${DISABLE_KUBELET_SERVING_CERTIFICATE_ROTATION}" == "true" ]; then
+        echo "kubelet serving certificate rotation is disabled by nodepool tags"
 
-        # set the --rotate-server-certificates flag to false if needed
+        # set --rotate-server-certificates flag and serverTLSBootstrap config file field to false
+        echo "reconfiguring kubelet flags and config as needed"
         KUBELET_FLAGS="${KUBELET_FLAGS/--rotate-server-certificates=true/--rotate-server-certificates=false}"
-
-        if [ "${KUBELET_CONFIG_FILE_ENABLED,,}" == "true" ]; then
+        if [ "${KUBELET_CONFIG_FILE_ENABLED}" == "true" ]; then
             set +x
-            # set the serverTLSBootstrap property to false if needed
             KUBELET_CONFIG_FILE_CONTENT=$(echo "$KUBELET_CONFIG_FILE_CONTENT" | base64 -d | jq 'if .serverTLSBootstrap == true then .serverTLSBootstrap = false else . end' | base64)
             set -x
         fi
 
+        # manually generate kubelet's self-signed serving certificate
+        echo "generating self-signed serving certificate with openssl"
+        generateSelfSignedKubeletServingCertificate
+
+        # make sure to eliminate the kubelet serving node label
+        echo "removing node label $KUBELET_SERVING_CERTIFICATE_ROTATION_LABEL"
         removeKubeletNodeLabel $KUBELET_SERVING_CERTIFICATE_ROTATION_LABEL
-        return 0
+    else
+        echo "kubelet serving certificate rotation is enabled"
+
+        # remove the --tls-cert-file and --tls-private-key-file flags, which are incompatible with serving certificate rotation
+        # NOTE: this step will not be needed once these flags are no longer defaulted by the bootstrapper
+        echo "removing --tls-cert-file and --tls-private-key-file from kubelet flags"
+        removeKubeletFlag "--tls-cert-file=$KUBELET_SERVER_CERT_PATH"
+        removeKubeletFlag "--tls-private-key-file=$KUBELET_SERVER_PRIVATE_KEY_PATH"
+        if [ "${KUBELET_CONFIG_FILE_ENABLED}" == "true" ]; then
+            set +x
+            KUBELET_CONFIG_FILE_CONTENT=$(echo "$KUBELET_CONFIG_FILE_CONTENT" | base64 -d | jq 'del(.tlsCertFile)' | jq 'del(.tlsPrivateKeyFile)' | base64)
+            set -x
+        fi
+
+        # make sure to add the kubelet serving node label
+        echo "adding node label $KUBELET_SERVING_CERTIFICATE_ROTATION_LABEL if needed"
+        addKubeletNodeLabel $KUBELET_SERVING_CERTIFICATE_ROTATION_LABEL
     fi
-    
-    echo "kubelet serving certificate rotation is enabled, will add node label if needed"
-    addKubeletNodeLabel $KUBELET_SERVING_CERTIFICATE_ROTATION_LABEL
 }
 
 ensureKubelet() {

parts/linux/cloud-init/artifacts/cse_helpers.sh

@@ -470,7 +470,7 @@ should_disable_kubelet_serving_certificate_rotation() {
       return $ret
     fi
     should_disable=$(echo "$body" | jq -r '.compute.tagsList[] | select(.name == "aks-disable-kubelet-serving-certificate-rotation") | .value')
-    echo "$should_disable"
+    echo "${should_disable,,}"
 }
 
 isMarinerOrAzureLinux() {
@@ -644,4 +644,16 @@ removeKubeletNodeLabel() {
     fi
 }
 
+# removes the specified FLAG_STRING (which should be in the form of 'key=value') from KUBELET_FLAGS
+removeKubeletFlag() {
+    local FLAG_STRING=$1
+    if grep -e ",${FLAG_STRING}" <<< "$KUBELET_FLAGS" > /dev/null 2>&1; then
+        KUBELET_FLAGS="${KUBELET_FLAGS/,${FLAG_STRING}/}"
+    elif grep -e "${FLAG_STRING}," <<< "$KUBELET_FLAGS" > /dev/null 2>&1; then
+        KUBELET_FLAGS="${KUBELET_FLAGS/${FLAG_STRING},/}"
+    elif grep -e "${FLAG_STRING}" <<< "$KUBELET_FLAGS" > /dev/null 2>&1; then
+        KUBELET_FLAGS="${KUBELET_FLAGS/${FLAG_STRING}/}"
+    fi
+}
+
 #HELPERSEOF

parts/linux/cloud-init/artifacts/cse_main.sh

@@ -222,7 +222,7 @@ mkdir -p "/etc/systemd/system/kubelet.service.d"
 
 # IMPORTANT NOTE: We do this here since this function can mutate kubelet flags and node labels, 
 # which is used by configureK8s and other functions. Thus, we need to make sure flag and label content is correct beforehand.
-logs_to_events "AKS.CSE.configureKubeletServingCertificateRotation" configureKubeletServingCertificateRotation
+logs_to_events "AKS.CSE.configureKubeletServing" configureKubeletServing
 
 logs_to_events "AKS.CSE.configureK8s" configureK8s
 

pkg/agent/baker.go

@@ -300,10 +300,6 @@ func ValidateAndSetLinuxNodeBootstrappingConfiguration(config *datamodel.NodeBoo
 	if IsKubeletServingCertificateRotationEnabled(config) {
 		// ensure the required feature gate is set
 		kubeletFlags["--feature-gates"] = addFeatureGateString(kubeletFlags["--feature-gates"], "RotateKubeletServerCertificate", true)
-		// backfill deletion of --tls-cert-file and --tls-private-key-file, which are incompatible with --rotate-server-certificates
-		// these are set as defaults on the RP-side for Linux
-		delete(kubeletFlags, "--tls-cert-file")
-		delete(kubeletFlags, "--tls-private-key-file")
 	}
 
 	if IsKubernetesVersionGe(config.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion, "1.24.0") {
@@ -347,9 +343,6 @@ func validateAndSetWindowsNodeBootstrappingConfiguration(config *datamodel.NodeB
 
 		if IsKubeletServingCertificateRotationEnabled(config) {
 			kubeletFlags["--feature-gates"] = addFeatureGateString(kubeletFlags["--feature-gates"], "RotateKubeletServerCertificate", true)
-			// RP doesn't currently set these flags for windows, though we filter them out anyways just to be safe
-			delete(kubeletFlags, "--tls-cert-file")
-			delete(kubeletFlags, "--tls-private-key-file")
 		}
 	}
 }

pkg/agent/baker_test.go

@@ -876,8 +876,6 @@ var _ = Describe("Assert generated customData and cseCmd", func() {
 			}, func(o *nodeBootstrappingOutput) {
 				Expect(o.vars["ENABLE_KUBELET_SERVING_CERTIFICATE_ROTATION"]).To(Equal("true"))
 				Expect(strings.Contains(o.vars["KUBELET_FLAGS"], "--rotate-server-certificates=true")).To(BeTrue())
-				Expect(strings.Contains(o.vars["KUBELET_FLAGS"], "--tls-cert-file")).To(BeFalse())
-				Expect(strings.Contains(o.vars["KUBELET_FLAGS"], "--tls-private-key-file")).To(BeFalse())
 			}),
 
 		Entry("AKSUbuntu2204 with kubelet serving certificate rotation disabled and custom kubelet config",