remove kubeadm.alpha.kubernetes.io/cri-socket annotation when kubeadm upgrade

Author: HirazawaUi

cmd/kubeadm/app/cmd/phases/upgrade/apply/uploadconfig.go

@@ -113,6 +113,10 @@ func runUploadKubeletConfig(c workflow.RunData) error {
 		if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil {
 			return errors.Wrap(err, "error writing CRISocket for this node")
 		}
+	} else {
+		if err := patchnodephase.RemoveCRISocketAnnotation(client, cfg.NodeRegistration.Name); err != nil {
+			return err
+		}
 	}
 
 	return nil

cmd/kubeadm/app/cmd/phases/upgrade/kubeletconfig.go

@@ -25,6 +25,8 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
+	patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
 )
 
@@ -69,6 +71,12 @@ func runKubeletConfigPhase(c workflow.RunData) error {
 		return err
 	}
 
+	if features.Enabled(data.InitCfg().ClusterConfiguration.FeatureGates, features.NodeLocalCRISocket) {
+		if err := patchnodephase.RemoveCRISocketAnnotation(data.Client(), data.InitCfg().NodeRegistration.Name); err != nil {
+			return err
+		}
+	}
+
 	fmt.Println("[upgrade/kubelet-config] The kubelet configuration for this node was successfully upgraded!")
 	return nil
 }

cmd/kubeadm/app/phases/patchnode/patchnode.go

@@ -17,6 +17,8 @@ limitations under the License.
 package patchnode
 
 import (
+	"github.com/pkg/errors"
+
 	"k8s.io/api/core/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
@@ -27,8 +29,7 @@ import (
 
 // AnnotateCRISocket annotates the node with the given crisocket
 func AnnotateCRISocket(client clientset.Interface, nodeName string, criSocket string) error {
-
-	klog.V(1).Infof("[patchnode] Uploading the CRI Socket information %q to the Node API object %q as an annotation\n", criSocket, nodeName)
+	klog.V(1).Infof("[patchnode] Uploading the CRI socket %q to Node %q as an annotation", criSocket, nodeName)
 
 	return apiclient.PatchNode(client, nodeName, func(n *v1.Node) {
 		annotateNodeWithCRISocket(n, criSocket)
@@ -41,3 +42,20 @@ func annotateNodeWithCRISocket(n *v1.Node, criSocket string) {
 	}
 	n.ObjectMeta.Annotations[constants.AnnotationKubeadmCRISocket] = criSocket
 }
+
+// RemoveCRISocketAnnotation removes the crisocket annotation from a node.
+func RemoveCRISocketAnnotation(client clientset.Interface, nodeName string) error {
+	klog.V(1).Infof("[patchnode] Removing the CRI socket annotation from Node %q", nodeName)
+
+	if err := apiclient.PatchNode(client, nodeName, removeNodeCRISocketAnnotation); err != nil {
+		return errors.Wrapf(err, "could not remove the CRI socket annotation from Node %q", nodeName)
+	}
+	return nil
+}
+
+func removeNodeCRISocketAnnotation(n *v1.Node) {
+	if n.ObjectMeta.Annotations == nil {
+		return
+	}
+	delete(n.ObjectMeta.Annotations, constants.AnnotationKubeadmCRISocket)
+}

cmd/kubeadm/app/util/apiclient/dryrun.go

@@ -568,7 +568,7 @@ func getNode(name string) *corev1.Node {
 				"kubernetes.io/hostname": name,
 			},
 			Annotations: map[string]string{
-				"kubeadm.alpha.kubernetes.io/cri-socket": "dry-run-cri-socket",
+				constants.AnnotationKubeadmCRISocket: "dry-run-cri-socket",
 			},
 		},
 	}

test/e2e_kubeadm/nodes_test.go

@@ -52,11 +52,22 @@ var _ = Describe("nodes", func() {
 			List(ctx, metav1.ListOptions{})
 		framework.ExpectNoError(err, "error reading nodes")
 
+		var nodeLocalCRISocketEnabled bool
+		cc := getClusterConfiguration(f.ClientSet)
+		if _, ok := cc["featureGates"]; ok {
+			fgCC := cc["featureGates"].(map[interface{}]interface{})
+			if fg, ok := fgCC["NodeLocalCRISocket"]; ok {
+				nodeLocalCRISocketEnabled = fg.(bool)
+			}
+		}
+
 		// Checks that the nodes have the CRI socket annotation
 		// and that it is prefixed with a URL scheme
 		for _, node := range nodes.Items {
-			gomega.Expect(node.Annotations).To(gomega.HaveKey(nodesCRISocketAnnotation))
-			gomega.Expect(node.Annotations[nodesCRISocketAnnotation]).To(gomega.HavePrefix("unix://"))
+			if !nodeLocalCRISocketEnabled {
+				gomega.Expect(node.Annotations).To(gomega.HaveKey(nodesCRISocketAnnotation))
+				gomega.Expect(node.Annotations[nodesCRISocketAnnotation]).To(gomega.HavePrefix("unix://"))
+			}
 		}
 	})