Merge pull request kubernetes#125202 from hjet/inplace-vertical-scaling-e2e-rollback

[FG:InPlacePodVerticalScaling] Roll back resize patches in doPodResizeTests

Author: k8s-ci-robot

test/e2e/node/pod_resize.go

@@ -18,6 +18,7 @@ package node
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"regexp"
 	"runtime"
@@ -81,6 +82,28 @@ type TestContainerInfo struct {
 	RestartCount int32
 }
 
+type containerPatch struct {
+	Name      string `json:"name"`
+	Resources struct {
+		Requests struct {
+			CPU     string `json:"cpu,omitempty"`
+			Memory  string `json:"memory,omitempty"`
+			EphStor string `json:"ephemeral-storage,omitempty"`
+		} `json:"requests"`
+		Limits struct {
+			CPU     string `json:"cpu,omitempty"`
+			Memory  string `json:"memory,omitempty"`
+			EphStor string `json:"ephemeral-storage,omitempty"`
+		} `json:"limits"`
+	} `json:"resources"`
+}
+
+type patchSpec struct {
+	Spec struct {
+		Containers []containerPatch `json:"containers"`
+	} `json:"spec"`
+}
+
 func isInPlaceResizeSupportedByRuntime(c clientset.Interface, nodeName string) bool {
 	//TODO(vinaykul,InPlacePodVerticalScaling): Can we optimize this?
 	node, err := c.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
@@ -418,11 +441,18 @@ func verifyPodContainersCgroupValues(pod *v1.Pod, tcInfo []TestContainerInfo, fl
 	return true
 }
 
-func waitForPodResizeActuation(c clientset.Interface, podClient *e2epod.PodClient, pod, patchedPod *v1.Pod, expectedContainers []TestContainerInfo) *v1.Pod {
+func waitForPodResizeActuation(c clientset.Interface, podClient *e2epod.PodClient, pod, patchedPod *v1.Pod, expectedContainers []TestContainerInfo, initialContainers []TestContainerInfo, isRollback bool) *v1.Pod {
 
 	waitForContainerRestart := func() error {
 		var restartContainersExpected []string
-		for _, ci := range expectedContainers {
+
+		restartContainers := expectedContainers
+		// if we're rolling back, extract restart counts from test case "expected" containers
+		if isRollback {
+			restartContainers = initialContainers
+		}
+
+		for _, ci := range restartContainers {
 			if ci.RestartCount > 0 {
 				restartContainersExpected = append(restartContainersExpected, ci.Name)
 			}
@@ -438,7 +468,12 @@ func waitForPodResizeActuation(c clientset.Interface, podClient *e2epod.PodClien
 			restartedContainersCount := 0
 			for _, cName := range restartContainersExpected {
 				cs, _ := podutil.GetContainerStatus(pod.Status.ContainerStatuses, cName)
-				if cs.RestartCount < 1 {
+				expectedRestarts := int32(1)
+				// if we're rolling back, we should have 2 container restarts
+				if isRollback {
+					expectedRestarts = int32(2)
+				}
+				if cs.RestartCount < expectedRestarts {
 					break
 				}
 				restartedContainersCount++
@@ -514,6 +549,28 @@ func waitForPodResizeActuation(c clientset.Interface, podClient *e2epod.PodClien
 	return resizedPod
 }
 
+func genPatchString(containers []TestContainerInfo) (string, error) {
+	var patch patchSpec
+
+	for _, container := range containers {
+		var cPatch containerPatch
+		cPatch.Name = container.Name
+		cPatch.Resources.Requests.CPU = container.Resources.CPUReq
+		cPatch.Resources.Requests.Memory = container.Resources.MemReq
+		cPatch.Resources.Limits.CPU = container.Resources.CPULim
+		cPatch.Resources.Limits.Memory = container.Resources.MemLim
+
+		patch.Spec.Containers = append(patch.Spec.Containers, cPatch)
+	}
+
+	patchBytes, err := json.Marshal(patch)
+	if err != nil {
+		return "", err
+	}
+
+	return string(patchBytes), nil
+}
+
 func doPodResizeTests() {
 	f := framework.NewDefaultFramework("pod-resize")
 	var podClient *e2epod.PodClient
@@ -1263,31 +1320,40 @@ func doPodResizeTests() {
 				}
 			}
 
-			ginkgo.By("patching pod for resize")
-			patchedPod, pErr = f.ClientSet.CoreV1().Pods(newPod.Namespace).Patch(context.TODO(), newPod.Name,
-				types.StrategicMergePatchType, []byte(tc.patchString), metav1.PatchOptions{})
-			framework.ExpectNoError(pErr, "failed to patch pod for resize")
-
-			ginkgo.By("verifying pod patched for resize")
-			verifyPodResources(patchedPod, tc.expected)
-			verifyPodAllocations(patchedPod, tc.containers, true)
+			patchAndVerify := func(patchString string, expectedContainers []TestContainerInfo, initialContainers []TestContainerInfo, opStr string, isRollback bool) {
+				ginkgo.By(fmt.Sprintf("patching pod for %s", opStr))
+				patchedPod, pErr = f.ClientSet.CoreV1().Pods(newPod.Namespace).Patch(context.TODO(), newPod.Name,
+					types.StrategicMergePatchType, []byte(patchString), metav1.PatchOptions{})
+				framework.ExpectNoError(pErr, fmt.Sprintf("failed to patch pod for %s", opStr))
+
+				ginkgo.By(fmt.Sprintf("verifying pod patched for %s", opStr))
+				verifyPodResources(patchedPod, expectedContainers)
+				verifyPodAllocations(patchedPod, initialContainers, true)
+
+				ginkgo.By(fmt.Sprintf("waiting for %s to be actuated", opStr))
+				resizedPod := waitForPodResizeActuation(f.ClientSet, podClient, newPod, patchedPod, expectedContainers, initialContainers, isRollback)
+
+				// Check cgroup values only for containerd versions before 1.6.9
+				if !isInPlaceResizeSupportedByRuntime(f.ClientSet, newPod.Spec.NodeName) {
+					ginkgo.By(fmt.Sprintf("verifying pod container's cgroup values after %s", opStr))
+					if !framework.NodeOSDistroIs("windows") {
+						verifyPodContainersCgroupValues(resizedPod, expectedContainers, true)
+					}
+				}
 
-			ginkgo.By("waiting for resize to be actuated")
-			resizedPod := waitForPodResizeActuation(f.ClientSet, podClient, newPod, patchedPod, tc.expected)
+				ginkgo.By(fmt.Sprintf("verifying pod resources after %s", opStr))
+				verifyPodResources(resizedPod, expectedContainers)
 
-			// Check cgroup values only for containerd versions before 1.6.9
-			if !isInPlaceResizeSupportedByRuntime(f.ClientSet, newPod.Spec.NodeName) {
-				ginkgo.By("verifying pod container's cgroup values after resize")
-				if !framework.NodeOSDistroIs("windows") {
-					verifyPodContainersCgroupValues(resizedPod, tc.expected, true)
-				}
+				ginkgo.By(fmt.Sprintf("verifying pod allocations after %s", opStr))
+				verifyPodAllocations(resizedPod, expectedContainers, true)
 			}
 
-			ginkgo.By("verifying pod resources after resize")
-			verifyPodResources(resizedPod, tc.expected)
+			patchAndVerify(tc.patchString, tc.expected, tc.containers, "resize", false)
 
-			ginkgo.By("verifying pod allocations after resize")
-			verifyPodAllocations(resizedPod, tc.expected, true)
+			rbPatchStr, err := genPatchString(tc.containers)
+			framework.ExpectNoError(err)
+			// Resize has been actuated, test rollback
+			patchAndVerify(rbPatchStr, tc.containers, tc.expected, "rollback", true)
 
 			ginkgo.By("deleting pod")
 			err = e2epod.DeletePodWithWait(ctx, f.ClientSet, newPod)
@@ -1372,7 +1438,7 @@ func doPodResizeResourceQuotaTests() {
 		verifyPodAllocations(patchedPod, containers, true)
 
 		ginkgo.By("waiting for resize to be actuated")
-		resizedPod := waitForPodResizeActuation(f.ClientSet, podClient, newPod1, patchedPod, expected)
+		resizedPod := waitForPodResizeActuation(f.ClientSet, podClient, newPod1, patchedPod, expected, containers, false)
 		if !isInPlaceResizeSupportedByRuntime(f.ClientSet, newPod1.Spec.NodeName) {
 			ginkgo.By("verifying pod container's cgroup values after resize")
 			if !framework.NodeOSDistroIs("windows") {