Skip to content

Commit 087e946

Browse files
maxcao13maxcao13
committed
VPA: add InPlaceOrRecreate e2e tests
Signed-off-by: Max Cao <[email protected]>
1 parent 8a9a4b8 commit 087e946

File tree

5 files changed

+660
-59
lines changed

5 files changed

+660
-59
lines changed

‎vertical-pod-autoscaler/e2e/v1/actuation.go‎

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import (
3535
"k8s.io/apimachinery/pkg/util/wait"
3636
"k8s.io/autoscaler/vertical-pod-autoscaler/e2e/utils"
3737
vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
38+
restriction "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/updater/restriction"
3839
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/annotations"
3940
clientset "k8s.io/client-go/kubernetes"
4041
"k8s.io/kubernetes/test/e2e/framework"
@@ -50,6 +51,236 @@ import (
5051
"github.com/onsi/gomega"
5152
)
5253

54+
var _ = ActuationSuiteE2eDescribe("Actuation [InPlaceOrRecreate]", func() {
55+
f := framework.NewDefaultFramework("vertical-pod-autoscaling")
56+
f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline
57+
58+
ginkgo.BeforeEach(func() {
59+
checkInPlaceOrRecreateTestsEnabled(f, true, true)
60+
})
61+
62+
ginkgo.It("still applies recommendations on restart when update mode is InPlaceOrRecreate", func() {
63+
ginkgo.By("Setting up a hamster deployment")
64+
SetupHamsterDeployment(f, "100m", "100Mi", defaultHamsterReplicas)
65+
podList, err := GetHamsterPods(f)
66+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
67+
podSet := MakePodSet(podList)
68+
69+
ginkgo.By("Setting up a VPA CRD in mode InPlaceOrRecreate")
70+
containerName := GetHamsterContainerNameByIndex(0)
71+
vpaCRD := test.VerticalPodAutoscaler().
72+
WithName("hamster-vpa").
73+
WithNamespace(f.Namespace.Name).
74+
WithTargetRef(hamsterTargetRef).
75+
WithUpdateMode(vpa_types.UpdateModeInPlaceOrRecreate).
76+
WithContainer(containerName).
77+
AppendRecommendation(
78+
test.Recommendation().
79+
WithContainer(containerName).
80+
WithTarget("200m", "").
81+
WithLowerBound("200m", "").
82+
WithUpperBound("200m", "").
83+
GetContainerResources()).
84+
Get()
85+
86+
InstallVPA(f, vpaCRD)
87+
updatedCPURequest := ParseQuantityOrDie("200m")
88+
89+
ginkgo.By(fmt.Sprintf("Waiting for pods to be evicted, hoping it won't happen, sleep for %s", VpaEvictionTimeout.String()))
90+
CheckNoPodsEvicted(f, podSet)
91+
ginkgo.By("Forcefully killing one pod")
92+
killPod(f, podList)
93+
94+
ginkgo.By("Checking that request was modified after forceful restart")
95+
updatedPodList, _ := GetHamsterPods(f)
96+
var foundUpdated int32
97+
for _, pod := range updatedPodList.Items {
98+
podRequest := getCPURequest(pod.Spec)
99+
framework.Logf("podReq: %v", podRequest)
100+
if podRequest.Cmp(updatedCPURequest) == 0 {
101+
foundUpdated += 1
102+
}
103+
}
104+
gomega.Expect(foundUpdated).To(gomega.Equal(defaultHamsterReplicas))
105+
})
106+
107+
// TODO: add e2e test to verify metrics are getting updated
108+
ginkgo.It("applies in-place updates to all containers when update mode is InPlaceOrRecreate", func() {
109+
ginkgo.By("Setting up a hamster deployment")
110+
d := NewNHamstersDeployment(f, 2 /*number of containers*/)
111+
d.Spec.Template.Spec.Containers[0].Resources.Requests = apiv1.ResourceList{
112+
apiv1.ResourceCPU: ParseQuantityOrDie("100m"),
113+
apiv1.ResourceMemory: ParseQuantityOrDie("100Mi"),
114+
}
115+
d.Spec.Template.Spec.Containers[1].Resources.Requests = apiv1.ResourceList{
116+
apiv1.ResourceCPU: ParseQuantityOrDie("100m"),
117+
apiv1.ResourceMemory: ParseQuantityOrDie("100Mi"),
118+
}
119+
targetCPU := "200m"
120+
targetMemory := "200Mi"
121+
_ = startDeploymentPods(f, d) // 3 replicas
122+
container1Name := GetHamsterContainerNameByIndex(0)
123+
container2Name := GetHamsterContainerNameByIndex(1)
124+
podList, err := GetHamsterPods(f)
125+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
126+
127+
ginkgo.By("Setting up a VPA CRD")
128+
vpaCRD := test.VerticalPodAutoscaler().
129+
WithName("hamster-vpa").
130+
WithNamespace(f.Namespace.Name).
131+
WithTargetRef(hamsterTargetRef).
132+
WithContainer(container1Name).
133+
WithContainer(container2Name).
134+
WithUpdateMode(vpa_types.UpdateModeInPlaceOrRecreate).
135+
AppendRecommendation(
136+
test.Recommendation().
137+
WithContainer(container1Name).
138+
WithTarget(targetCPU, targetMemory).
139+
WithLowerBound(targetCPU, targetMemory).
140+
WithUpperBound(targetCPU, targetMemory).
141+
GetContainerResources()).
142+
AppendRecommendation(
143+
test.Recommendation().
144+
WithContainer(container2Name).
145+
WithTarget(targetCPU, targetMemory).
146+
WithLowerBound(targetCPU, targetMemory).
147+
WithUpperBound(targetCPU, targetMemory).
148+
GetContainerResources()).
149+
Get()
150+
151+
InstallVPA(f, vpaCRD)
152+
153+
ginkgo.By("Checking that resources were modified due to in-place update, not due to evictions")
154+
err = WaitForPodsUpdatedWithoutEviction(f, podList)
155+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
156+
157+
ginkgo.By("Checking that container resources were actually updated")
158+
gomega.Eventually(func() error {
159+
updatedPodList, err := GetHamsterPods(f)
160+
if err != nil {
161+
return err
162+
}
163+
for _, pod := range updatedPodList.Items {
164+
for _, container := range pod.Status.ContainerStatuses {
165+
cpuRequest := container.Resources.Requests[apiv1.ResourceCPU]
166+
memoryRequest := container.Resources.Requests[apiv1.ResourceMemory]
167+
if cpuRequest.Cmp(ParseQuantityOrDie(targetCPU)) != 0 {
168+
framework.Logf("%v/%v has not been updated to %v yet: currently=%v", pod.Name, container.Name, targetCPU, cpuRequest.String())
169+
return fmt.Errorf("%s CPU request not updated", container.Name)
170+
}
171+
if memoryRequest.Cmp(ParseQuantityOrDie(targetMemory)) != 0 {
172+
framework.Logf("%v/%v has not been updated to %v yet: currently=%v", pod.Name, container.Name, targetMemory, memoryRequest.String())
173+
return fmt.Errorf("%s Memory request not updated", container.Name)
174+
}
175+
}
176+
}
177+
return nil
178+
}, VpaInPlaceTimeout*3, 15*time.Second).Should(gomega.Succeed())
179+
})
180+
181+
ginkgo.It("falls back to evicting pods when in-place update is Infeasible when update mode is InPlaceOrRecreate", func() {
182+
ginkgo.By("Setting up a hamster deployment")
183+
replicas := int32(2)
184+
SetupHamsterDeployment(f, "100m", "100Mi", replicas)
185+
podList, err := GetHamsterPods(f)
186+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
187+
188+
ginkgo.By("Setting up a VPA CRD")
189+
containerName := GetHamsterContainerNameByIndex(0)
190+
updatedCPU := "999" // infeasible target
191+
vpaCRD := test.VerticalPodAutoscaler().
192+
WithName("hamster-vpa").
193+
WithNamespace(f.Namespace.Name).
194+
WithTargetRef(hamsterTargetRef).
195+
WithContainer(containerName).
196+
WithUpdateMode(vpa_types.UpdateModeInPlaceOrRecreate).
197+
AppendRecommendation(
198+
test.Recommendation().
199+
WithContainer(containerName).
200+
WithTarget(updatedCPU, "").
201+
WithLowerBound("200m", "").
202+
WithUpperBound("200m", "").
203+
GetContainerResources()).
204+
Get()
205+
206+
InstallVPA(f, vpaCRD)
207+
208+
ginkgo.By("Waiting for pods to be evicted")
209+
err = WaitForPodsEvicted(f, podList)
210+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
211+
})
212+
213+
ginkgo.It("falls back to evicting pods when resize is Deferred and more than 5 minute has elapsed since last in-place update when update mode is InPlaceOrRecreate", func() {
214+
ginkgo.By("Setting up a hamster deployment")
215+
replicas := int32(2)
216+
SetupHamsterDeployment(f, "100m", "100Mi", replicas)
217+
podList, err := GetHamsterPods(f)
218+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
219+
220+
ginkgo.By("Setting up a VPA CRD")
221+
containerName := GetHamsterContainerNameByIndex(0)
222+
223+
// we can force deferred resize by setting the target CPU to the allocatable CPU of the node
224+
// it will be close enough to the node capacity, such that the kubelet defers instead of marking it infeasible
225+
nodeName := podList.Items[0].Spec.NodeName
226+
node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
227+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
228+
allocatableCPU := node.Status.Allocatable[apiv1.ResourceCPU]
229+
updatedCPU := allocatableCPU.String()
230+
231+
vpaCRD := test.VerticalPodAutoscaler().
232+
WithName("hamster-vpa").
233+
WithNamespace(f.Namespace.Name).
234+
WithTargetRef(hamsterTargetRef).
235+
WithContainer(containerName).
236+
WithUpdateMode(vpa_types.UpdateModeInPlaceOrRecreate).
237+
AppendRecommendation(
238+
test.Recommendation().
239+
WithContainer(containerName).
240+
WithTarget(updatedCPU, "").
241+
WithLowerBound("200m", "").
242+
WithUpperBound("200m", "").
243+
GetContainerResources()).
244+
Get()
245+
246+
InstallVPA(f, vpaCRD)
247+
248+
ginkgo.By("Waiting for status to be Deferred")
249+
gomega.Eventually(func() error {
250+
updatedPodList, err := GetHamsterPods(f)
251+
if err != nil {
252+
return err
253+
}
254+
for _, pod := range updatedPodList.Items {
255+
if pod.Status.Resize == apiv1.PodResizeStatusDeferred {
256+
return nil
257+
}
258+
}
259+
return fmt.Errorf("status not deferred")
260+
}, VpaInPlaceTimeout, 5*time.Second).Should(gomega.Succeed())
261+
262+
ginkgo.By("Making sure pods are not evicted yet")
263+
gomega.Consistently(func() error {
264+
updatedPodList, err := GetHamsterPods(f)
265+
if err != nil {
266+
return fmt.Errorf("failed to get pods: %v", err)
267+
}
268+
for _, pod := range updatedPodList.Items {
269+
request := getCPURequestFromStatus(pod.Status)
270+
if request.Cmp(ParseQuantityOrDie(updatedCPU)) == 0 {
271+
framework.Logf("%v/%v updated to %v, that wasn't supposed to happen this early", pod.Name, containerName, updatedCPU)
272+
return fmt.Errorf("%s CPU request should not have been updated", containerName)
273+
}
274+
}
275+
return nil
276+
}, restriction.DeferredResizeUpdateTimeout, 10*time.Second).Should(gomega.Succeed())
277+
278+
ginkgo.By("Waiting for pods to be evicted")
279+
err = WaitForPodsEvicted(f, podList)
280+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
281+
})
282+
})
283+
53284
var _ = ActuationSuiteE2eDescribe("Actuation", func() {
54285
f := framework.NewDefaultFramework("vertical-pod-autoscaling")
55286
f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline
@@ -519,6 +750,10 @@ func getCPURequest(podSpec apiv1.PodSpec) resource.Quantity {
519750
return podSpec.Containers[0].Resources.Requests[apiv1.ResourceCPU]
520751
}
521752

753+
func getCPURequestFromStatus(podStatus apiv1.PodStatus) resource.Quantity {
754+
return podStatus.ContainerStatuses[0].Resources.Requests[apiv1.ResourceCPU]
755+
}
756+
522757
func killPod(f *framework.Framework, podList *apiv1.PodList) {
523758
f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(context.TODO(), podList.Items[0].Name, metav1.DeleteOptions{})
524759
err := WaitForPodsRestarted(f, podList)

‎vertical-pod-autoscaler/e2e/v1/admission_controller.go‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,19 @@ import (
3737
"github.com/onsi/gomega"
3838
)
3939

40+
const (
41+
webhookConfigName = "vpa-webhook-config"
42+
webhookName = "vpa.k8s.io"
43+
)
44+
4045
var _ = AdmissionControllerE2eDescribe("Admission-controller", func() {
4146
f := framework.NewDefaultFramework("vertical-pod-autoscaling")
4247
f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline
4348

49+
ginkgo.BeforeEach(func() {
50+
waitForVpaWebhookRegistration(f)
51+
})
52+
4453
ginkgo.It("starts pods with new recommended request", func() {
4554
d := NewHamsterDeploymentWithResources(f, ParseQuantityOrDie("100m") /*cpu*/, ParseQuantityOrDie("100Mi") /*memory*/)
4655

@@ -908,6 +917,40 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() {
908917
gomega.Expect(err.Error()).To(gomega.MatchRegexp(`.*admission webhook .*vpa.* denied the request: .*`), "Admission controller did not inspect the object")
909918
})
910919

920+
ginkgo.It("starts pods with new recommended request with InPlaceOrRecreate mode", func() {
921+
checkInPlaceOrRecreateTestsEnabled(f, true, false)
922+
923+
d := NewHamsterDeploymentWithResources(f, ParseQuantityOrDie("100m") /*cpu*/, ParseQuantityOrDie("100Mi") /*memory*/)
924+
925+
ginkgo.By("Setting up a VPA CRD")
926+
containerName := GetHamsterContainerNameByIndex(0)
927+
vpaCRD := test.VerticalPodAutoscaler().
928+
WithName("hamster-vpa").
929+
WithNamespace(f.Namespace.Name).
930+
WithTargetRef(hamsterTargetRef).
931+
WithContainer(containerName).
932+
WithUpdateMode(vpa_types.UpdateModeInPlaceOrRecreate).
933+
AppendRecommendation(
934+
test.Recommendation().
935+
WithContainer(containerName).
936+
WithTarget("250m", "200Mi").
937+
WithLowerBound("250m", "200Mi").
938+
WithUpperBound("250m", "200Mi").
939+
GetContainerResources()).
940+
Get()
941+
942+
InstallVPA(f, vpaCRD)
943+
944+
ginkgo.By("Setting up a hamster deployment")
945+
podList := startDeploymentPods(f, d)
946+
947+
// Originally Pods had 100m CPU, 100Mi of memory, but admission controller
948+
// should change it to recommended 250m CPU and 200Mi of memory.
949+
for _, pod := range podList.Items {
950+
gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceCPU]).To(gomega.Equal(ParseQuantityOrDie("250m")))
951+
gomega.Expect(pod.Spec.Containers[0].Resources.Requests[apiv1.ResourceMemory]).To(gomega.Equal(ParseQuantityOrDie("200Mi")))
952+
}
953+
})
911954
})
912955

913956
func startDeploymentPods(f *framework.Framework, deployment *appsv1.Deployment) *apiv1.PodList {
@@ -962,3 +1005,17 @@ func startDeploymentPods(f *framework.Framework, deployment *appsv1.Deployment)
9621005
gomega.Expect(err).NotTo(gomega.HaveOccurred(), "when listing pods after deployment resize")
9631006
return podList
9641007
}
1008+
1009+
func waitForVpaWebhookRegistration(f *framework.Framework) {
1010+
ginkgo.By("Waiting for VPA webhook registration")
1011+
gomega.Eventually(func() bool {
1012+
webhook, err := f.ClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(context.TODO(), webhookConfigName, metav1.GetOptions{})
1013+
if err != nil {
1014+
return false
1015+
}
1016+
if webhook != nil && len(webhook.Webhooks) > 0 && webhook.Webhooks[0].Name == webhookName {
1017+
return true
1018+
}
1019+
return false
1020+
}, 3*time.Minute, 5*time.Second).Should(gomega.BeTrue(), "Webhook was not registered in the cluster")
1021+
}

0 commit comments

Comments
 (0)