Skip to content

Commit 9385626

Browse files
jbartosikjbartosik
committed
Squashed commits reverting kubernetes#3016
Revert "Revert "Add test for handling OOMing containers"" This reverts commit 644aa7a. Revert "Revert "Add docekrfile for OOMing pods under VPA test"" This reverts commit 68c12d3. Revert "Revert "Tweak OOM test"" This reverts commit c48f7f8. Revert "Revert "Enable tests for pods under VPA"" This reverts commit e3c38fa.
1 parent ae6c831 commit 9385626

File tree

7 files changed

+189
-27
lines changed

7 files changed

+189
-27
lines changed

‎vertical-pod-autoscaler/e2e/utils/ooming_resource_consumer/Dockerfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
FROM gcr.io/google-containers/stress:v1
2+
ENTRYPOINT ["/stress", "--mem-total", "10000000000", "--logtostderr", "--mem-alloc-size", "8000"]

‎vertical-pod-autoscaler/e2e/v1/autoscaling_utils.go‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@ const (
6060
customMetricName = "QPS"
6161
serviceInitializationTimeout = 2 * time.Minute
6262
serviceInitializationInterval = 15 * time.Second
63+
// TODO(jbartosik): put the image in a VPA project
64+
stressImage = "gcr.io/jbartosik-gke-dev/stress:0.10"
6365
)
6466

6567
var (
@@ -363,7 +365,7 @@ func runServiceAndWorkloadForResourceConsumer(c clientset.Interface, ns, name st
363365
Timeout: timeoutRC,
364366
Replicas: replicas,
365367
CpuRequest: cpuRequestMillis,
366-
MemRequest: memRequestMb * 1024 * 1024, // MemLimit is in bytes
368+
MemRequest: memRequestMb * 1024 * 1024, // Mem Request is in bytes
367369
Annotations: podAnnotations,
368370
}
369371

@@ -427,3 +429,27 @@ func runServiceAndWorkloadForResourceConsumer(c clientset.Interface, ns, name st
427429
framework.ExpectNoError(framework.WaitForServiceEndpointsNum(
428430
c, ns, controllerName, 1, startServiceInterval, startServiceTimeout))
429431
}
432+
433+
func runOomingReplicationController(c clientset.Interface, ns, name string, replicas int) {
434+
ginkgo.By(fmt.Sprintf("Running OOMing RC %s with %v replicas", name, replicas))
435+
436+
rcConfig := testutils.RCConfig{
437+
Client: c,
438+
Image: stressImage,
439+
Name: name,
440+
Namespace: ns,
441+
Timeout: timeoutRC,
442+
Replicas: replicas,
443+
Annotations: make(map[string]string),
444+
MemRequest: 1024 * 1024 * 1024,
445+
MemLimit: 1024 * 1024 * 1024,
446+
}
447+
448+
dpConfig := testutils.DeploymentConfig{
449+
RCConfig: rcConfig,
450+
}
451+
ginkgo.By(fmt.Sprintf("Creating deployment %s in namespace %s", dpConfig.Name, dpConfig.Namespace))
452+
dpConfig.NodeDumpFunc = framework.DumpNodeDebugInfo
453+
dpConfig.ContainerDumpFunc = framework.LogFailedContainers
454+
framework.ExpectNoError(testutils.RunDeployment(dpConfig))
455+
}

‎vertical-pod-autoscaler/e2e/v1/common.go‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -192,12 +192,17 @@ func NewHamsterDeploymentWithResourcesAndLimits(f *framework.Framework, cpuQuant
192192
return d
193193
}
194194

195+
func getPodSelectorExcludingDonePodsOrDie() string {
196+
stringSelector := "status.phase!=" + string(apiv1.PodSucceeded) +
197+
",status.phase!=" + string(apiv1.PodFailed)
198+
selector := fields.ParseSelectorOrDie(stringSelector)
199+
return selector.String()
200+
}
201+
195202
// GetHamsterPods returns running hamster pods (matched by hamsterLabels)
196203
func GetHamsterPods(f *framework.Framework) (*apiv1.PodList, error) {
197204
label := labels.SelectorFromSet(labels.Set(hamsterLabels))
198-
selector := fields.ParseSelectorOrDie("status.phase!=" + string(apiv1.PodSucceeded) +
199-
",status.phase!=" + string(apiv1.PodFailed))
200-
options := metav1.ListOptions{LabelSelector: label.String(), FieldSelector: selector.String()}
205+
options := metav1.ListOptions{LabelSelector: label.String(), FieldSelector: getPodSelectorExcludingDonePodsOrDie()}
201206
return f.ClientSet.CoreV1().Pods(f.Namespace.Name).List(options)
202207
}
203208

‎vertical-pod-autoscaler/e2e/v1/full_vpa.go‎

Lines changed: 59 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package autoscaling
1818

1919
import (
2020
"fmt"
21+
"time"
2122

2223
autoscaling "k8s.io/api/autoscaling/v1"
2324
apiv1 "k8s.io/api/core/v1"
@@ -91,37 +92,80 @@ var _ = FullVpaE2eDescribe("Pods under VPA", func() {
9192
ginkgo.It("have cpu requests growing with usage", func() {
9293
// initial CPU usage is low so a minimal recommendation is expected
9394
err := waitForResourceRequestInRangeInPods(
94-
f, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
95+
f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
9596
ParseQuantityOrDie(minimalCPULowerBound), ParseQuantityOrDie(minimalCPUUpperBound))
9697
gomega.Expect(err).NotTo(gomega.HaveOccurred())
9798

9899
// consume more CPU to get a higher recommendation
99100
rc.ConsumeCPU(600 * replicas)
100101
err = waitForResourceRequestInRangeInPods(
101-
f, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
102+
f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
102103
ParseQuantityOrDie("500m"), ParseQuantityOrDie("900m"))
103104
gomega.Expect(err).NotTo(gomega.HaveOccurred())
104105
})
105106

106107
ginkgo.It("have memory requests growing with usage", func() {
107108
// initial memory usage is low so a minimal recommendation is expected
108109
err := waitForResourceRequestInRangeInPods(
109-
f, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceMemory,
110+
f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceMemory,
110111
ParseQuantityOrDie(minimalMemoryLowerBound), ParseQuantityOrDie(minimalMemoryUpperBound))
111112
gomega.Expect(err).NotTo(gomega.HaveOccurred())
112113

113114
// consume more memory to get a higher recommendation
114115
// NOTE: large range given due to unpredictability of actual memory usage
115116
rc.ConsumeMem(1024 * replicas)
116117
err = waitForResourceRequestInRangeInPods(
117-
f, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceMemory,
118+
f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceMemory,
118119
ParseQuantityOrDie("900Mi"), ParseQuantityOrDie("4000Mi"))
119120
gomega.Expect(err).NotTo(gomega.HaveOccurred())
120121
})
121122
})
122123

123-
func waitForPodsMatch(f *framework.Framework, listOptions metav1.ListOptions, matcher func(pod apiv1.Pod) bool) error {
124-
return wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) {
124+
var _ = FullVpaE2eDescribe("OOMing pods under VPA", func() {
125+
var (
126+
vpaClientSet *vpa_clientset.Clientset
127+
vpaCRD *vpa_types.VerticalPodAutoscaler
128+
)
129+
const replicas = 3
130+
131+
f := framework.NewDefaultFramework("vertical-pod-autoscaling")
132+
133+
ginkgo.BeforeEach(func() {
134+
ns := f.Namespace.Name
135+
ginkgo.By("Setting up a hamster deployment")
136+
137+
runOomingReplicationController(
138+
f.ClientSet,
139+
ns,
140+
"hamster",
141+
replicas)
142+
ginkgo.By("Setting up a VPA CRD")
143+
config, err := framework.LoadConfig()
144+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
145+
146+
vpaCRD = NewVPA(f, "hamster-vpa", &autoscaling.CrossVersionObjectReference{
147+
APIVersion: "v1",
148+
Kind: "Deployment",
149+
Name: "hamster",
150+
})
151+
152+
vpaClientSet = vpa_clientset.NewForConfigOrDie(config)
153+
vpaClient := vpaClientSet.AutoscalingV1()
154+
_, err = vpaClient.VerticalPodAutoscalers(ns).Create(vpaCRD)
155+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
156+
})
157+
158+
ginkgo.It("have memory requests growing with OOMs", func() {
159+
listOptions := metav1.ListOptions{LabelSelector: "name=hamster", FieldSelector: getPodSelectorExcludingDonePodsOrDie()}
160+
err := waitForResourceRequestInRangeInPods(
161+
f, 7*time.Minute, listOptions, apiv1.ResourceMemory,
162+
ParseQuantityOrDie("1400Mi"), ParseQuantityOrDie("10000Mi"))
163+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
164+
})
165+
})
166+
167+
func waitForPodsMatch(f *framework.Framework, timeout time.Duration, listOptions metav1.ListOptions, matcher func(pod apiv1.Pod) bool) error {
168+
return wait.PollImmediate(pollInterval, timeout, func() (bool, error) {
125169

126170
ns := f.Namespace.Name
127171
c := f.ClientSet
@@ -135,18 +179,23 @@ func waitForPodsMatch(f *framework.Framework, listOptions metav1.ListOptions, ma
135179
return false, nil
136180
}
137181

182+
// Run matcher on all pods, even if we find pod that doesn't match early.
183+
// This allows the matcher to write logs for all pods. This in turns makes
184+
// it easier to spot some problems (for example unexpected pods in the list
185+
// results).
186+
result := true
138187
for _, pod := range podList.Items {
139188
if !matcher(pod) {
140-
return false, nil
189+
result = false
141190
}
142191
}
143-
return true, nil
192+
return result, nil
144193

145194
})
146195
}
147196

148-
func waitForResourceRequestInRangeInPods(f *framework.Framework, listOptions metav1.ListOptions, resourceName apiv1.ResourceName, lowerBound, upperBound resource.Quantity) error {
149-
err := waitForPodsMatch(f, listOptions,
197+
func waitForResourceRequestInRangeInPods(f *framework.Framework, timeout time.Duration, listOptions metav1.ListOptions, resourceName apiv1.ResourceName, lowerBound, upperBound resource.Quantity) error {
198+
err := waitForPodsMatch(f, timeout, listOptions,
150199
func(pod apiv1.Pod) bool {
151200
resourceRequest, found := pod.Spec.Containers[0].Resources.Requests[resourceName]
152201
framework.Logf("Comparing %v request %v against range of (%v, %v)", resourceName, resourceRequest, lowerBound, upperBound)

‎vertical-pod-autoscaler/e2e/v1beta2/autoscaling_utils.go‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@ const (
6060
customMetricName = "QPS"
6161
serviceInitializationTimeout = 2 * time.Minute
6262
serviceInitializationInterval = 15 * time.Second
63+
// TODO(jbartosik): put the image in a VPA project
64+
stressImage = "gcr.io/jbartosik-gke-dev/stress:0.10"
6365
)
6466

6567
var (
@@ -427,3 +429,27 @@ func runServiceAndWorkloadForResourceConsumer(c clientset.Interface, ns, name st
427429
framework.ExpectNoError(framework.WaitForServiceEndpointsNum(
428430
c, ns, controllerName, 1, startServiceInterval, startServiceTimeout))
429431
}
432+
433+
func runOomingReplicationController(c clientset.Interface, ns, name string, replicas int) {
434+
ginkgo.By(fmt.Sprintf("Running OOMing RC %s with %v replicas", name, replicas))
435+
436+
rcConfig := testutils.RCConfig{
437+
Client: c,
438+
Image: stressImage,
439+
Name: name,
440+
Namespace: ns,
441+
Timeout: timeoutRC,
442+
Replicas: replicas,
443+
Annotations: make(map[string]string),
444+
MemRequest: 1024 * 1024 * 1024,
445+
MemLimit: 1024 * 1024 * 1024,
446+
}
447+
448+
dpConfig := testutils.DeploymentConfig{
449+
RCConfig: rcConfig,
450+
}
451+
ginkgo.By(fmt.Sprintf("Creating deployment %s in namespace %s", dpConfig.Name, dpConfig.Namespace))
452+
dpConfig.NodeDumpFunc = framework.DumpNodeDebugInfo
453+
dpConfig.ContainerDumpFunc = framework.LogFailedContainers
454+
framework.ExpectNoError(testutils.RunDeployment(dpConfig))
455+
}

‎vertical-pod-autoscaler/e2e/v1beta2/common.go‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -192,12 +192,17 @@ func NewHamsterDeploymentWithResourcesAndLimits(f *framework.Framework, cpuQuant
192192
return d
193193
}
194194

195+
func getPodSelectorExcludingDonePodsOrDie() string {
196+
stringSelector := "status.phase!=" + string(apiv1.PodSucceeded) +
197+
",status.phase!=" + string(apiv1.PodFailed)
198+
selector := fields.ParseSelectorOrDie(stringSelector)
199+
return selector.String()
200+
}
201+
195202
// GetHamsterPods returns running hamster pods (matched by hamsterLabels)
196203
func GetHamsterPods(f *framework.Framework) (*apiv1.PodList, error) {
197204
label := labels.SelectorFromSet(labels.Set(hamsterLabels))
198-
selector := fields.ParseSelectorOrDie("status.phase!=" + string(apiv1.PodSucceeded) +
199-
",status.phase!=" + string(apiv1.PodFailed))
200-
options := metav1.ListOptions{LabelSelector: label.String(), FieldSelector: selector.String()}
205+
options := metav1.ListOptions{LabelSelector: label.String(), FieldSelector: getPodSelectorExcludingDonePodsOrDie()}
201206
return f.ClientSet.CoreV1().Pods(f.Namespace.Name).List(options)
202207
}
203208

‎vertical-pod-autoscaler/e2e/v1beta2/full_vpa.go‎

Lines changed: 59 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package autoscaling
1818

1919
import (
2020
"fmt"
21+
"time"
2122

2223
autoscaling "k8s.io/api/autoscaling/v1"
2324
apiv1 "k8s.io/api/core/v1"
@@ -91,37 +92,80 @@ var _ = FullVpaE2eDescribe("Pods under VPA", func() {
9192
ginkgo.It("have cpu requests growing with usage", func() {
9293
// initial CPU usage is low so a minimal recommendation is expected
9394
err := waitForResourceRequestInRangeInPods(
94-
f, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
95+
f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
9596
ParseQuantityOrDie(minimalCPULowerBound), ParseQuantityOrDie(minimalCPUUpperBound))
9697
gomega.Expect(err).NotTo(gomega.HaveOccurred())
9798

9899
// consume more CPU to get a higher recommendation
99100
rc.ConsumeCPU(600 * replicas)
100101
err = waitForResourceRequestInRangeInPods(
101-
f, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
102+
f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
102103
ParseQuantityOrDie("500m"), ParseQuantityOrDie("900m"))
103104
gomega.Expect(err).NotTo(gomega.HaveOccurred())
104105
})
105106

106107
ginkgo.It("have memory requests growing with usage", func() {
107108
// initial memory usage is low so a minimal recommendation is expected
108109
err := waitForResourceRequestInRangeInPods(
109-
f, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceMemory,
110+
f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceMemory,
110111
ParseQuantityOrDie(minimalMemoryLowerBound), ParseQuantityOrDie(minimalMemoryUpperBound))
111112
gomega.Expect(err).NotTo(gomega.HaveOccurred())
112113

113114
// consume more memory to get a higher recommendation
114115
// NOTE: large range given due to unpredictability of actual memory usage
115116
rc.ConsumeMem(1024 * replicas)
116117
err = waitForResourceRequestInRangeInPods(
117-
f, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceMemory,
118+
f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceMemory,
118119
ParseQuantityOrDie("900Mi"), ParseQuantityOrDie("4000Mi"))
119120
gomega.Expect(err).NotTo(gomega.HaveOccurred())
120121
})
121122
})
122123

123-
func waitForPodsMatch(f *framework.Framework, listOptions metav1.ListOptions, matcher func(pod apiv1.Pod) bool) error {
124-
return wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) {
124+
var _ = FullVpaE2eDescribe("OOMing pods under VPA", func() {
125+
var (
126+
vpaClientSet *vpa_clientset.Clientset
127+
vpaCRD *vpa_types.VerticalPodAutoscaler
128+
)
129+
const replicas = 3
130+
131+
f := framework.NewDefaultFramework("vertical-pod-autoscaling")
132+
133+
ginkgo.BeforeEach(func() {
134+
ns := f.Namespace.Name
135+
ginkgo.By("Setting up a hamster deployment")
136+
137+
runOomingReplicationController(
138+
f.ClientSet,
139+
ns,
140+
"hamster",
141+
replicas)
142+
ginkgo.By("Setting up a VPA CRD")
143+
config, err := framework.LoadConfig()
144+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
145+
146+
vpaCRD = NewVPA(f, "hamster-vpa", &autoscaling.CrossVersionObjectReference{
147+
APIVersion: "apps/v1",
148+
Kind: "Deployment",
149+
Name: "hamster",
150+
})
151+
152+
vpaClientSet = vpa_clientset.NewForConfigOrDie(config)
153+
vpaClient := vpaClientSet.AutoscalingV1beta2()
154+
_, err = vpaClient.VerticalPodAutoscalers(ns).Create(vpaCRD)
155+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
156+
})
157+
158+
ginkgo.It("have memory requests growing with OOMs", func() {
159+
listOptions := metav1.ListOptions{LabelSelector: "name=hamster", FieldSelector: getPodSelectorExcludingDonePodsOrDie()}
160+
err := waitForResourceRequestInRangeInPods(
161+
f, 7*time.Minute, listOptions, apiv1.ResourceMemory,
162+
ParseQuantityOrDie("1400Mi"), ParseQuantityOrDie("10000Mi"))
163+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
164+
})
165+
})
166+
167+
func waitForPodsMatch(f *framework.Framework, timeout time.Duration, listOptions metav1.ListOptions, matcher func(pod apiv1.Pod) bool) error {
168+
return wait.PollImmediate(pollInterval, timeout, func() (bool, error) {
125169

126170
ns := f.Namespace.Name
127171
c := f.ClientSet
@@ -135,18 +179,23 @@ func waitForPodsMatch(f *framework.Framework, listOptions metav1.ListOptions, ma
135179
return false, nil
136180
}
137181

182+
// Run matcher on all pods, even if we find pod that doesn't match early.
183+
// This allows the matcher to write logs for all pods. This in turns makes
184+
// it easier to spot some problems (for example unexpected pods in the list
185+
// results).
186+
result := true
138187
for _, pod := range podList.Items {
139188
if !matcher(pod) {
140-
return false, nil
189+
result = false
141190
}
142191
}
143-
return true, nil
192+
return result, nil
144193

145194
})
146195
}
147196

148-
func waitForResourceRequestInRangeInPods(f *framework.Framework, listOptions metav1.ListOptions, resourceName apiv1.ResourceName, lowerBound, upperBound resource.Quantity) error {
149-
err := waitForPodsMatch(f, listOptions,
197+
func waitForResourceRequestInRangeInPods(f *framework.Framework, timeout time.Duration, listOptions metav1.ListOptions, resourceName apiv1.ResourceName, lowerBound, upperBound resource.Quantity) error {
198+
err := waitForPodsMatch(f, timeout, listOptions,
150199
func(pod apiv1.Pod) bool {
151200
resourceRequest, found := pod.Spec.Containers[0].Resources.Requests[resourceName]
152201
framework.Logf("Comparing %v request %v against range of (%v, %v)", resourceName, resourceRequest, lowerBound, upperBound)

0 commit comments

Comments
 (0)