Skip to content

Commit 73f7c19

Browse files
AnishShahAnishShah
committed
test: refactor logic to add/remove extended resources
1 parent 79cca27 commit 73f7c19

File tree

3 files changed

+90
-136
lines changed

3 files changed

+90
-136
lines changed

‎test/e2e/common/node/pod_resize.go‎

Lines changed: 2 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,13 @@ package node
1818

1919
import (
2020
"context"
21-
"encoding/json"
2221
"fmt"
2322
"strconv"
2423
"time"
2524

2625
"k8s.io/apimachinery/pkg/api/resource"
2726
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2827
"k8s.io/apimachinery/pkg/types"
29-
"k8s.io/apimachinery/pkg/util/strategicpatch"
30-
clientset "k8s.io/client-go/kubernetes"
3128
"k8s.io/kubernetes/test/e2e/feature"
3229
"k8s.io/kubernetes/test/e2e/framework"
3330
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
@@ -43,78 +40,6 @@ const (
4340
fakeExtendedResource = "dummy.com/dummy"
4441
)
4542

46-
func patchNode(ctx context.Context, client clientset.Interface, old *v1.Node, new *v1.Node) error {
47-
oldData, err := json.Marshal(old)
48-
if err != nil {
49-
return err
50-
}
51-
52-
newData, err := json.Marshal(new)
53-
if err != nil {
54-
return err
55-
}
56-
patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, &v1.Node{})
57-
if err != nil {
58-
return fmt.Errorf("failed to create merge patch for node %q: %w", old.Name, err)
59-
}
60-
_, err = client.CoreV1().Nodes().Patch(ctx, old.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status")
61-
return err
62-
}
63-
64-
func addExtendedResource(clientSet clientset.Interface, nodeName, extendedResourceName string, extendedResourceQuantity resource.Quantity) {
65-
extendedResource := v1.ResourceName(extendedResourceName)
66-
67-
ginkgo.By("Adding a custom resource")
68-
OriginalNode, err := clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{})
69-
framework.ExpectNoError(err)
70-
71-
node := OriginalNode.DeepCopy()
72-
node.Status.Capacity[extendedResource] = extendedResourceQuantity
73-
node.Status.Allocatable[extendedResource] = extendedResourceQuantity
74-
err = patchNode(context.Background(), clientSet, OriginalNode.DeepCopy(), node)
75-
framework.ExpectNoError(err)
76-
77-
gomega.Eventually(func() error {
78-
node, err = clientSet.CoreV1().Nodes().Get(context.Background(), node.Name, metav1.GetOptions{})
79-
framework.ExpectNoError(err)
80-
81-
fakeResourceCapacity, exists := node.Status.Capacity[extendedResource]
82-
if !exists {
83-
return fmt.Errorf("node %s has no %s resource capacity", node.Name, extendedResourceName)
84-
}
85-
if expectedResource := resource.MustParse("123"); fakeResourceCapacity.Cmp(expectedResource) != 0 {
86-
return fmt.Errorf("node %s has resource capacity %s, expected: %s", node.Name, fakeResourceCapacity.String(), expectedResource.String())
87-
}
88-
89-
return nil
90-
}).WithTimeout(30 * time.Second).WithPolling(time.Second).ShouldNot(gomega.HaveOccurred())
91-
}
92-
93-
func removeExtendedResource(clientSet clientset.Interface, nodeName, extendedResourceName string) {
94-
extendedResource := v1.ResourceName(extendedResourceName)
95-
96-
ginkgo.By("Removing a custom resource")
97-
originalNode, err := clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{})
98-
framework.ExpectNoError(err)
99-
100-
node := originalNode.DeepCopy()
101-
delete(node.Status.Capacity, extendedResource)
102-
delete(node.Status.Allocatable, extendedResource)
103-
err = patchNode(context.Background(), clientSet, originalNode.DeepCopy(), node)
104-
framework.ExpectNoError(err)
105-
106-
gomega.Eventually(func() error {
107-
node, err = clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{})
108-
framework.ExpectNoError(err)
109-
110-
if _, exists := node.Status.Capacity[extendedResource]; exists {
111-
return fmt.Errorf("node %s has resource capacity %s which is expected to be removed", node.Name, extendedResourceName)
112-
}
113-
114-
return nil
115-
}).WithTimeout(30 * time.Second).WithPolling(time.Second).ShouldNot(gomega.HaveOccurred())
116-
}
117-
11843
func doPodResizeTests(f *framework.Framework) {
11944
type testCase struct {
12045
name string
@@ -870,11 +795,11 @@ func doPodResizeTests(f *framework.Framework) {
870795
framework.ExpectNoError(err)
871796

872797
for _, node := range nodes.Items {
873-
addExtendedResource(f.ClientSet, node.Name, fakeExtendedResource, resource.MustParse("123"))
798+
e2enode.AddExtendedResource(ctx, f.ClientSet, node.Name, fakeExtendedResource, resource.MustParse("123"))
874799
}
875800
defer func() {
876801
for _, node := range nodes.Items {
877-
removeExtendedResource(f.ClientSet, node.Name, fakeExtendedResource)
802+
e2enode.RemoveExtendedResource(ctx, f.ClientSet, node.Name, fakeExtendedResource)
878803
}
879804
}()
880805
}

‎test/e2e/framework/node/helper.go‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,17 @@ package node
1818

1919
import (
2020
"context"
21+
"encoding/json"
2122
"fmt"
2223
"time"
2324

2425
"github.com/onsi/ginkgo/v2"
2526
"github.com/onsi/gomega"
2627
v1 "k8s.io/api/core/v1"
28+
"k8s.io/apimachinery/pkg/api/resource"
2729
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
30+
"k8s.io/apimachinery/pkg/types"
31+
"k8s.io/apimachinery/pkg/util/strategicpatch"
2832
"k8s.io/apimachinery/pkg/util/wait"
2933
clientset "k8s.io/client-go/kubernetes"
3034

@@ -175,3 +179,78 @@ func IsARM64(node *v1.Node) bool {
175179

176180
return false
177181
}
182+
183+
// patchNode sends a patch request to update k8s Node resource.
184+
func patchNode(ctx context.Context, client clientset.Interface, old *v1.Node, new *v1.Node) error {
185+
oldData, err := json.Marshal(old)
186+
if err != nil {
187+
return err
188+
}
189+
190+
newData, err := json.Marshal(new)
191+
if err != nil {
192+
return err
193+
}
194+
patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, &v1.Node{})
195+
if err != nil {
196+
return fmt.Errorf("failed to create merge patch for node %q: %w", old.Name, err)
197+
}
198+
_, err = client.CoreV1().Nodes().Patch(ctx, old.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status")
199+
return err
200+
}
201+
202+
// AddExtendedResource adds a fake resource to k8s Node status.
203+
func AddExtendedResource(ctx context.Context, clientSet clientset.Interface, nodeName string, extendedResourceName v1.ResourceName, extendedResourceQuantity resource.Quantity) {
204+
extendedResource := v1.ResourceName(extendedResourceName)
205+
206+
ginkgo.By("Adding a custom resource")
207+
OriginalNode, err := clientSet.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
208+
framework.ExpectNoError(err)
209+
210+
node := OriginalNode.DeepCopy()
211+
node.Status.Capacity[extendedResource] = extendedResourceQuantity
212+
node.Status.Allocatable[extendedResource] = extendedResourceQuantity
213+
err = patchNode(ctx, clientSet, OriginalNode.DeepCopy(), node)
214+
framework.ExpectNoError(err)
215+
216+
gomega.Eventually(func() error {
217+
node, err = clientSet.CoreV1().Nodes().Get(ctx, node.Name, metav1.GetOptions{})
218+
framework.ExpectNoError(err)
219+
220+
fakeResourceCapacity, exists := node.Status.Capacity[extendedResource]
221+
if !exists {
222+
return fmt.Errorf("node %s has no %s resource capacity", node.Name, extendedResourceName)
223+
}
224+
if expectedResource := resource.MustParse("123"); fakeResourceCapacity.Cmp(expectedResource) != 0 {
225+
return fmt.Errorf("node %s has resource capacity %s, expected: %s", node.Name, fakeResourceCapacity.String(), expectedResource.String())
226+
}
227+
228+
return nil
229+
}).WithTimeout(30 * time.Second).WithPolling(time.Second).ShouldNot(gomega.HaveOccurred())
230+
}
231+
232+
// RemoveExtendedResource removes a fake resource to k8s Node status.
233+
func RemoveExtendedResource(ctx context.Context, clientSet clientset.Interface, nodeName string, extendedResourceName v1.ResourceName) {
234+
extendedResource := v1.ResourceName(extendedResourceName)
235+
236+
ginkgo.By("Removing a custom resource")
237+
originalNode, err := clientSet.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
238+
framework.ExpectNoError(err)
239+
240+
node := originalNode.DeepCopy()
241+
delete(node.Status.Capacity, extendedResource)
242+
delete(node.Status.Allocatable, extendedResource)
243+
err = patchNode(ctx, clientSet, originalNode.DeepCopy(), node)
244+
framework.ExpectNoError(err)
245+
246+
gomega.Eventually(func() error {
247+
node, err = clientSet.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
248+
framework.ExpectNoError(err)
249+
250+
if _, exists := node.Status.Capacity[extendedResource]; exists {
251+
return fmt.Errorf("node %s has resource capacity %s which is expected to be removed", node.Name, extendedResourceName)
252+
}
253+
254+
return nil
255+
}).WithTimeout(30 * time.Second).WithPolling(time.Second).ShouldNot(gomega.HaveOccurred())
256+
}

‎test/e2e/scheduling/preemption.go‎

Lines changed: 9 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -83,11 +83,7 @@ var _ = SIGDescribe("SchedulerPreemption", framework.WithSerial(), func() {
8383
_ = cs.SchedulingV1().PriorityClasses().Delete(ctx, pair.name, *metav1.NewDeleteOptions(0))
8484
}
8585
for _, node := range nodeList.Items {
86-
nodeCopy := node.DeepCopy()
87-
delete(nodeCopy.Status.Capacity, testExtendedResource)
88-
delete(nodeCopy.Status.Allocatable, testExtendedResource)
89-
err := patchNode(ctx, cs, &node, nodeCopy)
90-
framework.ExpectNoError(err)
86+
e2enode.RemoveExtendedResource(ctx, cs, node.Name, testExtendedResource)
9187
}
9288
})
9389

@@ -134,11 +130,7 @@ var _ = SIGDescribe("SchedulerPreemption", framework.WithSerial(), func() {
134130
// One of them has low priority, making it the victim for preemption.
135131
for i, node := range nodeList.Items {
136132
// Update each node to advertise 3 available extended resources
137-
nodeCopy := node.DeepCopy()
138-
nodeCopy.Status.Capacity[testExtendedResource] = resource.MustParse("5")
139-
nodeCopy.Status.Allocatable[testExtendedResource] = resource.MustParse("5")
140-
err := patchNode(ctx, cs, &node, nodeCopy)
141-
framework.ExpectNoError(err)
133+
e2enode.AddExtendedResource(ctx, cs, node.Name, testExtendedResource, resource.MustParse("5"))
142134

143135
for j := 0; j < 2; j++ {
144136
// Request 2 of the available resources for the victim pods
@@ -225,11 +217,7 @@ var _ = SIGDescribe("SchedulerPreemption", framework.WithSerial(), func() {
225217
pods := make([]*v1.Pod, 0, len(nodeList.Items))
226218
for i, node := range nodeList.Items {
227219
// Update each node to advertise 3 available extended resources
228-
nodeCopy := node.DeepCopy()
229-
nodeCopy.Status.Capacity[testExtendedResource] = resource.MustParse("5")
230-
nodeCopy.Status.Allocatable[testExtendedResource] = resource.MustParse("5")
231-
err := patchNode(ctx, cs, &node, nodeCopy)
232-
framework.ExpectNoError(err)
220+
e2enode.AddExtendedResource(ctx, cs, node.Name, testExtendedResource, resource.MustParse("5"))
233221

234222
for j := 0; j < 2; j++ {
235223
// Request 2 of the available resources for the victim pods
@@ -332,11 +320,7 @@ var _ = SIGDescribe("SchedulerPreemption", framework.WithSerial(), func() {
332320
ginkgo.By("Select a node to run the lower and higher priority pods")
333321
gomega.Expect(nodeList.Items).ToNot(gomega.BeEmpty(), "We need at least one node for the test to run")
334322
node := nodeList.Items[0]
335-
nodeCopy := node.DeepCopy()
336-
nodeCopy.Status.Capacity[testExtendedResource] = resource.MustParse("1")
337-
nodeCopy.Status.Allocatable[testExtendedResource] = resource.MustParse("1")
338-
err := patchNode(ctx, cs, &node, nodeCopy)
339-
framework.ExpectNoError(err)
323+
e2enode.AddExtendedResource(ctx, cs, node.Name, testExtendedResource, resource.MustParse("1"))
340324

341325
// prepare node affinity to make sure both the lower and higher priority pods are scheduled on the same node
342326
testNodeAffinity := v1.Affinity{
@@ -385,7 +369,7 @@ var _ = SIGDescribe("SchedulerPreemption", framework.WithSerial(), func() {
385369
framework.Logf("Created pod: %v", preemptorPod.Name)
386370

387371
ginkgo.By("Waiting for the victim pod to be terminating")
388-
err = e2epod.WaitForPodTerminatingInNamespaceTimeout(ctx, f.ClientSet, victimPod.Name, victimPod.Namespace, framework.PodDeleteTimeout)
372+
err := e2epod.WaitForPodTerminatingInNamespaceTimeout(ctx, f.ClientSet, victimPod.Name, victimPod.Namespace, framework.PodDeleteTimeout)
389373
framework.ExpectNoError(err)
390374

391375
ginkgo.By("Verifying the pod has the pod disruption condition")
@@ -412,11 +396,7 @@ var _ = SIGDescribe("SchedulerPreemption", framework.WithSerial(), func() {
412396
framework.ExpectNoError(err)
413397
// update Node API object with a fake resource
414398
ginkgo.By(fmt.Sprintf("Apply 10 fake resource to node %v.", node.Name))
415-
nodeCopy := node.DeepCopy()
416-
nodeCopy.Status.Capacity[fakeRes] = resource.MustParse("10")
417-
nodeCopy.Status.Allocatable[fakeRes] = resource.MustParse("10")
418-
err = patchNode(ctx, cs, node, nodeCopy)
419-
framework.ExpectNoError(err)
399+
e2enode.AddExtendedResource(ctx, cs, node.Name, fakeRes, resource.MustParse("10"))
420400
nodes = append(nodes, node)
421401
}
422402
})
@@ -425,11 +405,7 @@ var _ = SIGDescribe("SchedulerPreemption", framework.WithSerial(), func() {
425405
e2enode.RemoveLabelOffNode(cs, nodeName, topologyKey)
426406
}
427407
for _, node := range nodes {
428-
nodeCopy := node.DeepCopy()
429-
delete(nodeCopy.Status.Capacity, fakeRes)
430-
delete(nodeCopy.Status.Allocatable, fakeRes)
431-
err := patchNode(ctx, cs, node, nodeCopy)
432-
framework.ExpectNoError(err)
408+
e2enode.RemoveExtendedResource(ctx, cs, node.Name, fakeRes)
433409
}
434410
})
435411

@@ -564,11 +540,7 @@ var _ = SIGDescribe("SchedulerPreemption", framework.WithSerial(), func() {
564540
}
565541

566542
if node != nil {
567-
nodeCopy := node.DeepCopy()
568-
delete(nodeCopy.Status.Capacity, fakecpu)
569-
delete(nodeCopy.Status.Allocatable, fakecpu)
570-
err := patchNode(ctx, cs, node, nodeCopy)
571-
framework.ExpectNoError(err)
543+
e2enode.RemoveExtendedResource(ctx, cs, node.Name, fakecpu)
572544
}
573545
for _, pair := range priorityPairs {
574546
_ = cs.SchedulingV1().PriorityClasses().Delete(ctx, pair.name, *metav1.NewDeleteOptions(0))
@@ -597,11 +569,7 @@ var _ = SIGDescribe("SchedulerPreemption", framework.WithSerial(), func() {
597569
}
598570

599571
// update Node API object with a fake resource
600-
nodeCopy := node.DeepCopy()
601-
nodeCopy.Status.Capacity[fakecpu] = resource.MustParse("1000")
602-
nodeCopy.Status.Allocatable[fakecpu] = resource.MustParse("1000")
603-
err = patchNode(ctx, cs, node, nodeCopy)
604-
framework.ExpectNoError(err)
572+
e2enode.AddExtendedResource(ctx, cs, node.Name, fakecpu, resource.MustParse("1000"))
605573

606574
// create four PriorityClass: p1, p2, p3, p4
607575
for i := 1; i <= 4; i++ {
@@ -920,24 +888,6 @@ func waitForPreemptingWithTimeout(ctx context.Context, f *framework.Framework, p
920888
framework.ExpectNoError(err, "pod %v/%v failed to preempt other pods", pod.Namespace, pod.Name)
921889
}
922890

923-
func patchNode(ctx context.Context, client clientset.Interface, old *v1.Node, new *v1.Node) error {
924-
oldData, err := json.Marshal(old)
925-
if err != nil {
926-
return err
927-
}
928-
929-
newData, err := json.Marshal(new)
930-
if err != nil {
931-
return err
932-
}
933-
patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, &v1.Node{})
934-
if err != nil {
935-
return fmt.Errorf("failed to create merge patch for node %q: %w", old.Name, err)
936-
}
937-
_, err = client.CoreV1().Nodes().Patch(ctx, old.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status")
938-
return err
939-
}
940-
941891
func patchPriorityClass(ctx context.Context, cs clientset.Interface, old, new *schedulingv1.PriorityClass) error {
942892
oldData, err := json.Marshal(old)
943893
if err != nil {

0 commit comments

Comments
 (0)