Skip to content

Commit 833ee85

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#128194 from AnishShah/extended-resource
test: refactor logic to add/remove extended resources
2 parents 0fad789 + 6191879 commit 833ee85

File tree

3 files changed

+47
-136
lines changed

3 files changed

+47
-136
lines changed

‎test/e2e/common/node/pod_resize.go‎

Lines changed: 2 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,13 @@ package node
1818

1919
import (
2020
"context"
21-
"encoding/json"
2221
"fmt"
2322
"strconv"
2423
"time"
2524

2625
"k8s.io/apimachinery/pkg/api/resource"
2726
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2827
"k8s.io/apimachinery/pkg/types"
29-
"k8s.io/apimachinery/pkg/util/strategicpatch"
30-
clientset "k8s.io/client-go/kubernetes"
3128
"k8s.io/kubernetes/test/e2e/feature"
3229
"k8s.io/kubernetes/test/e2e/framework"
3330
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
@@ -43,78 +40,6 @@ const (
4340
fakeExtendedResource = "dummy.com/dummy"
4441
)
4542

46-
func patchNode(ctx context.Context, client clientset.Interface, old *v1.Node, new *v1.Node) error {
47-
oldData, err := json.Marshal(old)
48-
if err != nil {
49-
return err
50-
}
51-
52-
newData, err := json.Marshal(new)
53-
if err != nil {
54-
return err
55-
}
56-
patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, &v1.Node{})
57-
if err != nil {
58-
return fmt.Errorf("failed to create merge patch for node %q: %w", old.Name, err)
59-
}
60-
_, err = client.CoreV1().Nodes().Patch(ctx, old.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status")
61-
return err
62-
}
63-
64-
func addExtendedResource(clientSet clientset.Interface, nodeName, extendedResourceName string, extendedResourceQuantity resource.Quantity) {
65-
extendedResource := v1.ResourceName(extendedResourceName)
66-
67-
ginkgo.By("Adding a custom resource")
68-
OriginalNode, err := clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{})
69-
framework.ExpectNoError(err)
70-
71-
node := OriginalNode.DeepCopy()
72-
node.Status.Capacity[extendedResource] = extendedResourceQuantity
73-
node.Status.Allocatable[extendedResource] = extendedResourceQuantity
74-
err = patchNode(context.Background(), clientSet, OriginalNode.DeepCopy(), node)
75-
framework.ExpectNoError(err)
76-
77-
gomega.Eventually(func() error {
78-
node, err = clientSet.CoreV1().Nodes().Get(context.Background(), node.Name, metav1.GetOptions{})
79-
framework.ExpectNoError(err)
80-
81-
fakeResourceCapacity, exists := node.Status.Capacity[extendedResource]
82-
if !exists {
83-
return fmt.Errorf("node %s has no %s resource capacity", node.Name, extendedResourceName)
84-
}
85-
if expectedResource := resource.MustParse("123"); fakeResourceCapacity.Cmp(expectedResource) != 0 {
86-
return fmt.Errorf("node %s has resource capacity %s, expected: %s", node.Name, fakeResourceCapacity.String(), expectedResource.String())
87-
}
88-
89-
return nil
90-
}).WithTimeout(30 * time.Second).WithPolling(time.Second).ShouldNot(gomega.HaveOccurred())
91-
}
92-
93-
func removeExtendedResource(clientSet clientset.Interface, nodeName, extendedResourceName string) {
94-
extendedResource := v1.ResourceName(extendedResourceName)
95-
96-
ginkgo.By("Removing a custom resource")
97-
originalNode, err := clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{})
98-
framework.ExpectNoError(err)
99-
100-
node := originalNode.DeepCopy()
101-
delete(node.Status.Capacity, extendedResource)
102-
delete(node.Status.Allocatable, extendedResource)
103-
err = patchNode(context.Background(), clientSet, originalNode.DeepCopy(), node)
104-
framework.ExpectNoError(err)
105-
106-
gomega.Eventually(func() error {
107-
node, err = clientSet.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{})
108-
framework.ExpectNoError(err)
109-
110-
if _, exists := node.Status.Capacity[extendedResource]; exists {
111-
return fmt.Errorf("node %s has resource capacity %s which is expected to be removed", node.Name, extendedResourceName)
112-
}
113-
114-
return nil
115-
}).WithTimeout(30 * time.Second).WithPolling(time.Second).ShouldNot(gomega.HaveOccurred())
116-
}
117-
11843
func doPodResizeTests(f *framework.Framework) {
11944
type testCase struct {
12045
name string
@@ -879,11 +804,11 @@ func doPodResizeTests(f *framework.Framework) {
879804
framework.ExpectNoError(err)
880805

881806
for _, node := range nodes.Items {
882-
addExtendedResource(f.ClientSet, node.Name, fakeExtendedResource, resource.MustParse("123"))
807+
e2enode.AddExtendedResource(ctx, f.ClientSet, node.Name, fakeExtendedResource, resource.MustParse("123"))
883808
}
884809
defer func() {
885810
for _, node := range nodes.Items {
886-
removeExtendedResource(f.ClientSet, node.Name, fakeExtendedResource)
811+
e2enode.RemoveExtendedResource(ctx, f.ClientSet, node.Name, fakeExtendedResource)
887812
}
888813
}()
889814
}

‎test/e2e/framework/node/helper.go‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,16 @@ package node
1818

1919
import (
2020
"context"
21+
"encoding/json"
2122
"fmt"
2223
"time"
2324

2425
"github.com/onsi/ginkgo/v2"
2526
"github.com/onsi/gomega"
2627
v1 "k8s.io/api/core/v1"
28+
"k8s.io/apimachinery/pkg/api/resource"
2729
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
30+
"k8s.io/apimachinery/pkg/types"
2831
"k8s.io/apimachinery/pkg/util/wait"
2932
clientset "k8s.io/client-go/kubernetes"
3033

@@ -175,3 +178,36 @@ func IsARM64(node *v1.Node) bool {
175178

176179
return false
177180
}
181+
182+
// AddExtendedResource adds a fake resource to the Node.
183+
func AddExtendedResource(ctx context.Context, clientSet clientset.Interface, nodeName string, extendedResourceName v1.ResourceName, extendedResourceQuantity resource.Quantity) {
184+
extendedResource := v1.ResourceName(extendedResourceName)
185+
186+
ginkgo.By("Adding a custom resource")
187+
extendedResourceList := v1.ResourceList{
188+
extendedResource: extendedResourceQuantity,
189+
}
190+
patchPayload, err := json.Marshal(v1.Node{
191+
Status: v1.NodeStatus{
192+
Capacity: extendedResourceList,
193+
Allocatable: extendedResourceList,
194+
},
195+
})
196+
framework.ExpectNoError(err, "Failed to marshal node JSON")
197+
198+
_, err = clientSet.CoreV1().Nodes().Patch(ctx, nodeName, types.StrategicMergePatchType, []byte(patchPayload), metav1.PatchOptions{}, "status")
199+
framework.ExpectNoError(err)
200+
}
201+
202+
// RemoveExtendedResource removes a fake resource from the Node.
203+
func RemoveExtendedResource(ctx context.Context, clientSet clientset.Interface, nodeName string, extendedResourceName v1.ResourceName) {
204+
extendedResource := v1.ResourceName(extendedResourceName)
205+
206+
ginkgo.By("Removing a custom resource")
207+
node, err := clientSet.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
208+
framework.ExpectNoError(err)
209+
delete(node.Status.Capacity, extendedResource)
210+
delete(node.Status.Allocatable, extendedResource)
211+
_, err = clientSet.CoreV1().Nodes().UpdateStatus(ctx, node, metav1.UpdateOptions{})
212+
framework.ExpectNoError(err)
213+
}

‎test/e2e/scheduling/preemption.go‎

Lines changed: 9 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -83,11 +83,7 @@ var _ = SIGDescribe("SchedulerPreemption", framework.WithSerial(), func() {
8383
_ = cs.SchedulingV1().PriorityClasses().Delete(ctx, pair.name, *metav1.NewDeleteOptions(0))
8484
}
8585
for _, node := range nodeList.Items {
86-
nodeCopy := node.DeepCopy()
87-
delete(nodeCopy.Status.Capacity, testExtendedResource)
88-
delete(nodeCopy.Status.Allocatable, testExtendedResource)
89-
err := patchNode(ctx, cs, &node, nodeCopy)
90-
framework.ExpectNoError(err)
86+
e2enode.RemoveExtendedResource(ctx, cs, node.Name, testExtendedResource)
9187
}
9288
})
9389

@@ -134,11 +130,7 @@ var _ = SIGDescribe("SchedulerPreemption", framework.WithSerial(), func() {
134130
// One of them has low priority, making it the victim for preemption.
135131
for i, node := range nodeList.Items {
136132
// Update each node to advertise 3 available extended resources
137-
nodeCopy := node.DeepCopy()
138-
nodeCopy.Status.Capacity[testExtendedResource] = resource.MustParse("5")
139-
nodeCopy.Status.Allocatable[testExtendedResource] = resource.MustParse("5")
140-
err := patchNode(ctx, cs, &node, nodeCopy)
141-
framework.ExpectNoError(err)
133+
e2enode.AddExtendedResource(ctx, cs, node.Name, testExtendedResource, resource.MustParse("5"))
142134

143135
for j := 0; j < 2; j++ {
144136
// Request 2 of the available resources for the victim pods
@@ -225,11 +217,7 @@ var _ = SIGDescribe("SchedulerPreemption", framework.WithSerial(), func() {
225217
pods := make([]*v1.Pod, 0, len(nodeList.Items))
226218
for i, node := range nodeList.Items {
227219
// Update each node to advertise 3 available extended resources
228-
nodeCopy := node.DeepCopy()
229-
nodeCopy.Status.Capacity[testExtendedResource] = resource.MustParse("5")
230-
nodeCopy.Status.Allocatable[testExtendedResource] = resource.MustParse("5")
231-
err := patchNode(ctx, cs, &node, nodeCopy)
232-
framework.ExpectNoError(err)
220+
e2enode.AddExtendedResource(ctx, cs, node.Name, testExtendedResource, resource.MustParse("5"))
233221

234222
for j := 0; j < 2; j++ {
235223
// Request 2 of the available resources for the victim pods
@@ -332,11 +320,7 @@ var _ = SIGDescribe("SchedulerPreemption", framework.WithSerial(), func() {
332320
ginkgo.By("Select a node to run the lower and higher priority pods")
333321
gomega.Expect(nodeList.Items).ToNot(gomega.BeEmpty(), "We need at least one node for the test to run")
334322
node := nodeList.Items[0]
335-
nodeCopy := node.DeepCopy()
336-
nodeCopy.Status.Capacity[testExtendedResource] = resource.MustParse("1")
337-
nodeCopy.Status.Allocatable[testExtendedResource] = resource.MustParse("1")
338-
err := patchNode(ctx, cs, &node, nodeCopy)
339-
framework.ExpectNoError(err)
323+
e2enode.AddExtendedResource(ctx, cs, node.Name, testExtendedResource, resource.MustParse("1"))
340324

341325
// prepare node affinity to make sure both the lower and higher priority pods are scheduled on the same node
342326
testNodeAffinity := v1.Affinity{
@@ -385,7 +369,7 @@ var _ = SIGDescribe("SchedulerPreemption", framework.WithSerial(), func() {
385369
framework.Logf("Created pod: %v", preemptorPod.Name)
386370

387371
ginkgo.By("Waiting for the victim pod to be terminating")
388-
err = e2epod.WaitForPodTerminatingInNamespaceTimeout(ctx, f.ClientSet, victimPod.Name, victimPod.Namespace, framework.PodDeleteTimeout)
372+
err := e2epod.WaitForPodTerminatingInNamespaceTimeout(ctx, f.ClientSet, victimPod.Name, victimPod.Namespace, framework.PodDeleteTimeout)
389373
framework.ExpectNoError(err)
390374

391375
ginkgo.By("Verifying the pod has the pod disruption condition")
@@ -412,11 +396,7 @@ var _ = SIGDescribe("SchedulerPreemption", framework.WithSerial(), func() {
412396
framework.ExpectNoError(err)
413397
// update Node API object with a fake resource
414398
ginkgo.By(fmt.Sprintf("Apply 10 fake resource to node %v.", node.Name))
415-
nodeCopy := node.DeepCopy()
416-
nodeCopy.Status.Capacity[fakeRes] = resource.MustParse("10")
417-
nodeCopy.Status.Allocatable[fakeRes] = resource.MustParse("10")
418-
err = patchNode(ctx, cs, node, nodeCopy)
419-
framework.ExpectNoError(err)
399+
e2enode.AddExtendedResource(ctx, cs, node.Name, fakeRes, resource.MustParse("10"))
420400
nodes = append(nodes, node)
421401
}
422402
})
@@ -425,11 +405,7 @@ var _ = SIGDescribe("SchedulerPreemption", framework.WithSerial(), func() {
425405
e2enode.RemoveLabelOffNode(cs, nodeName, topologyKey)
426406
}
427407
for _, node := range nodes {
428-
nodeCopy := node.DeepCopy()
429-
delete(nodeCopy.Status.Capacity, fakeRes)
430-
delete(nodeCopy.Status.Allocatable, fakeRes)
431-
err := patchNode(ctx, cs, node, nodeCopy)
432-
framework.ExpectNoError(err)
408+
e2enode.RemoveExtendedResource(ctx, cs, node.Name, fakeRes)
433409
}
434410
})
435411

@@ -564,11 +540,7 @@ var _ = SIGDescribe("SchedulerPreemption", framework.WithSerial(), func() {
564540
}
565541

566542
if node != nil {
567-
nodeCopy := node.DeepCopy()
568-
delete(nodeCopy.Status.Capacity, fakecpu)
569-
delete(nodeCopy.Status.Allocatable, fakecpu)
570-
err := patchNode(ctx, cs, node, nodeCopy)
571-
framework.ExpectNoError(err)
543+
e2enode.RemoveExtendedResource(ctx, cs, node.Name, fakecpu)
572544
}
573545
for _, pair := range priorityPairs {
574546
_ = cs.SchedulingV1().PriorityClasses().Delete(ctx, pair.name, *metav1.NewDeleteOptions(0))
@@ -597,11 +569,7 @@ var _ = SIGDescribe("SchedulerPreemption", framework.WithSerial(), func() {
597569
}
598570

599571
// update Node API object with a fake resource
600-
nodeCopy := node.DeepCopy()
601-
nodeCopy.Status.Capacity[fakecpu] = resource.MustParse("1000")
602-
nodeCopy.Status.Allocatable[fakecpu] = resource.MustParse("1000")
603-
err = patchNode(ctx, cs, node, nodeCopy)
604-
framework.ExpectNoError(err)
572+
e2enode.AddExtendedResource(ctx, cs, node.Name, fakecpu, resource.MustParse("1000"))
605573

606574
// create four PriorityClass: p1, p2, p3, p4
607575
for i := 1; i <= 4; i++ {
@@ -920,24 +888,6 @@ func waitForPreemptingWithTimeout(ctx context.Context, f *framework.Framework, p
920888
framework.ExpectNoError(err, "pod %v/%v failed to preempt other pods", pod.Namespace, pod.Name)
921889
}
922890

923-
func patchNode(ctx context.Context, client clientset.Interface, old *v1.Node, new *v1.Node) error {
924-
oldData, err := json.Marshal(old)
925-
if err != nil {
926-
return err
927-
}
928-
929-
newData, err := json.Marshal(new)
930-
if err != nil {
931-
return err
932-
}
933-
patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, &v1.Node{})
934-
if err != nil {
935-
return fmt.Errorf("failed to create merge patch for node %q: %w", old.Name, err)
936-
}
937-
_, err = client.CoreV1().Nodes().Patch(ctx, old.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status")
938-
return err
939-
}
940-
941891
func patchPriorityClass(ctx context.Context, cs clientset.Interface, old, new *schedulingv1.PriorityClass) error {
942892
oldData, err := json.Marshal(old)
943893
if err != nil {

0 commit comments

Comments
 (0)