Skip to content

Commit fc0b16b

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #549 from Gekko0114/integration-test
Coscheduling: integration-test for PodGroup status
2 parents ac60529 + ad751e3 commit fc0b16b

File tree

6 files changed

+377
-4
lines changed

6 files changed

+377
-4
lines changed

‎pkg/controllers/podgroup_controller.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -122,8 +122,7 @@ func (r *PodGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
122122
pgCopy.Status.Phase = schedv1alpha1.PodGroupScheduled
123123
}
124124

125-
if pgCopy.Status.Succeeded+pgCopy.Status.Running >= pg.Spec.MinMember &&
126-
pgCopy.Status.Phase == schedv1alpha1.PodGroupScheduled {
125+
if pgCopy.Status.Succeeded+pgCopy.Status.Running >= pg.Spec.MinMember {
127126
pgCopy.Status.Phase = schedv1alpha1.PodGroupRunning
128127
}
129128
// Final state of pod group

‎test/integration/coscheduling_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ func TestCoschedulingPlugin(t *testing.T) {
103103

104104
// Create a Node.
105105
nodeName := "fake-node"
106-
node := st.MakeNode().Name("fake-node").Label("node", nodeName).Obj()
106+
node := st.MakeNode().Name(nodeName).Label("node", nodeName).Obj()
107107
node.Status.Allocatable = v1.ResourceList{
108108
v1.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI),
109109
v1.ResourceMemory: *resource.NewQuantity(300, resource.DecimalSI),

‎test/integration/elasticquota_controller_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ func TestElasticController(t *testing.T) {
7070
}
7171

7272
go func() {
73-
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
73+
if err := mgr.Start(signalHandler); err != nil {
7474
panic(err)
7575
}
7676
}()

‎test/integration/podgroup_controller_test.go

Lines changed: 359 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,359 @@
1+
/*
2+
Copyright 2023 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package integration
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"testing"
23+
"time"
24+
25+
v1 "k8s.io/api/core/v1"
26+
"k8s.io/apimachinery/pkg/api/resource"
27+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28+
"k8s.io/apimachinery/pkg/util/runtime"
29+
"k8s.io/apimachinery/pkg/util/uuid"
30+
"k8s.io/apimachinery/pkg/util/wait"
31+
"k8s.io/client-go/kubernetes"
32+
"k8s.io/client-go/kubernetes/scheme"
33+
"k8s.io/klog/v2"
34+
"k8s.io/kubernetes/pkg/scheduler"
35+
fwkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime"
36+
st "k8s.io/kubernetes/pkg/scheduler/testing"
37+
ctrl "sigs.k8s.io/controller-runtime"
38+
"sigs.k8s.io/controller-runtime/pkg/manager"
39+
40+
"sigs.k8s.io/scheduler-plugins/apis/scheduling"
41+
"sigs.k8s.io/scheduler-plugins/apis/scheduling/v1alpha1"
42+
"sigs.k8s.io/scheduler-plugins/pkg/controllers"
43+
"sigs.k8s.io/scheduler-plugins/pkg/coscheduling"
44+
"sigs.k8s.io/scheduler-plugins/pkg/generated/clientset/versioned"
45+
"sigs.k8s.io/scheduler-plugins/test/util"
46+
47+
gocmp "github.com/google/go-cmp/cmp"
48+
"github.com/google/go-cmp/cmp/cmpopts"
49+
)
50+
51+
func TestPodGroupController(t *testing.T) {
52+
testCtx := &testContext{}
53+
testCtx.Ctx, testCtx.CancelFn = context.WithCancel(context.Background())
54+
55+
cs := kubernetes.NewForConfigOrDie(globalKubeConfig)
56+
extClient := versioned.NewForConfigOrDie(globalKubeConfig)
57+
testCtx.ClientSet = cs
58+
testCtx.KubeConfig = globalKubeConfig
59+
60+
s := scheme.Scheme
61+
62+
runtime.Must(v1alpha1.AddToScheme(s))
63+
64+
mgrOpts := manager.Options{
65+
Scheme: s,
66+
MetricsBindAddress: "0", // disable metrics to avoid conflicts between packages.
67+
}
68+
69+
mgr, _ := ctrl.NewManager(globalKubeConfig, mgrOpts)
70+
if err := (&controllers.PodGroupReconciler{
71+
Client: mgr.GetClient(),
72+
Scheme: mgr.GetScheme(),
73+
}).SetupWithManager(mgr); err != nil {
74+
t.Fatal("unable to create controller", "controller", "PodGroup", err)
75+
}
76+
77+
go func() {
78+
if err := mgr.Start(signalHandler); err != nil {
79+
panic(err)
80+
}
81+
}()
82+
83+
if err := wait.Poll(100*time.Millisecond, 3*time.Second, func() (done bool, err error) {
84+
groupList, _, err := cs.ServerGroupsAndResources()
85+
if err != nil {
86+
return false, nil
87+
}
88+
for _, group := range groupList {
89+
if group.Name == scheduling.GroupName {
90+
t.Log("The CRD is ready to serve")
91+
return true, nil
92+
}
93+
}
94+
return false, nil
95+
}); err != nil {
96+
t.Fatalf("Timed out waiting for CRD to be ready: %v", err)
97+
}
98+
99+
ns := fmt.Sprintf("integration-test-%v", string(uuid.NewUUID()))
100+
createNamespace(t, testCtx, ns)
101+
102+
testCtx = initTestSchedulerWithOptions(
103+
t,
104+
testCtx,
105+
scheduler.WithFrameworkOutOfTreeRegistry(fwkruntime.Registry{coscheduling.Name: coscheduling.New}),
106+
)
107+
syncInformerFactory(testCtx)
108+
go testCtx.Scheduler.Run(testCtx.Ctx)
109+
t.Log("Init scheduler success")
110+
defer cleanupTest(t, testCtx)
111+
112+
// Create a Node.
113+
nodeName := "fake-node"
114+
node := st.MakeNode().Name(nodeName).Label("node", nodeName).Obj()
115+
node.Status.Allocatable = v1.ResourceList{
116+
v1.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI),
117+
v1.ResourceMemory: *resource.NewQuantity(300, resource.DecimalSI),
118+
}
119+
node.Status.Capacity = v1.ResourceList{
120+
v1.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI),
121+
v1.ResourceMemory: *resource.NewQuantity(300, resource.DecimalSI),
122+
}
123+
if _, err := cs.CoreV1().Nodes().Create(testCtx.Ctx, node, metav1.CreateOptions{}); err != nil {
124+
t.Fatalf("Failed to create Node %q: %v", nodeName, err)
125+
}
126+
ignoreOpts := cmpopts.IgnoreFields(v1alpha1.PodGroupStatus{}, "ScheduleStartTime")
127+
// TODO: Update the number of scheduled pods when changing the Reconcile logic.
128+
// PostBind is not running in this test, so the number of Scheduled pods in PodGroup is 0.
129+
for _, tt := range []struct {
130+
name string
131+
podGroups []*v1alpha1.PodGroup
132+
existingPods []*v1.Pod
133+
intermediatePGState []*v1alpha1.PodGroup
134+
incomingPods []*v1.Pod
135+
expectedPGState []*v1alpha1.PodGroup
136+
}{
137+
{
138+
name: "Statuses of all pods change from pending to running",
139+
podGroups: []*v1alpha1.PodGroup{
140+
util.MakePG("pg1-1", ns, 3, nil, nil),
141+
},
142+
existingPods: []*v1.Pod{
143+
st.MakePod().Namespace(ns).Name("t1-p1-1").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
144+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Obj(),
145+
st.MakePod().Namespace(ns).Name("t1-p1-2").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
146+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Obj(),
147+
st.MakePod().Namespace(ns).Name("t1-p1-3").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
148+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Obj(),
149+
},
150+
intermediatePGState: []*v1alpha1.PodGroup{
151+
util.UpdatePGStatus(util.MakePG("pg1-1", ns, 3, nil, nil), "PreScheduling", "", 0, 0, 0, 0),
152+
},
153+
incomingPods: []*v1.Pod{
154+
st.MakePod().Namespace(ns).Name("t1-p1-1").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
155+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Phase(v1.PodRunning).Obj(),
156+
st.MakePod().Namespace(ns).Name("t1-p1-2").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
157+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Phase(v1.PodRunning).Obj(),
158+
st.MakePod().Namespace(ns).Name("t1-p1-3").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
159+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Phase(v1.PodRunning).Obj(),
160+
},
161+
expectedPGState: []*v1alpha1.PodGroup{
162+
util.UpdatePGStatus(util.MakePG("pg1-1", ns, 3, nil, nil), "Running", "", 0, 3, 0, 0),
163+
},
164+
},
165+
{
166+
name: "Statuses of all pods change from running to succeeded",
167+
podGroups: []*v1alpha1.PodGroup{
168+
util.MakePG("pg1-1", ns, 3, nil, nil),
169+
},
170+
existingPods: []*v1.Pod{
171+
st.MakePod().Namespace(ns).Name("t1-p1-1").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
172+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Phase(v1.PodRunning).Obj(),
173+
st.MakePod().Namespace(ns).Name("t1-p1-2").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
174+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Phase(v1.PodRunning).Obj(),
175+
st.MakePod().Namespace(ns).Name("t1-p1-3").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
176+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Phase(v1.PodRunning).Obj(),
177+
},
178+
intermediatePGState: []*v1alpha1.PodGroup{
179+
util.UpdatePGStatus(util.MakePG("pg1-1", ns, 3, nil, nil), "Running", "", 0, 3, 0, 0),
180+
},
181+
incomingPods: []*v1.Pod{
182+
st.MakePod().Namespace(ns).Name("t1-p1-1").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
183+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Phase(v1.PodSucceeded).Obj(),
184+
st.MakePod().Namespace(ns).Name("t1-p1-2").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
185+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Phase(v1.PodSucceeded).Obj(),
186+
st.MakePod().Namespace(ns).Name("t1-p1-3").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
187+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Phase(v1.PodSucceeded).Obj(),
188+
},
189+
expectedPGState: []*v1alpha1.PodGroup{
190+
util.UpdatePGStatus(util.MakePG("pg1-1", ns, 3, nil, nil), "Finished", "", 0, 0, 3, 0),
191+
},
192+
},
193+
{
194+
name: "The status of one pod changes from running to succeeded",
195+
podGroups: []*v1alpha1.PodGroup{
196+
util.MakePG("pg1-1", ns, 3, nil, nil),
197+
},
198+
existingPods: []*v1.Pod{
199+
st.MakePod().Namespace(ns).Name("t1-p1-1").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
200+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Phase(v1.PodRunning).Obj(),
201+
st.MakePod().Namespace(ns).Name("t1-p1-2").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
202+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Phase(v1.PodRunning).Obj(),
203+
st.MakePod().Namespace(ns).Name("t1-p1-3").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
204+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Phase(v1.PodRunning).Obj(),
205+
},
206+
intermediatePGState: []*v1alpha1.PodGroup{
207+
util.UpdatePGStatus(util.MakePG("pg1-1", ns, 3, nil, nil), "Running", "", 0, 3, 0, 0),
208+
},
209+
incomingPods: []*v1.Pod{
210+
st.MakePod().Namespace(ns).Name("t1-p1-1").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
211+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Phase(v1.PodSucceeded).Obj(),
212+
},
213+
expectedPGState: []*v1alpha1.PodGroup{
214+
util.UpdatePGStatus(util.MakePG("pg1-1", ns, 3, nil, nil), "Running", "", 0, 2, 1, 0),
215+
},
216+
},
217+
{
218+
name: "The status of the pod changes from running to failed",
219+
podGroups: []*v1alpha1.PodGroup{
220+
util.MakePG("pg1-1", ns, 3, nil, nil),
221+
},
222+
existingPods: []*v1.Pod{
223+
st.MakePod().Namespace(ns).Name("t1-p1-1").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
224+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Phase(v1.PodRunning).Obj(),
225+
st.MakePod().Namespace(ns).Name("t1-p1-2").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
226+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Phase(v1.PodRunning).Obj(),
227+
st.MakePod().Namespace(ns).Name("t1-p1-3").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
228+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Phase(v1.PodRunning).Obj(),
229+
},
230+
intermediatePGState: []*v1alpha1.PodGroup{
231+
util.UpdatePGStatus(util.MakePG("pg1-1", ns, 3, nil, nil), "Running", "", 0, 3, 0, 0),
232+
},
233+
incomingPods: []*v1.Pod{
234+
st.MakePod().Namespace(ns).Name("t1-p1-1").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
235+
midPriority).Label(v1alpha1.PodGroupLabel, "pg1-1").Node(nodeName).Phase(v1.PodFailed).Obj(),
236+
},
237+
expectedPGState: []*v1alpha1.PodGroup{
238+
util.UpdatePGStatus(util.MakePG("pg1-1", ns, 3, nil, nil), "Failed", "", 0, 2, 0, 1),
239+
},
240+
},
241+
{
242+
name: "The status of no podGroup label pod changes from pending to running",
243+
podGroups: []*v1alpha1.PodGroup{
244+
util.MakePG("pg1-1", ns, 3, nil, nil),
245+
},
246+
existingPods: []*v1.Pod{
247+
st.MakePod().Namespace(ns).Name("t1-p1-1").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
248+
midPriority).Node(nodeName).Obj(),
249+
st.MakePod().Namespace(ns).Name("t1-p1-2").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
250+
midPriority).Node(nodeName).Obj(),
251+
st.MakePod().Namespace(ns).Name("t1-p1-3").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
252+
midPriority).Node(nodeName).Obj(),
253+
},
254+
intermediatePGState: []*v1alpha1.PodGroup{
255+
util.UpdatePGStatus(util.MakePG("pg1-1", ns, 3, nil, nil), "Pending", "", 0, 0, 0, 0),
256+
},
257+
incomingPods: []*v1.Pod{
258+
st.MakePod().Namespace(ns).Name("t1-p1-1").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
259+
midPriority).Node(nodeName).Phase(v1.PodRunning).Obj(),
260+
st.MakePod().Namespace(ns).Name("t1-p1-2").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
261+
midPriority).Node(nodeName).Phase(v1.PodRunning).Obj(),
262+
st.MakePod().Namespace(ns).Name("t1-p1-3").Req(map[v1.ResourceName]string{v1.ResourceMemory: "50"}).Priority(
263+
midPriority).Node(nodeName).Phase(v1.PodRunning).Obj(),
264+
},
265+
expectedPGState: []*v1alpha1.PodGroup{
266+
util.UpdatePGStatus(util.MakePG("pg1-1", ns, 3, nil, nil), "Pending", "", 0, 0, 0, 0),
267+
},
268+
},
269+
} {
270+
t.Run(tt.name, func(t *testing.T) {
271+
defer cleanupPodGroups(testCtx.Ctx, extClient, tt.podGroups)
272+
defer cleanupPods(t, testCtx, tt.existingPods)
273+
defer cleanupPods(t, testCtx, tt.incomingPods)
274+
// create pod group
275+
if err := createPodGroups(testCtx.Ctx, extClient, tt.podGroups); err != nil {
276+
t.Fatal(err)
277+
}
278+
279+
// create Pods
280+
for _, pod := range tt.existingPods {
281+
klog.InfoS("Creating pod ", "podName", pod.Name)
282+
if _, err := cs.CoreV1().Pods(pod.Namespace).Create(testCtx.Ctx, pod, metav1.CreateOptions{}); err != nil {
283+
t.Fatalf("Failed to create Pod %q: %v", pod.Name, err)
284+
}
285+
if pod.Status.Phase == v1.PodRunning {
286+
if _, err := cs.CoreV1().Pods(pod.Namespace).UpdateStatus(testCtx.Ctx, pod, metav1.UpdateOptions{}); err != nil {
287+
t.Fatalf("Failed to update Pod status %q: %v", pod.Name, err)
288+
}
289+
}
290+
}
291+
if err := wait.Poll(time.Millisecond*200, 10*time.Second, func() (bool, error) {
292+
for _, pod := range tt.incomingPods {
293+
if !podScheduled(cs, ns, pod.Name) {
294+
return false, nil
295+
}
296+
}
297+
return true, nil
298+
}); err != nil {
299+
t.Fatalf("%v Waiting existPods create error: %v", tt.name, err.Error())
300+
}
301+
302+
if err := wait.Poll(time.Millisecond*200, 10*time.Second, func() (bool, error) {
303+
for _, v := range tt.intermediatePGState {
304+
pg, err := extClient.SchedulingV1alpha1().PodGroups(v.Namespace).Get(testCtx.Ctx, v.Name, metav1.GetOptions{})
305+
if err != nil {
306+
// This could be a connection error so we want to retry.
307+
klog.ErrorS(err, "Failed to obtain the PodGroup clientSet")
308+
return false, err
309+
}
310+
if diff := gocmp.Diff(pg.Status, v.Status, ignoreOpts); diff != "" {
311+
t.Error(diff)
312+
return false, nil
313+
}
314+
}
315+
return true, nil
316+
}); err != nil {
317+
t.Fatalf("%v Waiting now PodGroup Status error: %v", tt.name, err.Error())
318+
}
319+
320+
// update Pods status to check if PodGroup.Status has changed as expected
321+
for _, pod := range tt.incomingPods {
322+
if _, err := cs.CoreV1().Pods(pod.Namespace).UpdateStatus(testCtx.Ctx, pod, metav1.UpdateOptions{}); err != nil {
323+
t.Fatalf("Failed to update Pod status %q: %v", pod.Name, err)
324+
}
325+
}
326+
// wait for all incomingPods to be scheduled
327+
if err := wait.Poll(time.Millisecond*200, 10*time.Second, func() (bool, error) {
328+
for _, pod := range tt.incomingPods {
329+
if !podScheduled(cs, pod.Namespace, pod.Name) {
330+
return false, nil
331+
}
332+
}
333+
return true, nil
334+
}); err != nil {
335+
t.Fatalf("%v Waiting incomingPods scheduled error: %v", tt.name, err.Error())
336+
}
337+
338+
if err := wait.Poll(time.Millisecond*200, 10*time.Second, func() (bool, error) {
339+
for _, v := range tt.expectedPGState {
340+
pg, err := extClient.SchedulingV1alpha1().PodGroups(v.Namespace).Get(testCtx.Ctx, v.Name, metav1.GetOptions{})
341+
if err != nil {
342+
// This could be a connection error so we want to retry.
343+
klog.ErrorS(err, "Failed to obtain the PodGroup clientSet")
344+
return false, err
345+
}
346+
347+
if diff := gocmp.Diff(pg.Status, v.Status, ignoreOpts); diff != "" {
348+
t.Error(diff)
349+
return false, nil
350+
}
351+
}
352+
return true, nil
353+
}); err != nil {
354+
t.Fatalf("%v Waiting PodGroup status update error: %v", tt.name, err.Error())
355+
}
356+
t.Logf("Case %v finished", tt.name)
357+
})
358+
}
359+
}

0 commit comments

Comments
 (0)