Add integration tests for DiscardReservedNodes NRT cache

Signed-off-by: pprokop <[email protected]>

Author: PiotrProkop

test/integration/noderesourcetopology_cache_test.go

@@ -35,6 +35,8 @@ import (
 	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/pkg/scheduler"
 	schedapi "k8s.io/kubernetes/pkg/scheduler/apis/config"
+	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder"
+	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort"
 	fwkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime"
 	st "k8s.io/kubernetes/pkg/scheduler/testing"
 
@@ -50,24 +52,26 @@ import (
 const (
 	defaultCacheResyncPeriodSeconds int64 = 5
 	anyNode                               = "*"
+	discardReservedSchedulerName          = "discardReserved"
 )
 
 var (
 	schedVerbose = "0"
 )
 
 type podDesc struct {
-	podName      string
-	isGuaranteed bool
-	isDelete     bool
-	resourcesMap map[string]string
-	expectedNode string
+	schedulerName string
+	podName       string
+	isGuaranteed  bool
+	isDelete      bool
+	resourcesMap  map[string]string
+	expectedNode  string
 	// autogenerated
 	pod *corev1.Pod
 }
 
 func (p *podDesc) SetupPod(ns string, initContainer bool) {
-	pt := st.MakePod().Namespace(ns).Name(p.podName)
+	pt := st.MakePod().Namespace(ns).Name(p.podName).SchedulerName(p.schedulerName)
 	if p.isGuaranteed {
 		p.pod = util.WithLimits(pt, p.resourcesMap, initContainer).Obj()
 	} else {
@@ -264,6 +268,109 @@ func TestTopologyCachePluginWithoutUpdates(t *testing.T) {
 						}).Obj(),
 			},
 		},
+		{
+			name: "GU pod: DiscardReservedNodes: allows scheduling on both Zones",
+			podDescs: []podDesc{
+				{
+					podName:      "nrt-pod-1000",
+					isGuaranteed: true,
+					resourcesMap: map[string]string{
+						string(corev1.ResourceCPU):    "16",
+						string(corev1.ResourceMemory): "24Gi",
+					},
+					expectedNode:  "fake-node-cache-1",
+					schedulerName: discardReservedSchedulerName,
+				},
+				{
+					podName:      "nrt-pod-2000",
+					isGuaranteed: true,
+					resourcesMap: map[string]string{
+						string(corev1.ResourceCPU):    "16",
+						string(corev1.ResourceMemory): "24Gi",
+					},
+					schedulerName: discardReservedSchedulerName,
+					expectedNode:  "fake-node-cache-1",
+				},
+			},
+			nodeResourceTopologies: []*topologyv1alpha2.NodeResourceTopology{
+				MakeNRT().Name("fake-node-cache-1").Policy(topologyv1alpha2.SingleNUMANodeContainerLevel).
+					Zone(
+						topologyv1alpha2.ResourceInfoList{
+							noderesourcetopology.MakeTopologyResInfo(cpu, "32", "30"),
+							noderesourcetopology.MakeTopologyResInfo(memory, "64Gi", "60Gi"),
+						}).
+					Zone(
+						topologyv1alpha2.ResourceInfoList{
+							noderesourcetopology.MakeTopologyResInfo(cpu, "32", "30"),
+							noderesourcetopology.MakeTopologyResInfo(memory, "64Gi", "62Gi"),
+						}).Obj(),
+				MakeNRT().Name("fake-node-cache-2").Policy(topologyv1alpha2.SingleNUMANodeContainerLevel).
+					Zone(
+						topologyv1alpha2.ResourceInfoList{
+							noderesourcetopology.MakeTopologyResInfo(cpu, "32", "10"),
+							noderesourcetopology.MakeTopologyResInfo(memory, "64Gi", "14Gi"),
+						}).
+					Zone(
+						topologyv1alpha2.ResourceInfoList{
+							noderesourcetopology.MakeTopologyResInfo(cpu, "32", "8"),
+							noderesourcetopology.MakeTopologyResInfo(memory, "64Gi", "10Gi"),
+						}).Obj(),
+			},
+		},
+		{
+			name: "GU pod: DiscardReservedNodes: new pod is successfully scheduled on the node, after deleting pod consuming most resources",
+			podDescs: []podDesc{
+				{
+					podName:      "nrt-pod-3000",
+					isGuaranteed: true,
+					resourcesMap: map[string]string{
+						string(corev1.ResourceCPU):    "30",
+						string(corev1.ResourceMemory): "60Gi",
+					},
+					schedulerName: discardReservedSchedulerName,
+					expectedNode:  "fake-node-cache-1",
+				},
+				{
+					podName:       "nrt-pod-3000",
+					isDelete:      true,
+					schedulerName: "discardReserved",
+				},
+				{
+					podName:      "nrt-pod-4000",
+					isGuaranteed: true,
+					resourcesMap: map[string]string{
+						string(corev1.ResourceCPU):    "16",
+						string(corev1.ResourceMemory): "24Gi",
+					},
+					schedulerName: discardReservedSchedulerName,
+					expectedNode:  "fake-node-cache-1",
+				},
+			},
+			nodeResourceTopologies: []*topologyv1alpha2.NodeResourceTopology{
+				MakeNRT().Name("fake-node-cache-1").Policy(topologyv1alpha2.SingleNUMANodeContainerLevel).
+					Zone(
+						topologyv1alpha2.ResourceInfoList{
+							noderesourcetopology.MakeTopologyResInfo(cpu, "32", "30"),
+							noderesourcetopology.MakeTopologyResInfo(memory, "64Gi", "60Gi"),
+						}).
+					Zone(
+						topologyv1alpha2.ResourceInfoList{
+							noderesourcetopology.MakeTopologyResInfo(cpu, "32", "30"),
+							noderesourcetopology.MakeTopologyResInfo(memory, "64Gi", "62Gi"),
+						}).Obj(),
+				MakeNRT().Name("fake-node-cache-2").Policy(topologyv1alpha2.SingleNUMANodeContainerLevel).
+					Zone(
+						topologyv1alpha2.ResourceInfoList{
+							noderesourcetopology.MakeTopologyResInfo(cpu, "32", "10"),
+							noderesourcetopology.MakeTopologyResInfo(memory, "64Gi", "14Gi"),
+						}).
+					Zone(
+						topologyv1alpha2.ResourceInfoList{
+							noderesourcetopology.MakeTopologyResInfo(cpu, "32", "8"),
+							noderesourcetopology.MakeTopologyResInfo(memory, "64Gi", "10Gi"),
+						}).Obj(),
+			},
+		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
 			// because caching, each testcase needs to run from a clean slate
@@ -297,6 +404,8 @@ func TestTopologyCachePluginWithoutUpdates(t *testing.T) {
 				},
 			})
 
+			cfg.Profiles = append(cfg.Profiles, discardReservedSchedulerProfile())
+
 			defer func() {
 				cleanupTest(t, testCtx)
 				klog.Infof("test environment cleaned up")
@@ -356,7 +465,7 @@ func TestTopologyCachePluginWithoutUpdates(t *testing.T) {
 						t.Fatalf("Failed to delete Pod %q: %v", p.podName, err)
 					}
 				} else {
-					klog.Infof("Creating Pod %q", p.pod.Name)
+					klog.Infof("Creating Pod %q: scheduler: %q", p.pod.Name, p.pod.Spec.SchedulerName)
 					_, err := cs.CoreV1().Pods(ns).Create(testCtx.Ctx, p.pod, metav1.CreateOptions{})
 					if err != nil {
 						t.Fatalf("Failed to create Pod %q: %v", p.pod.Name, err)
@@ -689,3 +798,52 @@ func mkPFP(nodeName string, pods ...*corev1.Pod) string {
 	klog.Infof("PFP for %q: %s", nodeName, st.Repr())
 	return pfp
 }
+
+func discardReservedSchedulerProfile() schedapi.KubeSchedulerProfile {
+	nodeLockingMatchArgs := schedconfig.NodeResourceTopologyMatchArgs{
+		ScoringStrategy:      schedconfig.ScoringStrategy{Type: schedconfig.LeastAllocated},
+		DiscardReservedNodes: true,
+	}
+
+	return schedapi.KubeSchedulerProfile{
+		SchedulerName: discardReservedSchedulerName,
+		Plugins: &schedapi.Plugins{
+			QueueSort: schedapi.PluginSet{
+				Enabled: []schedapi.Plugin{
+					{Name: queuesort.Name},
+				},
+			},
+			Filter: schedapi.PluginSet{
+				Enabled: []schedapi.Plugin{
+					{Name: noderesourcetopology.Name},
+				},
+			},
+			Score: schedapi.PluginSet{
+				Enabled: []schedapi.Plugin{
+					{Name: noderesourcetopology.Name},
+				},
+			},
+			Reserve: schedapi.PluginSet{
+				Enabled: []schedapi.Plugin{
+					{Name: noderesourcetopology.Name},
+				},
+			},
+			PostBind: schedapi.PluginSet{
+				Enabled: []schedapi.Plugin{
+					{Name: noderesourcetopology.Name},
+				},
+			},
+			Bind: schedapi.PluginSet{
+				Enabled: []schedapi.Plugin{
+					{Name: defaultbinder.Name},
+				},
+			},
+		},
+		PluginConfig: []schedapi.PluginConfig{
+			{
+				Name: noderesourcetopology.Name,
+				Args: &nodeLockingMatchArgs,
+			},
+		},
+	}
+}