Merge pull request kubernetes#127615 from macsko/add_node_add_event_benchmark_to_scheduler_perf

k8s-ci-robot · web-flow · commit 2f7df335ad2d · 2024-10-11T18:10:19.000+01:00
Add scheduler_perf test case for NodeAdd event handling
diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-node-high-capacity.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-node-high-capacity.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Node
+metadata:
+  generateName: node-high-capacity-
+spec:
+  taints:
+    - key: allow
+      effect: NoSchedule
+status:
+  capacity:
+    pods: "3000"
+    cpu: "200"
+    memory: 200Gi
+  conditions:
+    - status: "True"
+      type: Ready
+  phase: Running
diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-node-low-capacity.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-node-low-capacity.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Node
+metadata:
+  generateName: node-low-capacity-
+spec:
+  taints:
+    - key: allow
+      effect: NoSchedule
+status:
+  capacity:
+    pods: "3000"
+    cpu: "0.1"
+    memory: 100Mi
+  conditions:
+    - status: "True"
+      type: Ready
+  phase: Running
diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-node-unschedulable.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-node-unschedulable.yaml
@@ -0,0 +1,23 @@
+# Node unschedulable is similar to node unblocker but has unschedulable: true.
+apiVersion: v1
+kind: Node
+metadata:
+  generateName: node-unschedulable-
+  labels:
+    topology.kubernetes.io/zone: zone1
+    affinity: allow
+    topology: allow
+spec:
+  unschedulable: true
+  taints:
+    - key: allow
+      effect: NoSchedule
+status:
+  capacity:
+    pods: "3000"
+    cpu: "4"
+    memory: 32Gi
+  conditions:
+    - status: "True"
+      type: Ready
+  phase: Running
diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-node-with-labels.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-node-with-labels.yaml
@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Node
+metadata:
+  generateName: node-unblocker-
+  labels:
+    topology.kubernetes.io/zone: zone1
+    affinity: allow
+    topology: allow
+spec:
+  taints:
+    - key: allow
+      effect: NoSchedule
+status:
+  capacity:
+    pods: "3000"
+    cpu: "4"
+    memory: 32Gi
+  conditions:
+    - status: "True"
+      type: Ready
+  phase: Running
diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-node-with-taint.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-node-with-taint.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Node
+metadata:
+  generateName: node-with-taint-
+spec:
+  taints:
+    - key: toleration
+      effect: NoSchedule
+status:
+  capacity:
+    pods: "3000"
+    cpu: "4"
+    memory: 32Gi
+  conditions:
+    - status: "True"
+      type: Ready
+  phase: Running
diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-interpodaffinity.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-interpodaffinity.yaml
@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  generateName: pod-unsched-
+  labels:
+    color: green
+spec:
+  tolerations:
+  - key: allow
+    operator: Exists
+    effect: NoSchedule
+  affinity:
+    podAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+      - labelSelector:
+          matchLabels:
+            color: green
+        topologyKey: topology.kubernetes.io/zone
+  containers:
+  - image: registry.k8s.io/pause:3.10
+    name: pause
diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-nodeaffinity.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-nodeaffinity.yaml
@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  generateName: pod-unsched-
+spec:
+  tolerations:
+  - key: allow
+    operator: Exists
+    effect: NoSchedule
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: affinity
+            operator: In
+            values:
+            - allow
+  containers:
+  - image: registry.k8s.io/pause:3.10
+    name: pause
diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-noderesources.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-noderesources.yaml
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  generateName: pod-unsched-
+spec:
+  tolerations:
+  - key: allow
+    operator: Exists
+    effect: NoSchedule
+  containers:
+  - image: registry.k8s.io/pause:3.10
+    name: pause
+    resources:
+      requests:
+        cpu: 0.2
+        memory: 200Mi
diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-nodeunschedulable.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-nodeunschedulable.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  generateName: pod-unsched-
+spec:
+  tolerations:
+  - key: allow
+    operator: Exists
+    effect: NoSchedule
+  containers:
+  - image: registry.k8s.io/pause:3.10
+    name: pause
diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-podtopologyspread.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-podtopologyspread.yaml
@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  generateName: pod-unsched-
+  labels:
+    color: blue
+spec:
+  tolerations:
+  - key: allow
+    operator: Exists
+    effect: NoSchedule
+  topologySpreadConstraints:
+    - maxSkew: 1000
+      minDomains: 10000
+      topologyKey: topology
+      whenUnsatisfiable: DoNotSchedule
+      labelSelector:
+        matchLabels:
+          color: blue
+  containers:
+  - image: registry.k8s.io/pause:3.10
+    name: pause
diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-tainttoleration.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-tainttoleration.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  generateName: pod-unsched-
+spec:
+  tolerations:
+  - key: toleration
+    operator: Exists
+    effect: NoSchedule
+  containers:
+  - image: registry.k8s.io/pause:3.10
+    name: pause
diff --git a/test/integration/scheduler_perf/config/performance-config.yaml b/test/integration/scheduler_perf/config/performance-config.yaml
@@ -1784,3 +1784,100 @@
     params:
       blockerPods: 1000
       measurePods: 1000
+
+# This test case is used to measure the performance of queuing hints when handling the NodeAdd events.
+# First, an unschedulable node is created, which prevents any pod from being scheduled on it.
+# Then multiple types of pods are created, and each group is filtered by different plugin.
+# Next, nodes are created where previously unscheduled pods can be scheduled.
+# The test case is divided into several stages to make sure that the pods are filtered by a specific plugin.
+# Plugins covered: InterPodAffinity, NodeAffinity, NodeResources, NodeUnschedulable, PodTopologySpread and TaintToleration.
+- name: EventHandlingNodeAdd
+  featureGates:
+    SchedulerQueueingHints: true
+  workloadTemplate:
+  # Collect metrics from all createPods ops.
+  - opcode: startCollectingMetrics
+    name: unschedPods
+    namespaces: [nodeunschedulable, noderesources, interpodaffinity, nodeaffinity, podtopologyspread, tainttoleration]
+  # Create one unschedulable node.
+  - opcode: createNodes
+    count: 1
+    nodeTemplatePath: config/event_handling/nodeadd-node-unschedulable.yaml
+  # Created pods blocked using NodeUnschedulable plugin.
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/event_handling/nodeadd-pod-nodeunschedulable.yaml
+    skipWaitToCompletion: true
+    namespace: nodeunschedulable
+  # Wait for unschedulable pods to be processed by the scheduler.
+  - opcode: barrier
+    stageRequirement: Attempted
+  # Create schedulable node with low capacity.
+  - opcode: createNodes
+    count: 1
+    nodeTemplatePath: config/event_handling/nodeadd-node-low-capacity.yaml
+  # Created pods blocked using NodeResources plugin.
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/event_handling/nodeadd-pod-noderesources.yaml
+    skipWaitToCompletion: true
+    namespace: noderesources
+  # Wait for unschedulable pods to be processed by the scheduler.
+  - opcode: barrier
+    stageRequirement: Attempted
+  # Create nodes that will have enough resource capacity for pods blocked by NodeResources plugin.
+  # These nodes will still block the next pods from being scheduled.
+  - opcode: createNodes
+    countParam: $nodes
+    nodeTemplatePath: config/event_handling/nodeadd-node-high-capacity.yaml
+  # Wait on barrier for NodeUnschedulable and NodeResources pods to be scheduled.
+  - opcode: barrier
+  # Created pods blocked using InterPodAffinity plugin.
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/event_handling/nodeadd-pod-interpodaffinity.yaml
+    skipWaitToCompletion: true
+    namespace: interpodaffinity
+  # Created pods blocked using NodeAffinity plugin.
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/event_handling/nodeadd-pod-nodeaffinity.yaml
+    skipWaitToCompletion: true
+    namespace: nodeaffinity
+  # Created pods blocked using PodTopologySpread plugin.
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/event_handling/nodeadd-pod-podtopologyspread.yaml
+    skipWaitToCompletion: true
+    namespace: podtopologyspread
+  # Wait for unschedulable pods to be processed by the scheduler.
+  - opcode: barrier
+    stageRequirement: Attempted
+  # Create nodes that will unblock most of the unschedulable pods.
+  - opcode: createNodes
+    countParam: $nodes
+    nodeTemplatePath: config/event_handling/nodeadd-node-with-labels.yaml
+  # Wait on barrier for InterPodAffinity, NodeAffinity and PodTopologySpread pods to be scheduled.
+  - opcode: barrier
+  # Created pods blocked using TaintToleration plugin.
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/event_handling/nodeadd-pod-tainttoleration.yaml
+    skipWaitToCompletion: true
+    namespace: tainttoleration
+  # Wait for unschedulable pods to be processed by the scheduler.
+  - opcode: barrier
+    stageRequirement: Attempted
+  # Create nodes that will unblock pods filtered out by TaintToleration plugin.
+  - opcode: createNodes
+    countParam: $nodes
+    nodeTemplatePath: config/event_handling/nodeadd-node-with-taint.yaml
+  # Wait on barrier for TaintToleration pods to be scheduled.
+  - opcode: barrier
+  - opcode: stopCollectingMetrics
+  workloads:
+  - name: 100Nodes_500Pods
+    labels: [performance, short]
+    params:
+      nodes: 100
+      measurePods: 1000 # Must be initNodes * 10
