Skip to content

Commit a442795

Browse files
tiraboschiopenshift-cherrypick-robot
tiraboschi
authored and
openshift-cherrypick-robot
committed
Align with KubeVirt defaults for concurrent evictions
By default KubeVirt is limiting the number of concurrent live migrations in the cluster to 5 with a second limit of 2 outgoing concurrent live migrations per node. When RelieveAndMigrate is selected, let's align by default with KubeVirt defaults for the sake of consistency. The cluster admin can override those values with parameters `evictionLimits.total` and `evictionLimits.node`. To keep the two operators decoupled, the cluster kube descheduler operator is not going to watch and automatically reach to changes on the KubeVirt configuration. Signed-off-by: Simone Tiraboschi <[email protected]>
1 parent 0ccd6f8 commit a442795

13 files changed

+144
-8
lines changed

‎README.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,10 @@ The profile exposes the following customization:
192192
By default, this profile will enable load-aware descheduling based on the `PrometheusCPUCombined` Prometheus query.
193193
By default, the thresholds will be dynamic (based on the distance from the average utilization) and asymmetric (all the nodes below the average will be considered as underutilized to help rebalancing overutilized outliers) tolerating low deviations (10%).
194194

195+
By default, this profile configures the descheduler to restrict the maximum number of overall parallel evictions to 5 and
196+
the maximum number of evictions per node to 2 aligning with KubeVirt defaults around concurrent live migrations.
197+
Those two values can be customized with `evictionLimits.total` and `evictionLimits.node` parameters.
198+
195199
### EvictPodsWithPVC
196200
By default, the operator prevents pods with PVCs from being evicted. Enabling this
197201
profile in combination with any of the above profiles allows pods with PVCs to be

‎pkg/operator/target_config_reconciler.go

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@ const DefaultImage = "quay.io/openshift/origin-descheduler:latest"
7070
const kubeVirtShedulableLabelSelector = "kubevirt.io/schedulable=true"
7171
const psiPath = "/proc/pressure/"
7272
const EXPERIMENTAL_DISABLE_PSI_CHECK = "EXPERIMENTAL_DISABLE_PSI_CHECK"
73+
const defaultKVParallelMigrationsPerCluster = 5
74+
const defaultKVParallelOutboundMigrationsPerNode = 2
7375

7476
// deschedulerCommand provides descheduler command with policyconfigfile mounted as volume and log-level for backwards
7577
// compatibility with 3.11
@@ -1295,14 +1297,7 @@ func (c *TargetConfigReconciler) manageConfigMap(descheduler *deschedulerv1.Kube
12951297
Profiles: []v1alpha2.DeschedulerProfile{},
12961298
}
12971299

1298-
if descheduler.Spec.EvictionLimits != nil {
1299-
if descheduler.Spec.EvictionLimits.Total != nil {
1300-
policy.MaxNoOfPodsToEvictTotal = utilptr.To[uint](uint(*descheduler.Spec.EvictionLimits.Total))
1301-
}
1302-
if descheduler.Spec.EvictionLimits.Node != nil {
1303-
policy.MaxNoOfPodsToEvictPerNode = utilptr.To[uint](uint(*descheduler.Spec.EvictionLimits.Node))
1304-
}
1305-
}
1300+
c.setEvictionsLimits(descheduler, policy)
13061301

13071302
if c.isPrometheusAsMetricsProviderForProfiles(descheduler) {
13081303
// detect the prometheus server url
@@ -1728,3 +1723,23 @@ func (c *TargetConfigReconciler) isPrometheusAsMetricsProviderForProfiles(desche
17281723
}
17291724
return false
17301725
}
1726+
1727+
func (c *TargetConfigReconciler) setEvictionsLimits(descheduler *deschedulerv1.KubeDescheduler, policy *v1alpha2.DeschedulerPolicy) {
1728+
if descheduler == nil || policy == nil {
1729+
return
1730+
}
1731+
1732+
if slices.Contains(descheduler.Spec.Profiles, deschedulerv1.RelieveAndMigrate) {
1733+
policy.MaxNoOfPodsToEvictTotal = utilptr.To[uint](uint(defaultKVParallelMigrationsPerCluster))
1734+
policy.MaxNoOfPodsToEvictPerNode = utilptr.To[uint](uint(defaultKVParallelOutboundMigrationsPerNode))
1735+
}
1736+
1737+
if descheduler.Spec.EvictionLimits != nil {
1738+
if descheduler.Spec.EvictionLimits.Total != nil {
1739+
policy.MaxNoOfPodsToEvictTotal = utilptr.To[uint](uint(*descheduler.Spec.EvictionLimits.Total))
1740+
}
1741+
if descheduler.Spec.EvictionLimits.Node != nil {
1742+
policy.MaxNoOfPodsToEvictPerNode = utilptr.To[uint](uint(*descheduler.Spec.EvictionLimits.Node))
1743+
}
1744+
}
1745+
}

‎pkg/operator/target_config_reconciler_test.go

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -332,6 +332,51 @@ func TestManageConfigMap(t *testing.T) {
332332
},
333333
},
334334
},
335+
{
336+
name: "RelieveAndMigrateEvictionLimits",
337+
descheduler: &deschedulerv1.KubeDescheduler{
338+
Spec: deschedulerv1.KubeDeschedulerSpec{
339+
Profiles: []deschedulerv1.DeschedulerProfile{"DevKubeVirtRelieveAndMigrate"},
340+
ProfileCustomizations: nil,
341+
EvictionLimits: &deschedulerv1.EvictionLimits{
342+
Total: utilptr.To[int32](10),
343+
Node: utilptr.To[int32](3),
344+
},
345+
},
346+
},
347+
want: &corev1.ConfigMap{
348+
TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ConfigMap"},
349+
Data: map[string]string{"policy.yaml": string(bindata.MustAsset("assets/relieveAndMigrateEvictionLimits.yaml"))},
350+
},
351+
routes: []runtime.Object{
352+
&routev1.Route{
353+
ObjectMeta: metav1.ObjectMeta{
354+
Namespace: "openshift-monitoring",
355+
Name: "prometheus-k8s",
356+
},
357+
Status: routev1.RouteStatus{Ingress: []routev1.RouteIngress{
358+
{
359+
Host: "prometheus-k8s-openshift-monitoring.apps.example.com",
360+
},
361+
},
362+
},
363+
},
364+
},
365+
nodes: []runtime.Object{
366+
&corev1.Node{
367+
ObjectMeta: metav1.ObjectMeta{
368+
Name: "node1",
369+
Labels: map[string]string{"kubevirt.io/schedulable": "true"},
370+
},
371+
},
372+
&corev1.Node{
373+
ObjectMeta: metav1.ObjectMeta{
374+
Name: "node2",
375+
Labels: map[string]string{"kubevirt.io/schedulable": "true"},
376+
},
377+
},
378+
},
379+
},
335380
{
336381
name: "RelieveAndMigrateLow",
337382
descheduler: &deschedulerv1.KubeDescheduler{

‎pkg/operator/testdata/assets/relieveAndMigrateDefaults.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
apiVersion: descheduler/v1alpha2
22
kind: DeschedulerPolicy
3+
maxNoOfPodsToEvictPerNode: 2
4+
maxNoOfPodsToEvictTotal: 5
35
metricsProviders:
46
- prometheus:
57
url: https://prometheus-k8s-openshift-monitoring.apps.example.com

‎pkg/operator/testdata/assets/relieveAndMigrateDeviationLowWithCombinedMetrics.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
apiVersion: descheduler/v1alpha2
22
kind: DeschedulerPolicy
3+
maxNoOfPodsToEvictPerNode: 2
4+
maxNoOfPodsToEvictTotal: 5
35
metricsProviders:
46
- prometheus:
57
url: https://prometheus-k8s-openshift-monitoring.apps.example.com

‎pkg/operator/testdata/assets/relieveAndMigrateDynamicThresholdsHigh.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
apiVersion: descheduler/v1alpha2
22
kind: DeschedulerPolicy
3+
maxNoOfPodsToEvictPerNode: 2
4+
maxNoOfPodsToEvictTotal: 5
35
metricsProviders:
46
- prometheus:
57
url: https://prometheus-k8s-openshift-monitoring.apps.example.com

‎pkg/operator/testdata/assets/relieveAndMigrateDynamicThresholdsLow.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
apiVersion: descheduler/v1alpha2
22
kind: DeschedulerPolicy
3+
maxNoOfPodsToEvictPerNode: 2
4+
maxNoOfPodsToEvictTotal: 5
35
metricsProviders:
46
- prometheus:
57
url: https://prometheus-k8s-openshift-monitoring.apps.example.com

‎pkg/operator/testdata/assets/relieveAndMigrateDynamicThresholdsMedium.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
apiVersion: descheduler/v1alpha2
22
kind: DeschedulerPolicy
3+
maxNoOfPodsToEvictPerNode: 2
4+
maxNoOfPodsToEvictTotal: 5
35
metricsProviders:
46
- prometheus:
57
url: https://prometheus-k8s-openshift-monitoring.apps.example.com

‎pkg/operator/testdata/assets/relieveAndMigrateEvictionLimits.yaml

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
apiVersion: descheduler/v1alpha2
2+
kind: DeschedulerPolicy
3+
maxNoOfPodsToEvictPerNode: 3
4+
maxNoOfPodsToEvictTotal: 10
5+
metricsProviders:
6+
- prometheus:
7+
url: https://prometheus-k8s-openshift-monitoring.apps.example.com
8+
source: Prometheus
9+
nodeSelector: kubevirt.io/schedulable=true
10+
profiles:
11+
- name: DevKubeVirtRelieveAndMigrate
12+
pluginConfig:
13+
- args:
14+
evictableNamespaces:
15+
exclude:
16+
- kube-system
17+
- hypershift
18+
- openshift
19+
- openshift-kube-descheduler-operator
20+
- openshift-kube-scheduler
21+
metricsUtilization:
22+
prometheus:
23+
query: descheduler:combined_utilization_and_pressure:avg1m
24+
source: Prometheus
25+
targetThresholds:
26+
MetricResource: 10
27+
thresholds:
28+
MetricResource: 0
29+
useDeviationThresholds: true
30+
name: LowNodeUtilization
31+
- args:
32+
evictLocalStoragePods: true
33+
name: DefaultEvictor
34+
plugins:
35+
balance:
36+
disabled: null
37+
enabled:
38+
- LowNodeUtilization
39+
deschedule:
40+
disabled: null
41+
enabled: null
42+
filter:
43+
disabled: null
44+
enabled:
45+
- DefaultEvictor
46+
preevictionfilter:
47+
disabled: null
48+
enabled: null
49+
presort:
50+
disabled: null
51+
enabled: null
52+
sort:
53+
disabled: null
54+
enabled: null

‎pkg/operator/testdata/assets/relieveAndMigrateHighConfig.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
apiVersion: descheduler/v1alpha2
22
kind: DeschedulerPolicy
3+
maxNoOfPodsToEvictPerNode: 2
4+
maxNoOfPodsToEvictTotal: 5
35
metricsProviders:
46
- prometheus:
57
url: https://prometheus-k8s-openshift-monitoring.apps.example.com

0 commit comments

Comments
 (0)