Skip to content

Commit 7c459fc

Browse files
ingvagabundingvagabund
authored
Merge pull request #612 from openshift-cherrypick-robot/cherry-pick-589-to-release-4.19
[release-4.19] [RelieveAndMigrate] Align with KubeVirt defaults for concurrent evictions
2 parents c99801e + a442795 commit 7c459fc

15 files changed

+236
-5
lines changed

‎README.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,10 @@ The profile exposes the following customization:
192192
By default, this profile will enable load-aware descheduling based on the `PrometheusCPUCombined` Prometheus query.
193193
By default, the thresholds will be dynamic (based on the distance from the average utilization) and asymmetric (all the nodes below the average will be considered as underutilized to help rebalancing overutilized outliers) tolerating low deviations (10%).
194194

195+
By default, this profile configures the descheduler to restrict the maximum number of overall parallel evictions to 5 and
196+
the maximum number of evictions per node to 2 aligning with KubeVirt defaults around concurrent live migrations.
197+
Those two values can be customized with `evictionLimits.total` and `evictionLimits.node` parameters.
198+
195199
### EvictPodsWithPVC
196200
By default, the operator prevents pods with PVCs from being evicted. Enabling this
197201
profile in combination with any of the above profiles allows pods with PVCs to be
@@ -294,3 +298,4 @@ The Descheduler operator exposes the following parameters in its CRD:
294298
|`mode`|`string`|Configures the descheduler to either evict pods or to simulate the eviction|
295299
|`evictionLimits`|`map`|Restrict the number of evictions during each descheduling run. Available fields are: `total`|
296300
|`evictionLimits.total`|`int32`|Restricts the maximum number of overall evictions|
301+
|`evictionLimits.node`|`int32`|Restricts the maximum number of of evictions per node|

‎pkg/apis/descheduler/v1/types_descheduler.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@ type KubeDeschedulerSpec struct {
5050
type EvictionLimits struct {
5151
// total restricts the maximum number of overall evictions
5252
Total *int32 `json:"total,omitempty"`
53+
54+
// node restricts the maximum number of evictions per node
55+
Node *int32 `json:"node,omitempty"`
5356
}
5457

5558
// ProfileCustomizations contains various parameters for modifying the default behavior of certain profiles

‎pkg/operator/target_config_reconciler.go

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@ const DefaultImage = "quay.io/openshift/origin-descheduler:latest"
7070
const kubeVirtShedulableLabelSelector = "kubevirt.io/schedulable=true"
7171
const psiPath = "/proc/pressure/"
7272
const EXPERIMENTAL_DISABLE_PSI_CHECK = "EXPERIMENTAL_DISABLE_PSI_CHECK"
73+
const defaultKVParallelMigrationsPerCluster = 5
74+
const defaultKVParallelOutboundMigrationsPerNode = 2
7375

7476
// deschedulerCommand provides descheduler command with policyconfigfile mounted as volume and log-level for backwards
7577
// compatibility with 3.11
@@ -1295,11 +1297,7 @@ func (c *TargetConfigReconciler) manageConfigMap(descheduler *deschedulerv1.Kube
12951297
Profiles: []v1alpha2.DeschedulerProfile{},
12961298
}
12971299

1298-
if descheduler.Spec.EvictionLimits != nil {
1299-
if descheduler.Spec.EvictionLimits.Total != nil {
1300-
policy.MaxNoOfPodsToEvictTotal = utilptr.To[uint](uint(*descheduler.Spec.EvictionLimits.Total))
1301-
}
1302-
}
1300+
c.setEvictionsLimits(descheduler, policy)
13031301

13041302
if c.isPrometheusAsMetricsProviderForProfiles(descheduler) {
13051303
// detect the prometheus server url
@@ -1725,3 +1723,23 @@ func (c *TargetConfigReconciler) isPrometheusAsMetricsProviderForProfiles(desche
17251723
}
17261724
return false
17271725
}
1726+
1727+
func (c *TargetConfigReconciler) setEvictionsLimits(descheduler *deschedulerv1.KubeDescheduler, policy *v1alpha2.DeschedulerPolicy) {
1728+
if descheduler == nil || policy == nil {
1729+
return
1730+
}
1731+
1732+
if slices.Contains(descheduler.Spec.Profiles, deschedulerv1.RelieveAndMigrate) {
1733+
policy.MaxNoOfPodsToEvictTotal = utilptr.To[uint](uint(defaultKVParallelMigrationsPerCluster))
1734+
policy.MaxNoOfPodsToEvictPerNode = utilptr.To[uint](uint(defaultKVParallelOutboundMigrationsPerNode))
1735+
}
1736+
1737+
if descheduler.Spec.EvictionLimits != nil {
1738+
if descheduler.Spec.EvictionLimits.Total != nil {
1739+
policy.MaxNoOfPodsToEvictTotal = utilptr.To[uint](uint(*descheduler.Spec.EvictionLimits.Total))
1740+
}
1741+
if descheduler.Spec.EvictionLimits.Node != nil {
1742+
policy.MaxNoOfPodsToEvictPerNode = utilptr.To[uint](uint(*descheduler.Spec.EvictionLimits.Node))
1743+
}
1744+
}
1745+
}

‎pkg/operator/target_config_reconciler_test.go

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,23 @@ func TestManageConfigMap(t *testing.T) {
274274
Data: map[string]string{"policy.yaml": string(bindata.MustAsset("assets/lowNodeUtilizationHighConfig.yaml"))},
275275
},
276276
},
277+
{
278+
name: "LowNodeUtilizationEvictionLimits",
279+
descheduler: &deschedulerv1.KubeDescheduler{
280+
Spec: deschedulerv1.KubeDeschedulerSpec{
281+
Profiles: []deschedulerv1.DeschedulerProfile{"LifecycleAndUtilization"},
282+
ProfileCustomizations: &deschedulerv1.ProfileCustomizations{DevLowNodeUtilizationThresholds: utilptr.To[deschedulerv1.LowNodeUtilizationThresholdsType]("")},
283+
EvictionLimits: &deschedulerv1.EvictionLimits{
284+
Total: utilptr.To[int32](10),
285+
Node: utilptr.To[int32](3),
286+
},
287+
},
288+
},
289+
want: &corev1.ConfigMap{
290+
TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ConfigMap"},
291+
Data: map[string]string{"policy.yaml": string(bindata.MustAsset("assets/lowNodeUtilizationEvictionLimits.yaml"))},
292+
},
293+
},
277294
{
278295
name: "RelieveAndMigrateWithoutCustomizations",
279296
descheduler: &deschedulerv1.KubeDescheduler{
@@ -315,6 +332,51 @@ func TestManageConfigMap(t *testing.T) {
315332
},
316333
},
317334
},
335+
{
336+
name: "RelieveAndMigrateEvictionLimits",
337+
descheduler: &deschedulerv1.KubeDescheduler{
338+
Spec: deschedulerv1.KubeDeschedulerSpec{
339+
Profiles: []deschedulerv1.DeschedulerProfile{"DevKubeVirtRelieveAndMigrate"},
340+
ProfileCustomizations: nil,
341+
EvictionLimits: &deschedulerv1.EvictionLimits{
342+
Total: utilptr.To[int32](10),
343+
Node: utilptr.To[int32](3),
344+
},
345+
},
346+
},
347+
want: &corev1.ConfigMap{
348+
TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ConfigMap"},
349+
Data: map[string]string{"policy.yaml": string(bindata.MustAsset("assets/relieveAndMigrateEvictionLimits.yaml"))},
350+
},
351+
routes: []runtime.Object{
352+
&routev1.Route{
353+
ObjectMeta: metav1.ObjectMeta{
354+
Namespace: "openshift-monitoring",
355+
Name: "prometheus-k8s",
356+
},
357+
Status: routev1.RouteStatus{Ingress: []routev1.RouteIngress{
358+
{
359+
Host: "prometheus-k8s-openshift-monitoring.apps.example.com",
360+
},
361+
},
362+
},
363+
},
364+
},
365+
nodes: []runtime.Object{
366+
&corev1.Node{
367+
ObjectMeta: metav1.ObjectMeta{
368+
Name: "node1",
369+
Labels: map[string]string{"kubevirt.io/schedulable": "true"},
370+
},
371+
},
372+
&corev1.Node{
373+
ObjectMeta: metav1.ObjectMeta{
374+
Name: "node2",
375+
Labels: map[string]string{"kubevirt.io/schedulable": "true"},
376+
},
377+
},
378+
},
379+
},
318380
{
319381
name: "RelieveAndMigrateLow",
320382
descheduler: &deschedulerv1.KubeDescheduler{

‎pkg/operator/testdata/assets/lowNodeUtilizationEvictionLimits.yaml

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
apiVersion: descheduler/v1alpha2
2+
kind: DeschedulerPolicy
3+
maxNoOfPodsToEvictPerNode: 3
4+
maxNoOfPodsToEvictTotal: 10
5+
profiles:
6+
- name: LifecycleAndUtilization
7+
pluginConfig:
8+
- args:
9+
maxPodLifeTimeSeconds: 86400
10+
namespaces:
11+
exclude:
12+
- kube-system
13+
- hypershift
14+
- openshift
15+
- openshift-kube-descheduler-operator
16+
- openshift-kube-scheduler
17+
name: PodLifeTime
18+
- args:
19+
includingInitContainers: true
20+
namespaces:
21+
exclude:
22+
- kube-system
23+
- hypershift
24+
- openshift
25+
- openshift-kube-descheduler-operator
26+
- openshift-kube-scheduler
27+
podRestartThreshold: 100
28+
name: RemovePodsHavingTooManyRestarts
29+
- args:
30+
evictableNamespaces:
31+
exclude:
32+
- kube-system
33+
- hypershift
34+
- openshift
35+
- openshift-kube-descheduler-operator
36+
- openshift-kube-scheduler
37+
targetThresholds:
38+
cpu: 50
39+
memory: 50
40+
pods: 50
41+
thresholds:
42+
cpu: 20
43+
memory: 20
44+
pods: 20
45+
name: LowNodeUtilization
46+
- args:
47+
ignorePvcPods: true
48+
name: DefaultEvictor
49+
plugins:
50+
balance:
51+
disabled: null
52+
enabled:
53+
- LowNodeUtilization
54+
deschedule:
55+
disabled: null
56+
enabled:
57+
- PodLifeTime
58+
- RemovePodsHavingTooManyRestarts
59+
filter:
60+
disabled: null
61+
enabled:
62+
- DefaultEvictor
63+
preevictionfilter:
64+
disabled: null
65+
enabled: null
66+
presort:
67+
disabled: null
68+
enabled: null
69+
sort:
70+
disabled: null
71+
enabled: null

‎pkg/operator/testdata/assets/relieveAndMigrateDefaults.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
apiVersion: descheduler/v1alpha2
22
kind: DeschedulerPolicy
3+
maxNoOfPodsToEvictPerNode: 2
4+
maxNoOfPodsToEvictTotal: 5
35
metricsProviders:
46
- prometheus:
57
url: https://prometheus-k8s-openshift-monitoring.apps.example.com

‎pkg/operator/testdata/assets/relieveAndMigrateDeviationLowWithCombinedMetrics.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
apiVersion: descheduler/v1alpha2
22
kind: DeschedulerPolicy
3+
maxNoOfPodsToEvictPerNode: 2
4+
maxNoOfPodsToEvictTotal: 5
35
metricsProviders:
46
- prometheus:
57
url: https://prometheus-k8s-openshift-monitoring.apps.example.com

‎pkg/operator/testdata/assets/relieveAndMigrateDynamicThresholdsHigh.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
apiVersion: descheduler/v1alpha2
22
kind: DeschedulerPolicy
3+
maxNoOfPodsToEvictPerNode: 2
4+
maxNoOfPodsToEvictTotal: 5
35
metricsProviders:
46
- prometheus:
57
url: https://prometheus-k8s-openshift-monitoring.apps.example.com

‎pkg/operator/testdata/assets/relieveAndMigrateDynamicThresholdsLow.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
apiVersion: descheduler/v1alpha2
22
kind: DeschedulerPolicy
3+
maxNoOfPodsToEvictPerNode: 2
4+
maxNoOfPodsToEvictTotal: 5
35
metricsProviders:
46
- prometheus:
57
url: https://prometheus-k8s-openshift-monitoring.apps.example.com

‎pkg/operator/testdata/assets/relieveAndMigrateDynamicThresholdsMedium.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
apiVersion: descheduler/v1alpha2
22
kind: DeschedulerPolicy
3+
maxNoOfPodsToEvictPerNode: 2
4+
maxNoOfPodsToEvictTotal: 5
35
metricsProviders:
46
- prometheus:
57
url: https://prometheus-k8s-openshift-monitoring.apps.example.com

0 commit comments

Comments
 (0)