[RelieveAndMigrate] provide defaults for DevDeviationThresholds and DevActualUtilizationProfile

Provide sane defaults for DevDeviationThresholds and
DevActualUtilizationProfile with the KubeVirtRelieveAndMigrate
profile.

Signed-off-by: Simone Tiraboschi <[email protected]>

Author: tiraboschi

README.md

@@ -180,6 +180,9 @@ The profile exposes the following customization:
 - `devDeviationThresholds`: Have the thresholds be based on the average utilization.
 - `devEnableSoftTainter`: Have the operator deploying the soft-tainter component to dynamically set/remove soft taints according to the same criteria used for load aware descheduling. Scheduling and descheduling decisions are in general fully independent. In case of load-aware descheduling we can potentially have a relevant asymmetry between the descheduling and successive scheduling decisions. The soft taints set by the descheduler soft-tainter act as a hint for the scheduler to mitigate this asymmetry and foster a quicker convergence.
 
+By default, this profile will enable load-aware descheduling based on the `PrometheusCPUCombinedProfile` Prometheus query.
+By default, the thresholds will be dynamic (based on the distance from the average utilization) and asymmetric (all the nodes below the average will be considered as underutilized to help rebalancing overutilized outliers) tolerating low deviations (10%).
+
 ### EvictPodsWithPVC
 By default, the operator prevents pods with PVCs from being evicted. Enabling this
 profile in combination with any of the above profiles allows pods with PVCs to be

pkg/operator/target_config_reconciler.go

@@ -881,6 +881,26 @@ func getLowNodeUtilizationThresholds(profileCustomizations *deschedulerv1.Profil
 	return lowThreshold, highThreshold, nil
 }
 
+func getRelieveAndMigrateThresholds(profileCustomizations *deschedulerv1.ProfileCustomizations, useDeviationThresholds bool) (deschedulerapi.Percentage, deschedulerapi.Percentage, error) {
+	if profileCustomizations != nil && (profileCustomizations.DevLowNodeUtilizationThresholds != nil || profileCustomizations.DevDeviationThresholds != nil) {
+		return getLowNodeUtilizationThresholds(profileCustomizations, false)
+	}
+
+	const defaultAssymetricDeviatonThresholdLow = 0
+	const defaultAssymetricDeviatonThresholdHigh = 10
+	const defaultThresholdLow = 20
+	const defaultThresholdHigh = 50
+
+	lowThreshold := deschedulerapi.Percentage(defaultAssymetricDeviatonThresholdLow)
+	highThreshold := deschedulerapi.Percentage(defaultAssymetricDeviatonThresholdHigh)
+	if !useDeviationThresholds {
+		lowThreshold = deschedulerapi.Percentage(defaultThresholdLow)
+		highThreshold = deschedulerapi.Percentage(defaultThresholdHigh)
+	}
+
+	return lowThreshold, highThreshold, nil
+}
+
 func lifecycleAndUtilizationProfile(profileCustomizations *deschedulerv1.ProfileCustomizations, includedNamespaces, excludedNamespaces, protectedNamespaces []string, ignorePVCPods, evictLocalStoragePods bool) (*v1alpha2.DeschedulerProfile, error) {
 	profile := &v1alpha2.DeschedulerProfile{
 		Name: string(deschedulerv1.LifecycleAndUtilization),
@@ -1035,17 +1055,33 @@ func relieveAndMigrateProfile(profileCustomizations *deschedulerv1.ProfileCustom
 		setExcludedNamespacesForLowNodeUtilizationPlugin(profile.PluginConfigs[0].Args.Object.(*nodeutilization.LowNodeUtilizationArgs), includedNamespaces, excludedNamespaces, protectedNamespaces)
 	}
 
-	lowThreshold, highThreshold, err := getLowNodeUtilizationThresholds(profileCustomizations, false)
+	args := profile.PluginConfigs[0].Args.Object.(*nodeutilization.LowNodeUtilizationArgs)
+
+	// profile defaults
+	const defaultActualUtilizationProfile = deschedulerv1.PrometheusCPUCombinedProfile
+	args.UseDeviationThresholds = true
+	query, err := utilizationProfileToPrometheusQuery(defaultActualUtilizationProfile)
 	if err != nil {
 		return nil, err
 	}
-
-	resourceNames := []v1.ResourceName{v1.ResourceCPU, v1.ResourceMemory, v1.ResourcePods}
-	args := profile.PluginConfigs[0].Args.Object.(*nodeutilization.LowNodeUtilizationArgs)
+	args.MetricsUtilization = &nodeutilization.MetricsUtilization{
+		Source: deschedulerapi.MetricsSource(v1alpha2.PrometheusMetrics),
+		Prometheus: &nodeutilization.Prometheus{
+			Query: query,
+		},
+	}
+	resourceNames := []v1.ResourceName{nodeutilization.MetricResource}
 
 	if profileCustomizations != nil {
 		// enable deviation
-		args.UseDeviationThresholds = profileCustomizations.DevDeviationThresholds != nil && *profileCustomizations.DevDeviationThresholds != ""
+		if profileCustomizations.DevDeviationThresholds != nil && profileCustomizations.DevLowNodeUtilizationThresholds != nil {
+			return nil, fmt.Errorf("only one of DevLowNodeUtilizationThresholds and DevDeviationThresholds customizations can be configured simultaneously")
+		}
+		if profileCustomizations.DevDeviationThresholds != nil {
+			args.UseDeviationThresholds = *profileCustomizations.DevDeviationThresholds != ""
+		} else if profileCustomizations.DevLowNodeUtilizationThresholds != nil {
+			args.UseDeviationThresholds = *profileCustomizations.DevLowNodeUtilizationThresholds == ""
+		}
 
 		if profileCustomizations.DevActualUtilizationProfile != "" {
 			query, err := utilizationProfileToPrometheusQuery(profileCustomizations.DevActualUtilizationProfile)
@@ -1066,6 +1102,10 @@ func relieveAndMigrateProfile(profileCustomizations *deschedulerv1.ProfileCustom
 		}
 	}
 
+	lowThreshold, highThreshold, err := getRelieveAndMigrateThresholds(profileCustomizations, args.UseDeviationThresholds)
+	if err != nil {
+		return nil, err
+	}
 	for _, resourceName := range resourceNames {
 		args.Thresholds[resourceName] = lowThreshold
 		args.TargetThresholds[resourceName] = highThreshold

pkg/operator/target_config_reconciler_test.go

@@ -260,6 +260,33 @@ func TestManageConfigMap(t *testing.T) {
 				Data:     map[string]string{"policy.yaml": string(bindata.MustAsset("assets/lowNodeUtilizationHighConfig.yaml"))},
 			},
 		},
+		{
+			name: "RelieveAndMigrateWithoutCustomizations",
+			descheduler: &deschedulerv1.KubeDescheduler{
+				Spec: deschedulerv1.KubeDeschedulerSpec{
+					Profiles:              []deschedulerv1.DeschedulerProfile{"DevKubeVirtRelieveAndMigrate"},
+					ProfileCustomizations: nil,
+				},
+			},
+			want: &corev1.ConfigMap{
+				TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ConfigMap"},
+				Data:     map[string]string{"policy.yaml": string(bindata.MustAsset("assets/relieveAndMigrateDefaults.yaml"))},
+			},
+			nodes: []runtime.Object{
+				&corev1.Node{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:   "node1",
+						Labels: map[string]string{"kubevirt.io/schedulable": "true"},
+					},
+				},
+				&corev1.Node{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:   "node2",
+						Labels: map[string]string{"kubevirt.io/schedulable": "true"},
+					},
+				},
+			},
+		},
 		{
 			name: "RelieveAndMigrateLow",
 			descheduler: &deschedulerv1.KubeDescheduler{

pkg/operator/testdata/assets/relieveAndMigrateDefaults.yaml

@@ -0,0 +1,48 @@
+apiVersion: descheduler/v1alpha2
+kind: DeschedulerPolicy
+nodeSelector: kubevirt.io/schedulable=true
+profiles:
+- name: DevKubeVirtRelieveAndMigrate
+  pluginConfig:
+  - args:
+      evictableNamespaces:
+        exclude:
+        - kube-system
+        - hypershift
+        - openshift
+        - openshift-kube-descheduler-operator
+        - openshift-kube-scheduler
+      metricsUtilization:
+        prometheus:
+          query: descheduler:combined_utilization_and_pressure:avg1m
+        source: Prometheus
+      targetThresholds:
+        MetricResource: 10
+      thresholds:
+        MetricResource: 0
+      useDeviationThresholds: true
+    name: LowNodeUtilization
+  - args:
+      evictLocalStoragePods: true
+    name: DefaultEvictor
+  plugins:
+    balance:
+      disabled: null
+      enabled:
+      - LowNodeUtilization
+    deschedule:
+      disabled: null
+      enabled: null
+    filter:
+      disabled: null
+      enabled:
+      - DefaultEvictor
+    preevictionfilter:
+      disabled: null
+      enabled: null
+    presort:
+      disabled: null
+      enabled: null
+    sort:
+      disabled: null
+      enabled: null

pkg/operator/testdata/assets/relieveAndMigrateDynamicThresholdsHigh.yaml

@@ -12,14 +12,14 @@ profiles:
         - openshift
         - openshift-kube-descheduler-operator
         - openshift-kube-scheduler
+      metricsUtilization:
+        prometheus:
+          query: descheduler:combined_utilization_and_pressure:avg1m
+        source: Prometheus
       targetThresholds:
-        cpu: 30
-        memory: 30
-        pods: 30
+        MetricResource: 30
       thresholds:
-        cpu: 30
-        memory: 30
-        pods: 30
+        MetricResource: 30
       useDeviationThresholds: true
     name: LowNodeUtilization
   - args:

pkg/operator/testdata/assets/relieveAndMigrateDynamicThresholdsLow.yaml

@@ -12,14 +12,14 @@ profiles:
         - openshift
         - openshift-kube-descheduler-operator
         - openshift-kube-scheduler
+      metricsUtilization:
+        prometheus:
+          query: descheduler:combined_utilization_and_pressure:avg1m
+        source: Prometheus
       targetThresholds:
-        cpu: 10
-        memory: 10
-        pods: 10
+        MetricResource: 10
       thresholds:
-        cpu: 10
-        memory: 10
-        pods: 10
+        MetricResource: 10
       useDeviationThresholds: true
     name: LowNodeUtilization
   - args:

pkg/operator/testdata/assets/relieveAndMigrateDynamicThresholdsMedium.yaml

@@ -12,14 +12,14 @@ profiles:
         - openshift
         - openshift-kube-descheduler-operator
         - openshift-kube-scheduler
+      metricsUtilization:
+        prometheus:
+          query: descheduler:combined_utilization_and_pressure:avg1m
+        source: Prometheus
       targetThresholds:
-        cpu: 20
-        memory: 20
-        pods: 20
+        MetricResource: 20
       thresholds:
-        cpu: 20
-        memory: 20
-        pods: 20
+        MetricResource: 20
       useDeviationThresholds: true
     name: LowNodeUtilization
   - args:

pkg/operator/testdata/assets/relieveAndMigrateHighConfig.yaml

@@ -12,14 +12,14 @@ profiles:
         - openshift
         - openshift-kube-descheduler-operator
         - openshift-kube-scheduler
+      metricsUtilization:
+        prometheus:
+          query: descheduler:combined_utilization_and_pressure:avg1m
+        source: Prometheus
       targetThresholds:
-        cpu: 70
-        memory: 70
-        pods: 70
+        MetricResource: 70
       thresholds:
-        cpu: 40
-        memory: 40
-        pods: 40
+        MetricResource: 40
     name: LowNodeUtilization
   - args:
       evictLocalStoragePods: true

pkg/operator/testdata/assets/relieveAndMigrateIncludedNamespace.yaml

@@ -12,14 +12,15 @@ profiles:
         - openshift
         - openshift-kube-descheduler-operator
         - openshift-kube-scheduler
+      metricsUtilization:
+        prometheus:
+          query: descheduler:combined_utilization_and_pressure:avg1m
+        source: Prometheus
       targetThresholds:
-        cpu: 50
-        memory: 50
-        pods: 50
+        MetricResource: 10
       thresholds:
-        cpu: 20
-        memory: 20
-        pods: 20
+        MetricResource: 0
+      useDeviationThresholds: true
     name: LowNodeUtilization
   - args:
       evictLocalStoragePods: true

pkg/operator/testdata/assets/relieveAndMigrateLowConfig.yaml

@@ -12,14 +12,14 @@ profiles:
         - openshift
         - openshift-kube-descheduler-operator
         - openshift-kube-scheduler
+      metricsUtilization:
+        prometheus:
+          query: descheduler:combined_utilization_and_pressure:avg1m
+        source: Prometheus
       targetThresholds:
-        cpu: 30
-        memory: 30
-        pods: 30
+        MetricResource: 30
       thresholds:
-        cpu: 10
-        memory: 10
-        pods: 10
+        MetricResource: 10
     name: LowNodeUtilization
   - args:
       evictLocalStoragePods: true