Merge pull request #510 from tiraboschi/rm_profile_defaults

[RelieveAndMigrate] provide defaults for DevDeviationThresholds and DevActualUtilizationProfile

Author: openshift-merge-bot[bot]

README.md

@@ -185,6 +185,9 @@ The profile exposes the following customization:
 - `devActualUtilizationProfile`: Enable load-aware descheduling.
 - `devDeviationThresholds`: Have the thresholds be based on the average utilization.
 
+By default, this profile will enable load-aware descheduling based on the `PrometheusCPUCombinedProfile` Prometheus query.
+By default, the thresholds will be dynamic (based on the distance from the average utilization) and asymmetric (all the nodes below the average will be considered as underutilized to help rebalancing overutilized outliers) tolerating low deviations (10%).
+
 ### EvictPodsWithPVC
 By default, the operator prevents pods with PVCs from being evicted. Enabling this
 profile in combination with any of the above profiles allows pods with PVCs to be

pkg/operator/target_config_reconciler.go

@@ -881,6 +881,26 @@ func getLowNodeUtilizationThresholds(profileCustomizations *deschedulerv1.Profil
 	return lowThreshold, highThreshold, nil
 }
 
+func getRelieveAndMigrateThresholds(profileCustomizations *deschedulerv1.ProfileCustomizations, useDeviationThresholds bool) (deschedulerapi.Percentage, deschedulerapi.Percentage, error) {
+	if profileCustomizations != nil && (profileCustomizations.DevLowNodeUtilizationThresholds != nil || profileCustomizations.DevDeviationThresholds != nil) {
+		return getLowNodeUtilizationThresholds(profileCustomizations, false)
+	}
+
+	const defaultAssymetricDeviatonThresholdLow = 0
+	const defaultAssymetricDeviatonThresholdHigh = 10
+	const defaultThresholdLow = 20
+	const defaultThresholdHigh = 50
+
+	lowThreshold := deschedulerapi.Percentage(defaultAssymetricDeviatonThresholdLow)
+	highThreshold := deschedulerapi.Percentage(defaultAssymetricDeviatonThresholdHigh)
+	if !useDeviationThresholds {
+		lowThreshold = deschedulerapi.Percentage(defaultThresholdLow)
+		highThreshold = deschedulerapi.Percentage(defaultThresholdHigh)
+	}
+
+	return lowThreshold, highThreshold, nil
+}
+
 func lifecycleAndUtilizationProfile(profileCustomizations *deschedulerv1.ProfileCustomizations, includedNamespaces, excludedNamespaces, protectedNamespaces []string, ignorePVCPods, evictLocalStoragePods bool) (*v1alpha2.DeschedulerProfile, error) {
 	profile := &v1alpha2.DeschedulerProfile{
 		Name: string(deschedulerv1.LifecycleAndUtilization),
@@ -1035,17 +1055,33 @@ func relieveAndMigrateProfile(profileCustomizations *deschedulerv1.ProfileCustom
 		setExcludedNamespacesForLowNodeUtilizationPlugin(profile.PluginConfigs[0].Args.Object.(*nodeutilization.LowNodeUtilizationArgs), includedNamespaces, excludedNamespaces, protectedNamespaces)
 	}
 
-	lowThreshold, highThreshold, err := getLowNodeUtilizationThresholds(profileCustomizations, false)
+	args := profile.PluginConfigs[0].Args.Object.(*nodeutilization.LowNodeUtilizationArgs)
+
+	// profile defaults
+	const defaultActualUtilizationProfile = deschedulerv1.PrometheusCPUCombinedProfile
+	args.UseDeviationThresholds = true
+	query, err := utilizationProfileToPrometheusQuery(defaultActualUtilizationProfile)
 	if err != nil {
 		return nil, err
 	}
-
-	resourceNames := []v1.ResourceName{v1.ResourceCPU, v1.ResourceMemory, v1.ResourcePods}
-	args := profile.PluginConfigs[0].Args.Object.(*nodeutilization.LowNodeUtilizationArgs)
+	args.MetricsUtilization = &nodeutilization.MetricsUtilization{
+		Source: deschedulerapi.MetricsSource(v1alpha2.PrometheusMetrics),
+		Prometheus: &nodeutilization.Prometheus{
+			Query: query,
+		},
+	}
+	resourceNames := []v1.ResourceName{nodeutilization.MetricResource}
 
 	if profileCustomizations != nil {
 		// enable deviation
-		args.UseDeviationThresholds = profileCustomizations.DevDeviationThresholds != nil && *profileCustomizations.DevDeviationThresholds != ""
+		if profileCustomizations.DevDeviationThresholds != nil && profileCustomizations.DevLowNodeUtilizationThresholds != nil {
+			return nil, fmt.Errorf("only one of DevLowNodeUtilizationThresholds and DevDeviationThresholds customizations can be configured simultaneously")
+		}
+		if profileCustomizations.DevDeviationThresholds != nil {
+			args.UseDeviationThresholds = *profileCustomizations.DevDeviationThresholds != ""
+		} else if profileCustomizations.DevLowNodeUtilizationThresholds != nil {
+			args.UseDeviationThresholds = *profileCustomizations.DevLowNodeUtilizationThresholds == ""
+		}
 
 		if profileCustomizations.DevActualUtilizationProfile != "" {
 			query, err := utilizationProfileToPrometheusQuery(profileCustomizations.DevActualUtilizationProfile)
@@ -1066,6 +1102,10 @@ func relieveAndMigrateProfile(profileCustomizations *deschedulerv1.ProfileCustom
 		}
 	}
 
+	lowThreshold, highThreshold, err := getRelieveAndMigrateThresholds(profileCustomizations, args.UseDeviationThresholds)
+	if err != nil {
+		return nil, err
+	}
 	for _, resourceName := range resourceNames {
 		args.Thresholds[resourceName] = lowThreshold
 		args.TargetThresholds[resourceName] = highThreshold

pkg/operator/target_config_reconciler_test.go

@@ -260,6 +260,33 @@ func TestManageConfigMap(t *testing.T) {
 				Data:     map[string]string{"policy.yaml": string(bindata.MustAsset("assets/lowNodeUtilizationHighConfig.yaml"))},
 			},
 		},
+		{
+			name: "RelieveAndMigrateWithoutCustomizations",
+			descheduler: &deschedulerv1.KubeDescheduler{
+				Spec: deschedulerv1.KubeDeschedulerSpec{
+					Profiles:              []deschedulerv1.DeschedulerProfile{"DevKubeVirtRelieveAndMigrate"},
+					ProfileCustomizations: nil,
+				},
+			},
+			want: &corev1.ConfigMap{
+				TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ConfigMap"},
+				Data:     map[string]string{"policy.yaml": string(bindata.MustAsset("assets/relieveAndMigrateDefaults.yaml"))},
+			},
+			nodes: []runtime.Object{
+				&corev1.Node{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:   "node1",
+						Labels: map[string]string{"kubevirt.io/schedulable": "true"},
+					},
+				},
+				&corev1.Node{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:   "node2",
+						Labels: map[string]string{"kubevirt.io/schedulable": "true"},
+					},
+				},
+			},
+		},
 		{
 			name: "RelieveAndMigrateLow",
 			descheduler: &deschedulerv1.KubeDescheduler{

pkg/operator/testdata/assets/relieveAndMigrateDefaults.yaml

@@ -0,0 +1,48 @@
+apiVersion: descheduler/v1alpha2
+kind: DeschedulerPolicy
+nodeSelector: kubevirt.io/schedulable=true
+profiles:
+- name: DevKubeVirtRelieveAndMigrate
+  pluginConfig:
+  - args:
+      evictableNamespaces:
+        exclude:
+        - kube-system
+        - hypershift
+        - openshift
+        - openshift-kube-descheduler-operator
+        - openshift-kube-scheduler
+      metricsUtilization:
+        prometheus:
+          query: descheduler:combined_utilization_and_pressure:avg1m
+        source: Prometheus
+      targetThresholds:
+        MetricResource: 10
+      thresholds:
+        MetricResource: 0
+      useDeviationThresholds: true
+    name: LowNodeUtilization
+  - args:
+      evictLocalStoragePods: true
+    name: DefaultEvictor
+  plugins:
+    balance:
+      disabled: null
+      enabled:
+      - LowNodeUtilization
+    deschedule:
+      disabled: null
+      enabled: null
+    filter:
+      disabled: null
+      enabled:
+      - DefaultEvictor
+    preevictionfilter:
+      disabled: null
+      enabled: null
+    presort:
+      disabled: null
+      enabled: null
+    sort:
+      disabled: null
+      enabled: null

pkg/operator/testdata/assets/relieveAndMigrateDynamicThresholdsHigh.yaml

@@ -12,14 +12,14 @@ profiles:
         - openshift
         - openshift-kube-descheduler-operator
         - openshift-kube-scheduler
+      metricsUtilization:
+        prometheus:
+          query: descheduler:combined_utilization_and_pressure:avg1m
+        source: Prometheus
       targetThresholds:
-        cpu: 30
-        memory: 30
-        pods: 30
+        MetricResource: 30
       thresholds:
-        cpu: 30
-        memory: 30
-        pods: 30
+        MetricResource: 30
       useDeviationThresholds: true
     name: LowNodeUtilization
   - args:

pkg/operator/testdata/assets/relieveAndMigrateDynamicThresholdsLow.yaml

@@ -12,14 +12,14 @@ profiles:
         - openshift
         - openshift-kube-descheduler-operator
         - openshift-kube-scheduler
+      metricsUtilization:
+        prometheus:
+          query: descheduler:combined_utilization_and_pressure:avg1m
+        source: Prometheus
       targetThresholds:
-        cpu: 10
-        memory: 10
-        pods: 10
+        MetricResource: 10
       thresholds:
-        cpu: 10
-        memory: 10
-        pods: 10
+        MetricResource: 10
       useDeviationThresholds: true
     name: LowNodeUtilization
   - args:

pkg/operator/testdata/assets/relieveAndMigrateDynamicThresholdsMedium.yaml

@@ -12,14 +12,14 @@ profiles:
         - openshift
         - openshift-kube-descheduler-operator
         - openshift-kube-scheduler
+      metricsUtilization:
+        prometheus:
+          query: descheduler:combined_utilization_and_pressure:avg1m
+        source: Prometheus
       targetThresholds:
-        cpu: 20
-        memory: 20
-        pods: 20
+        MetricResource: 20
       thresholds:
-        cpu: 20
-        memory: 20
-        pods: 20
+        MetricResource: 20
       useDeviationThresholds: true
     name: LowNodeUtilization
   - args:

pkg/operator/testdata/assets/relieveAndMigrateHighConfig.yaml

@@ -12,14 +12,14 @@ profiles:
         - openshift
         - openshift-kube-descheduler-operator
         - openshift-kube-scheduler
+      metricsUtilization:
+        prometheus:
+          query: descheduler:combined_utilization_and_pressure:avg1m
+        source: Prometheus
       targetThresholds:
-        cpu: 70
-        memory: 70
-        pods: 70
+        MetricResource: 70
       thresholds:
-        cpu: 40
-        memory: 40
-        pods: 40
+        MetricResource: 40
     name: LowNodeUtilization
   - args:
       evictLocalStoragePods: true

pkg/operator/testdata/assets/relieveAndMigrateIncludedNamespace.yaml

@@ -12,14 +12,15 @@ profiles:
         - openshift
         - openshift-kube-descheduler-operator
         - openshift-kube-scheduler
+      metricsUtilization:
+        prometheus:
+          query: descheduler:combined_utilization_and_pressure:avg1m
+        source: Prometheus
       targetThresholds:
-        cpu: 50
-        memory: 50
-        pods: 50
+        MetricResource: 10
       thresholds:
-        cpu: 20
-        memory: 20
-        pods: 20
+        MetricResource: 0
+      useDeviationThresholds: true
     name: LowNodeUtilization
   - args:
       evictLocalStoragePods: true

pkg/operator/testdata/assets/relieveAndMigrateLowConfig.yaml

@@ -12,14 +12,14 @@ profiles:
         - openshift
         - openshift-kube-descheduler-operator
         - openshift-kube-scheduler
+      metricsUtilization:
+        prometheus:
+          query: descheduler:combined_utilization_and_pressure:avg1m
+        source: Prometheus
       targetThresholds:
-        cpu: 30
-        memory: 30
-        pods: 30
+        MetricResource: 30
       thresholds:
-        cpu: 10
-        memory: 10
-        pods: 10
+        MetricResource: 10
     name: LowNodeUtilization
   - args:
       evictLocalStoragePods: true