[feat] Add default affinities to ensure pods:

1. prefer to be in different zones (soft rule)
2. must not be scheduled on the same node

Author: jiangpengcheng

charts/function-mesh-operator/templates/controller-manager-deployment.yaml

@@ -57,14 +57,15 @@ spec:
         args:
           - --enable-leader-election={{ .Values.controllerManager.enableLeaderElection }}
           - --enable-pprof={{ .Values.controllerManager.pprof.enable }}
-          - --metrics-addr=:{{ .Values.controllerManager.metrics.port }}
-          - --health-probe-addr=:{{ .Values.controllerManager.healthProbe.port }}
-          - --pprof-addr=:{{ .Values.controllerManager.pprof.port }}
+          - --metrics-addr={{ .Values.controllerManager.metrics.port }}
+          - --health-probe-addr={{ .Values.controllerManager.healthProbe.port }}
+          - --pprof-addr={{ .Values.controllerManager.pprof.port }}
           - --config-file={{ .Values.controllerManager.configFile }}
           - --enable-init-containers={{ .Values.controllerManager.enableInitContainers }}
           - --global-backend-config={{ .Values.controllerManager.globalBackendConfig }}
           - --global-backend-config-namespace={{ .Values.controllerManager.globalBackendConfigNamespace }}
           - --namespaced-backend-config={{ .Values.controllerManager.namespacedBackendConfig }}
+          - --add-default-affinity={{ .Values.controllerManager.addDefaultAffinity }}
         env:
           - name: NAMESPACE
             valueFrom:

charts/function-mesh-operator/values.yaml

@@ -73,6 +73,7 @@ controllerManager:
   globalBackendConfig: global-backend-config
   globalBackendConfigNamespace: default
   namespacedBackendConfig: backend-config
+  addDefaultAffinity: true
 
 admissionWebhook:
   enabled: true

controllers/spec/common.go

@@ -463,9 +463,10 @@ func makePodTemplate(container *corev1.Container, filebeatContainer *corev1.Cont
 	if filebeatContainer != nil {
 		containers = append(containers, *filebeatContainer)
 	}
+	mergedLabels := mergeMaps(labels, Configs.ResourceLabels, policy.Labels)
 	return &corev1.PodTemplateSpec{
 		ObjectMeta: metav1.ObjectMeta{
-			Labels:      mergeMaps(labels, Configs.ResourceLabels, policy.Labels),
+			Labels:      mergedLabels,
 			Annotations: generateAnnotations(Configs.ResourceAnnotations, policy.Annotations),
 		},
 		Spec: corev1.PodSpec{
@@ -474,7 +475,7 @@ func makePodTemplate(container *corev1.Container, filebeatContainer *corev1.Cont
 			TerminationGracePeriodSeconds: policy.TerminationGracePeriodSeconds,
 			Volumes:                       volumes,
 			NodeSelector:                  policy.NodeSelector,
-			Affinity:                      policy.Affinity,
+			Affinity:                      GenerateAffinity(policy.Affinity, mergedLabels),
 			Tolerations:                   policy.Tolerations,
 			SecurityContext:               podSecurityContext,
 			ImagePullSecrets:              policy.ImagePullSecrets,
@@ -483,6 +484,50 @@ func makePodTemplate(container *corev1.Container, filebeatContainer *corev1.Cont
 	}
 }
 
+func GenerateAffinity(affinity *corev1.Affinity, labels map[string]string) *corev1.Affinity {
+	if !utils.AddDefaultAffinity {
+		return affinity
+	}
+	if affinity == nil {
+		affinity = &corev1.Affinity{}
+	}
+	if affinity.PodAntiAffinity == nil {
+		affinity.PodAntiAffinity = &corev1.PodAntiAffinity{}
+	}
+
+	// add default pod anti-affinity rules to ensure replica pods doesn't run on the same node
+	if affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil {
+		affinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution = []corev1.PodAffinityTerm{}
+	}
+	affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append(
+		affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution,
+		corev1.PodAffinityTerm{
+			LabelSelector: &metav1.LabelSelector{
+				MatchLabels: labels,
+			},
+			TopologyKey: "kubernetes.io/hostname",
+		},
+	)
+
+	// add default pod anti-affinity rules to ensure replica pods prefer running in different zones
+	if affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution == nil {
+		affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution = []corev1.WeightedPodAffinityTerm{}
+	}
+	affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append(
+		affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution,
+		corev1.WeightedPodAffinityTerm{
+			Weight: 100,
+			PodAffinityTerm: corev1.PodAffinityTerm{
+				LabelSelector: &metav1.LabelSelector{
+					MatchLabels: labels,
+				},
+				TopologyKey: "topology.kubernetes.io/zone",
+			},
+		},
+	)
+	return affinity
+}
+
 func MakeJavaFunctionCommand(downloadPath, packageFile, name, clusterName, generateLogConfigCommand, logLevel, details, extraDependenciesDir, uid string,
 	memory *resource.Quantity, javaOpts []string, hasPulsarctl, hasWget, authProvided, tlsProvided bool, secretMaps map[string]v1alpha1.SecretRef,
 	state *v1alpha1.Stateful,

main.go

@@ -79,6 +79,7 @@ func main() {
 	var globalBackendConfigNamespace string
 	var namespacedBackendConfig string
 	var secureMetrics bool
+	var addDefaultAffinity bool
 	flag.StringVar(&metricsAddr, "metrics-addr", lookupEnvOrString("METRICS_ADDR", ":8443"),
 		"The address the metric endpoint binds to.")
 	flag.StringVar(&leaderElectionID, "leader-election-id",
@@ -112,13 +113,16 @@ func main() {
 		"The backend config name used for functions&sinks&sources in the same namespace")
 	flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS."+
 		" Use --metrics-secure=false to use HTTP instead.")
+	flag.BoolVar(&addDefaultAffinity, "add-default-affinity", lookupEnvOrBool("ADD_DEFAULT_AFFINITY", true), "If set, the generated pod will add two default podAntiAffinities:"+
+		" one is make pods prefer to be in different zones (soft rule), and one is make pods must not be scheduled on the same node (hard rule).")
 	flag.Parse()
 
 	ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
 	utils.EnableInitContainers = enableInitContainers
 	utils.GlobalBackendConfig = globalBackendConfig
 	utils.GlobalBackendConfigNamespace = globalBackendConfigNamespace
 	utils.NamespacedBackendConfig = namespacedBackendConfig
+	utils.AddDefaultAffinity = addDefaultAffinity
 
 	// enable pprof
 	if enablePprof {

utils/configs.go

@@ -22,3 +22,4 @@ var EnableInitContainers = false
 var GlobalBackendConfig = ""
 var GlobalBackendConfigNamespace = "default"
 var NamespacedBackendConfig = ""
+var AddDefaultAffinity = true