[release-4.16] CNV-43973: HCP custom tolerations integration

bindata/cloud-network-config-controller/managed/controller.yaml

@@ -203,6 +203,11 @@ spec:
           readOnly: true
         terminationMessagePolicy: FallbackToLogsOnError
       tolerations:
+    {{- if .HCPTolerations }}
+      {{- range $t := .HCPTolerations }}
+      {{ $t }}
+      {{- end }}
+    {{- end }}
       - key: "hypershift.openshift.io/control-plane"
         operator: "Equal"
         value: "true"

bindata/network/multus-admission-controller/admission-controller.yaml

@@ -261,6 +261,11 @@ spec:
         runAsUser: {{.RunAsUser}}
 {{- end }}
       tolerations:
+{{- if .HCPTolerations }}
+        {{- range $t := .HCPTolerations }}
+        {{ $t }}
+        {{- end }}
+{{- end }}
         - key: "hypershift.openshift.io/control-plane"
           operator: "Equal"
           value: "true"

bindata/network/node-identity/managed/node-identity.yaml

@@ -276,6 +276,11 @@ spec:
               - key: additional-pod-admission-cond.json
                 path: additional-pod-admission-cond.json
       tolerations:
+      {{- if .HCPTolerations }}
+        {{- range $t := .HCPTolerations }}
+        {{ $t }}
+        {{- end }}
+      {{- end }}
         - key: "hypershift.openshift.io/control-plane"
           operator: "Equal"
           value: "true"

bindata/network/ovn-kubernetes/managed/ovnkube-control-plane.yaml

@@ -249,7 +249,6 @@ spec:
         - name: KUBECONFIG
           value: "/etc/kubernetes/kubeconfig"
         terminationMessagePolicy: FallbackToLogsOnError
-
       {{ if .HCPNodeSelector }}
       nodeSelector:
         {{ range $key, $value := .HCPNodeSelector }}
@@ -286,6 +285,11 @@ spec:
             - key: ca.crt
               path: ca.crt
       tolerations:
+      {{- if .HCPTolerations }}
+        {{- range $t := .HCPTolerations }}
+        {{ $t }}
+        {{- end }}
+      {{- end }}
         - key: "hypershift.openshift.io/control-plane"
           operator: "Equal"
           value: "true"

pkg/bootstrap/types.go

@@ -11,6 +11,7 @@ type OVNHyperShiftBootstrapResult struct {
 	ClusterID            string
 	Namespace            string
 	HCPNodeSelector      map[string]string
+	HCPTolerations       []string
 	ControlPlaneReplicas int
 	ReleaseImage         string
 	ControlPlaneImage    string

pkg/hypershift/hypershift.go

@@ -10,6 +10,8 @@ import (
 
 	configv1 "github.com/openshift/api/config/v1"
 	operv1 "github.com/openshift/api/operator/v1"
+	"gopkg.in/yaml.v2"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -56,6 +58,7 @@ type HostedControlPlane struct {
 	ClusterID                    string
 	ControllerAvailabilityPolicy AvailabilityPolicy
 	NodeSelector                 map[string]string
+	Tolerations                  []string
 	AdvertiseAddress             string
 	AdvertisePort                int
 	PriorityClass                string
@@ -152,6 +155,64 @@ func ParseHostedControlPlane(hcp *unstructured.Unstructured) (*HostedControlPlan
 		return nil, fmt.Errorf("failed extract nodeSelector: %v", err)
 	}
 
+	var tolerations []corev1.Toleration
+	var tolerationsYaml []string
+	tolerationsArray, tolerationsArrayFound, err := unstructured.NestedFieldCopy(hcp.UnstructuredContent(), "spec", "tolerations")
+	if err != nil {
+		return nil, fmt.Errorf("failed extract tolerations: %v", err)
+	}
+	if tolerationsArrayFound {
+		tolerationsArrayConverted, hasConverted := tolerationsArray.([]interface{})
+		if hasConverted {
+			for _, entry := range tolerationsArrayConverted {
+				tolerationConverted, hasConverted := entry.(map[string]interface{})
+				if hasConverted {
+					toleration := corev1.Toleration{}
+					raw, ok := tolerationConverted["key"]
+					if ok {
+						str, isString := raw.(string)
+						if isString {
+							toleration.Key = str
+						}
+					}
+					raw, ok = tolerationConverted["operator"]
+					if ok {
+						op, isOperator := raw.(string)
+						if isOperator {
+							toleration.Operator = corev1.TolerationOperator(op)
+						}
+					}
+					raw, ok = tolerationConverted["value"]
+					if ok {
+						str, isString := raw.(string)
+						if isString {
+							toleration.Value = str
+						}
+					}
+					raw, ok = tolerationConverted["effect"]
+					if ok {
+						effect, isEffect := raw.(string)
+						if isEffect {
+							toleration.Effect = corev1.TaintEffect(effect)
+						}
+					}
+					raw, ok = tolerationConverted["tolerationSeconds"]
+					if ok {
+						seconds, isSeconds := raw.(*int64)
+						if isSeconds {
+							toleration.TolerationSeconds = seconds
+						}
+					}
+					tolerations = append(tolerations, toleration)
+				}
+			}
+		}
+		tolerationsYaml, err = tolerationsToStringSliceYaml(tolerations)
+		if err != nil {
+			return nil, fmt.Errorf("failed to yaml marshal tolerations: %v", err)
+		}
+	}
+
 	advertiseAddress, valueFound, err := unstructured.NestedString(hcp.UnstructuredContent(), "spec", "networking", "apiServer", "advertiseAddress")
 	if err != nil {
 		return nil, fmt.Errorf("failed extract advertiseAddress: %v", err)
@@ -192,6 +253,7 @@ func ParseHostedControlPlane(hcp *unstructured.Unstructured) (*HostedControlPlan
 		ControllerAvailabilityPolicy: AvailabilityPolicy(controllerAvailabilityPolicy),
 		ClusterID:                    clusterID,
 		NodeSelector:                 nodeSelector,
+		Tolerations:                  tolerationsYaml,
 		AdvertiseAddress:             advertiseAddress,
 		AdvertisePort:                int(advertisePort),
 		PriorityClass:                controlPlanePriorityClassAnnotation,
@@ -252,3 +314,29 @@ func SetHostedControlPlaneConditions(hcp *unstructured.Unstructured, operStatus
 	hcp.Object["status"].(map[string]interface{})["conditions"] = conditions
 	return conditions, nil
 }
+
+// tolerationsToStringSliceYaml converts a slice of tolerations into a slice of
+// strings that represent the toleration in yaml syntax where each string
+// is a line of yaml. The resulting string slice can be easily used in
+// yaml manifest templating.
+func tolerationsToStringSliceYaml(tolerations []corev1.Toleration) ([]string, error) {
+	if len(tolerations) == 0 {
+		return nil, nil
+	}
+
+	yamlBytes, err := yaml.Marshal(tolerations)
+	if err != nil {
+		return nil, err
+	}
+
+	yamlStrs := []string{}
+	for _, arg := range strings.Split(string(yamlBytes), "\n") {
+
+		// filter out null and empty strings
+		if strings.Contains(arg, ": null") || strings.Contains(arg, ": \"\"") {
+			continue
+		}
+		yamlStrs = append(yamlStrs, arg)
+	}
+	return yamlStrs, nil
+}

pkg/network/cloud_network.go

@@ -90,6 +90,7 @@ func renderCloudNetworkConfigController(conf *operv1.NetworkSpec, bootstrapResul
 		data.Data["HostedClusterNamespace"] = hcpCfg.Namespace
 		data.Data["ReleaseImage"] = hcpCfg.ReleaseImage
 		data.Data["HCPNodeSelector"] = cloudBootstrapResult.HostedControlPlane.NodeSelector
+		data.Data["HCPTolerations"] = cloudBootstrapResult.HostedControlPlane.Tolerations
 		data.Data["RunAsUser"] = hcpCfg.RunAsUser
 		// In HyperShift CloudNetworkConfigController is deployed as a part of the hosted cluster controlplane
 		// which means that it is created in the management cluster.

pkg/network/multus_admission_controller.go

@@ -110,6 +110,7 @@ func renderMultusAdmissonControllerConfig(manifestDir string, externalControlPla
 		data.Data["ClusterIDLabel"] = hypershift.ClusterIDLabel
 		data.Data["ClusterID"] = bootstrapResult.Infra.HostedControlPlane.ClusterID
 		data.Data["HCPNodeSelector"] = bootstrapResult.Infra.HostedControlPlane.NodeSelector
+		data.Data["HCPTolerations"] = bootstrapResult.Infra.HostedControlPlane.Tolerations
 		data.Data["PriorityClass"] = bootstrapResult.Infra.HostedControlPlane.PriorityClass
 
 		// Preserve any existing multus container resource requests which may have been modified by an external source

pkg/network/node_identity.go

@@ -97,6 +97,8 @@ func renderNetworkNodeIdentity(conf *operv1.NetworkSpec, bootstrapResult *bootst
 		data.Data["TokenMinterImage"] = os.Getenv("TOKEN_MINTER_IMAGE")
 		data.Data["TokenAudience"] = os.Getenv("TOKEN_AUDIENCE")
 		data.Data["HCPNodeSelector"] = bootstrapResult.Infra.HostedControlPlane.NodeSelector
+		data.Data["HCPTolerations"] = bootstrapResult.Infra.HostedControlPlane.Tolerations
+
 		data.Data["NetworkNodeIdentityImage"] = hcpCfg.ControlPlaneImage // OVN_CONTROL_PLANE_IMAGE
 		localAPIServer := bootstrapResult.Infra.APIServers[bootstrap.APIServerDefaultLocal]
 		data.Data["K8S_LOCAL_APISERVER"] = "https://" + net.JoinHostPort(localAPIServer.Host, localAPIServer.Port)

pkg/network/ovn_kubernetes.go

@@ -218,6 +218,7 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo
 	data.Data["ClusterID"] = bootstrapResult.OVN.OVNKubernetesConfig.HyperShiftConfig.ClusterID
 	data.Data["ClusterIDLabel"] = hypershift.ClusterIDLabel
 	data.Data["HCPNodeSelector"] = bootstrapResult.OVN.OVNKubernetesConfig.HyperShiftConfig.HCPNodeSelector
+	data.Data["HCPTolerations"] = bootstrapResult.OVN.OVNKubernetesConfig.HyperShiftConfig.HCPTolerations
 	data.Data["OVN_NB_INACTIVITY_PROBE"] = nb_inactivity_probe
 	data.Data["OVN_CERT_CN"] = OVN_CERT_CN
 	data.Data["OVN_NORTHD_PROBE_INTERVAL"] = os.Getenv("OVN_NORTHD_PROBE_INTERVAL")
@@ -710,6 +711,8 @@ func bootstrapOVNHyperShiftConfig(hc *hypershift.HyperShiftConfig, kubeClient cn
 
 	ovnHypershiftResult.ClusterID = hcp.ClusterID
 	ovnHypershiftResult.HCPNodeSelector = hcp.NodeSelector
+	ovnHypershiftResult.HCPTolerations = hcp.Tolerations
+
 	switch hcp.ControllerAvailabilityPolicy {
 	case hypershift.HighlyAvailable:
 		ovnHypershiftResult.ControlPlaneReplicas = 3