WIP: add unit test on rendered objects

bindata/cloud-network-config-controller/managed/controller.yaml

@@ -202,6 +202,11 @@ spec:
           readOnly: true
         terminationMessagePolicy: FallbackToLogsOnError
       tolerations:
+    {{- if .HCPTolerations }}
+      {{- range $t := .HCPTolerations }}
+      {{ $t }}
+      {{- end }}
+    {{- end }}
       - key: "hypershift.openshift.io/control-plane"
         operator: "Equal"
         value: "true"

bindata/network/multus-admission-controller/admission-controller.yaml

@@ -259,6 +259,11 @@ spec:
         runAsUser: {{.RunAsUser}}
 {{- end }}
       tolerations:
+{{- if .HCPTolerations }}
+        {{- range $t := .HCPTolerations }}
+        {{ $t }}
+        {{- end }}
+{{- end }}
         - key: "hypershift.openshift.io/control-plane"
           operator: "Equal"
           value: "true"

bindata/network/node-identity/managed/node-identity.yaml

@@ -273,6 +273,11 @@ spec:
               - key: additional-pod-admission-cond.json
                 path: additional-pod-admission-cond.json
       tolerations:
+      {{- if .HCPTolerations }}
+        {{- range $t := .HCPTolerations }}
+        {{ $t }}
+        {{- end }}
+      {{- end }}
         - key: "hypershift.openshift.io/control-plane"
           operator: "Equal"
           value: "true"

bindata/network/ovn-kubernetes/managed/ovnkube-control-plane.yaml

@@ -264,7 +264,6 @@ spec:
         - name: KUBECONFIG
           value: "/etc/kubernetes/kubeconfig"
         terminationMessagePolicy: FallbackToLogsOnError
-
       {{ if .HCPNodeSelector }}
       nodeSelector:
         {{ range $key, $value := .HCPNodeSelector }}
@@ -301,6 +300,11 @@ spec:
             - key: ca.crt
               path: ca.crt
       tolerations:
+      {{- if .HCPTolerations }}
+        {{- range $t := .HCPTolerations }}
+        {{ $t }}
+        {{- end }}
+      {{- end }}
         - key: "hypershift.openshift.io/control-plane"
           operator: "Equal"
           value: "true"

pkg/bootstrap/types.go

@@ -11,6 +11,7 @@ type OVNHyperShiftBootstrapResult struct {
 	ClusterID            string
 	Namespace            string
 	HCPNodeSelector      map[string]string
+	HCPTolerations       []string
 	ControlPlaneReplicas int
 	ReleaseImage         string
 	ControlPlaneImage    string

pkg/hypershift/hypershift.go

@@ -10,6 +10,8 @@ import (
 
 	configv1 "github.com/openshift/api/config/v1"
 	operv1 "github.com/openshift/api/operator/v1"
+	"gopkg.in/yaml.v2"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -56,6 +58,7 @@ type HostedControlPlane struct {
 	ClusterID                    string
 	ControllerAvailabilityPolicy AvailabilityPolicy
 	NodeSelector                 map[string]string
+	Tolerations                  []corev1.Toleration
 	AdvertiseAddress             string
 	AdvertisePort                int
 	PriorityClass                string
@@ -152,6 +155,56 @@ func ParseHostedControlPlane(hcp *unstructured.Unstructured) (*HostedControlPlan
 		return nil, fmt.Errorf("failed extract nodeSelector: %v", err)
 	}
 
+	var tolerations []corev1.Toleration
+	tolerationsArray, tolerationsArrayFound, err := unstructured.NestedFieldCopy(hcp.UnstructuredContent(), "spec", "tolerations")
+	if tolerationsArrayFound {
+		tolerationsArrayConverted, hasConverted := tolerationsArray.([]interface{})
+		if hasConverted {
+			for _, entry := range tolerationsArrayConverted {
+				tolerationConverted, hasConverted := entry.(map[string]interface{})
+				if hasConverted {
+					toleration := corev1.Toleration{}
+					raw, ok := tolerationConverted["key"]
+					if ok {
+						str, isString := raw.(string)
+						if isString {
+							toleration.Key = str
+						}
+					}
+					raw, ok = tolerationConverted["operator"]
+					if ok {
+						op, isOperator := raw.(string)
+						if isOperator {
+							toleration.Operator = corev1.TolerationOperator(op)
+						}
+					}
+					raw, ok = tolerationConverted["value"]
+					if ok {
+						str, isString := raw.(string)
+						if isString {
+							toleration.Value = str
+						}
+					}
+					raw, ok = tolerationConverted["effect"]
+					if ok {
+						effect, isEffect := raw.(string)
+						if isEffect {
+							toleration.Effect = corev1.TaintEffect(effect)
+						}
+					}
+					raw, ok = tolerationConverted["tolerationSeconds"]
+					if ok {
+						seconds, isSeconds := raw.(*int64)
+						if isSeconds {
+							toleration.TolerationSeconds = seconds
+						}
+					}
+					tolerations = append(tolerations, toleration)
+				}
+			}
+		}
+	}
+
 	advertiseAddress, valueFound, err := unstructured.NestedString(hcp.UnstructuredContent(), "spec", "networking", "apiServer", "advertiseAddress")
 	if err != nil {
 		return nil, fmt.Errorf("failed extract advertiseAddress: %v", err)
@@ -192,6 +245,7 @@ func ParseHostedControlPlane(hcp *unstructured.Unstructured) (*HostedControlPlan
 		ControllerAvailabilityPolicy: AvailabilityPolicy(controllerAvailabilityPolicy),
 		ClusterID:                    clusterID,
 		NodeSelector:                 nodeSelector,
+		Tolerations:                  tolerations,
 		AdvertiseAddress:             advertiseAddress,
 		AdvertisePort:                int(advertisePort),
 		PriorityClass:                controlPlanePriorityClassAnnotation,
@@ -252,3 +306,21 @@ func SetHostedControlPlaneConditions(hcp *unstructured.Unstructured, operStatus
 	hcp.Object["status"].(map[string]interface{})["conditions"] = conditions
 	return conditions, nil
 }
+
+func TolerationsToStringArray(tolerations []corev1.Toleration) ([]string, error) {
+	yamlBytes, err := yaml.Marshal(tolerations)
+	if err != nil {
+		return nil, err
+	}
+
+	yamlStrs := []string{}
+	for _, arg := range strings.Split(string(yamlBytes), "\n") {
+
+		// filter out null and empty strings
+		if strings.Contains(arg, ": null") || strings.Contains(arg, ": \"\"") {
+			continue
+		}
+		yamlStrs = append(yamlStrs, arg)
+	}
+	return yamlStrs, nil
+}

pkg/network/cloud_network.go

@@ -83,6 +83,16 @@ func renderCloudNetworkConfigController(conf *operv1.NetworkSpec, bootstrapResul
 	manifestDirs := make([]string, 0, 2)
 	manifestDirs = append(manifestDirs, filepath.Join(manifestDir, "cloud-network-config-controller/common"))
 	if hcpCfg := hypershift.NewHyperShiftConfig(); hcpCfg.Enabled {
+		if len(cloudBootstrapResult.HostedControlPlane.Tolerations) != 0 {
+			tolerations, err := hypershift.TolerationsToStringArray(cloudBootstrapResult.HostedControlPlane.Tolerations)
+			if err != nil {
+				return nil, err
+			}
+			data.Data["HCPTolerations"] = tolerations
+		} else {
+			data.Data["HCPTolerations"] = ""
+		}
+
 		data.Data["CLIImage"] = os.Getenv("CLI_IMAGE")
 		data.Data["TokenMinterImage"] = os.Getenv("TOKEN_MINTER_IMAGE")
 		data.Data["TokenAudience"] = os.Getenv("TOKEN_AUDIENCE")

pkg/network/multus_admission_controller.go

@@ -112,6 +112,16 @@ func renderMultusAdmissonControllerConfig(manifestDir string, externalControlPla
 		data.Data["HCPNodeSelector"] = bootstrapResult.Infra.HostedControlPlane.NodeSelector
 		data.Data["PriorityClass"] = bootstrapResult.Infra.HostedControlPlane.PriorityClass
 
+		if len(bootstrapResult.Infra.HostedControlPlane.Tolerations) != 0 {
+			tolerations, err := hypershift.TolerationsToStringArray(bootstrapResult.Infra.HostedControlPlane.Tolerations)
+			if err != nil {
+				return nil, err
+			}
+			data.Data["HCPTolerations"] = tolerations
+		} else {
+			data.Data["HCPTolerations"] = ""
+		}
+
 		// Preserve any existing multus container resource requests which may have been modified by an external source
 		multusDeploy := &appsv1.Deployment{}
 		err = client.ClientFor(clientName).CRClient().Get(

pkg/network/node_identity.go

@@ -97,6 +97,17 @@ func renderNetworkNodeIdentity(conf *operv1.NetworkSpec, bootstrapResult *bootst
 		data.Data["TokenMinterImage"] = os.Getenv("TOKEN_MINTER_IMAGE")
 		data.Data["TokenAudience"] = os.Getenv("TOKEN_AUDIENCE")
 		data.Data["HCPNodeSelector"] = bootstrapResult.Infra.HostedControlPlane.NodeSelector
+
+		if len(bootstrapResult.Infra.HostedControlPlane.Tolerations) != 0 {
+			tolerations, err := hypershift.TolerationsToStringArray(bootstrapResult.Infra.HostedControlPlane.Tolerations)
+			if err != nil {
+				return nil, err
+			}
+			data.Data["HCPTolerations"] = tolerations
+		} else {
+			data.Data["HCPTolerations"] = ""
+		}
+
 		data.Data["NetworkNodeIdentityImage"] = hcpCfg.ControlPlaneImage // OVN_CONTROL_PLANE_IMAGE
 		localAPIServer := bootstrapResult.Infra.APIServers[bootstrap.APIServerDefaultLocal]
 		data.Data["K8S_LOCAL_APISERVER"] = "https://" + net.JoinHostPort(localAPIServer.Host, localAPIServer.Port)

pkg/network/ovn_kubernetes.go

@@ -215,6 +215,11 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo
 	data.Data["ClusterID"] = bootstrapResult.OVN.OVNKubernetesConfig.HyperShiftConfig.ClusterID
 	data.Data["ClusterIDLabel"] = hypershift.ClusterIDLabel
 	data.Data["HCPNodeSelector"] = bootstrapResult.OVN.OVNKubernetesConfig.HyperShiftConfig.HCPNodeSelector
+	if len(bootstrapResult.OVN.OVNKubernetesConfig.HyperShiftConfig.HCPTolerations) != 0 {
+		data.Data["HCPTolerations"] = bootstrapResult.OVN.OVNKubernetesConfig.HyperShiftConfig.HCPTolerations
+	} else {
+		data.Data["HCPTolerations"] = ""
+	}
 	data.Data["OVN_NB_INACTIVITY_PROBE"] = nb_inactivity_probe
 	data.Data["OVN_CERT_CN"] = OVN_CERT_CN
 	data.Data["OVN_NORTHD_PROBE_INTERVAL"] = os.Getenv("OVN_NORTHD_PROBE_INTERVAL")
@@ -703,6 +708,13 @@ func bootstrapOVNHyperShiftConfig(hc *hypershift.HyperShiftConfig, kubeClient cn
 
 	ovnHypershiftResult.ClusterID = hcp.ClusterID
 	ovnHypershiftResult.HCPNodeSelector = hcp.NodeSelector
+
+	tolerations, err := hypershift.TolerationsToStringArray(hcp.Tolerations)
+	if err != nil {
+		return nil, err
+	}
+	ovnHypershiftResult.HCPTolerations = tolerations
+
 	switch hcp.ControllerAvailabilityPolicy {
 	case hypershift.HighlyAvailable:
 		ovnHypershiftResult.ControlPlaneReplicas = 3

pkg/network/render_test.go

@@ -2,6 +2,9 @@ package network
 
 import (
 	"fmt"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"reflect"
 	"strings"
 
@@ -81,6 +84,133 @@ func TestClusterNetworkChangeOkOnMigration(t *testing.T) {
 
 }
 
+// HostedControlPlane tolerations
+// =================================
+func TestHyperShiftTolerations(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	config := operv1.Network{
+		Spec: operv1.NetworkSpec{
+			ServiceNetwork: []string{"172.30.0.0/16"},
+			ClusterNetwork: []operv1.ClusterNetworkEntry{
+				{
+					CIDR:       "10.128.0.0/15",
+					HostPrefix: 23,
+				},
+				{
+					CIDR:       "10.0.0.0/14",
+					HostPrefix: 24,
+				},
+			},
+			DefaultNetwork: operv1.DefaultNetworkDefinition{
+				Type: "MyAwesomeThirdPartyPlugin",
+			},
+		},
+	}
+
+	// Bootstrap a client with an infrastructure object
+	if err := configv1.AddToScheme(scheme.Scheme); err != nil {
+		t.Fatalf("failed to add configv1 to scheme: %v", err)
+	}
+	infrastructure := &configv1.Infrastructure{
+		ObjectMeta: metav1.ObjectMeta{Name: "cluster"},
+		Status: configv1.InfrastructureStatus{
+			PlatformStatus: &configv1.PlatformStatus{},
+		},
+	}
+
+	client := fake.NewFakeClient(infrastructure)
+	err := createProxy(client)
+	g.Expect(err).NotTo(HaveOccurred())
+
+	err = Validate(&config.Spec)
+	g.Expect(err).NotTo(HaveOccurred())
+
+	prev := config.Spec.DeepCopy()
+	fillDefaults(prev, nil)
+	next := config.Spec.DeepCopy()
+	fillDefaults(next, nil)
+
+	err = IsChangeSafe(prev, next, &fakeBootstrapResult().Infra)
+	g.Expect(err).NotTo(HaveOccurred())
+
+	featureGatesCNO := featuregates.NewFeatureGate([]configv1.FeatureGateName{}, []configv1.FeatureGateName{})
+
+	hcpTolerations := []corev1.Toleration{
+		{
+			Key:               "node.kubernetes.io/not-ready",
+			Operator:          "Exists",
+			Value:             "",
+			Effect:            "NoExecute",
+			TolerationSeconds: nil,
+		},
+		{
+			Key:               "node.kubernetes.io/unreachable",
+			Operator:          "Exists",
+			Value:             "",
+			Effect:            "NoExecute",
+			TolerationSeconds: nil,
+		},
+		{
+			Key:               "node.kubernetes.io/memory-pressure",
+			Operator:          "Exists",
+			Value:             "",
+			Effect:            "NoSchedule",
+			TolerationSeconds: nil,
+		},
+		{
+			Key:               "key1",
+			Operator:          "Equal",
+			Value:             "value1",
+			Effect:            "NoSchedule",
+			TolerationSeconds: nil,
+		},
+		{
+			Key:               "key1",
+			Operator:          "Exists",
+			Value:             "",
+			Effect:            "NoSchedule",
+			TolerationSeconds: nil,
+		},
+	}
+
+	// fake that we are in HyperShift hosted cluster
+	bootstrapResult := fakeBootstrapResult()
+	bootstrapResult.Infra = bootstrap.InfraStatus{}
+	bootstrapResult.Infra.HostedControlPlane = &hypershift.HostedControlPlane{
+		ClusterID:                    "",
+		ControllerAvailabilityPolicy: "",
+		NodeSelector: map[string]string{
+			"kubernetes.io/os": "linux",
+		},
+		Tolerations:      hcpTolerations,
+		AdvertiseAddress: "",
+		AdvertisePort:    0,
+		PriorityClass:    "",
+	}
+
+	err = Validate(&config.Spec)
+	g.Expect(err).NotTo(HaveOccurred())
+
+	objs, _, err := Render(prev, &configv1.NetworkSpec{}, manifestDir, client, featureGatesCNO, bootstrapResult)
+	g.Expect(err).NotTo(HaveOccurred())
+
+	for _, obj := range objs {
+		if obj.GetKind() == "Deployment" {
+			deployment := &appsv1.Deployment{}
+			err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, deployment)
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(deployment.Spec.Template.Spec.Tolerations).To(ContainElements(hcpTolerations))
+		} else if obj.GetKind() == "DaemonSet" && obj.GetName() != "multus" {
+			daemonset := &appsv1.DaemonSet{}
+			err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, daemonset)
+			g.Expect(daemonset.Spec.Template.Spec.Tolerations).To(ContainElements(hcpTolerations))
+		}
+
+	}
+
+}
+
 // Invalid miscellaneous migration validation
 // =================================
 func TestServiceNetworkChangeNotOkOnMigration(t *testing.T) {