diff --git a/bindata/cloud-network-config-controller/managed/controller.yaml b/bindata/cloud-network-config-controller/managed/controller.yaml index 9f0e943de8..eb79e1490f 100644 --- a/bindata/cloud-network-config-controller/managed/controller.yaml +++ b/bindata/cloud-network-config-controller/managed/controller.yaml @@ -203,6 +203,11 @@ spec: readOnly: true terminationMessagePolicy: FallbackToLogsOnError tolerations: + {{- if .HCPTolerations }} + {{- range $t := .HCPTolerations }} + {{ $t }} + {{- end }} + {{- end }} - key: "hypershift.openshift.io/control-plane" operator: "Equal" value: "true" diff --git a/bindata/network/multus-admission-controller/admission-controller.yaml b/bindata/network/multus-admission-controller/admission-controller.yaml index c4c4c779b4..4ff5778611 100644 --- a/bindata/network/multus-admission-controller/admission-controller.yaml +++ b/bindata/network/multus-admission-controller/admission-controller.yaml @@ -261,6 +261,11 @@ spec: runAsUser: {{.RunAsUser}} {{- end }} tolerations: +{{- if .HCPTolerations }} + {{- range $t := .HCPTolerations }} + {{ $t }} + {{- end }} +{{- end }} - key: "hypershift.openshift.io/control-plane" operator: "Equal" value: "true" diff --git a/bindata/network/node-identity/managed/node-identity.yaml b/bindata/network/node-identity/managed/node-identity.yaml index 01d78e232b..276ebc1792 100644 --- a/bindata/network/node-identity/managed/node-identity.yaml +++ b/bindata/network/node-identity/managed/node-identity.yaml @@ -276,6 +276,11 @@ spec: - key: additional-pod-admission-cond.json path: additional-pod-admission-cond.json tolerations: + {{- if .HCPTolerations }} + {{- range $t := .HCPTolerations }} + {{ $t }} + {{- end }} + {{- end }} - key: "hypershift.openshift.io/control-plane" operator: "Equal" value: "true" diff --git a/bindata/network/ovn-kubernetes/managed/ovnkube-control-plane.yaml b/bindata/network/ovn-kubernetes/managed/ovnkube-control-plane.yaml index 11210a85ae..86811ef39b 100644 --- a/bindata/network/ovn-kubernetes/managed/ovnkube-control-plane.yaml +++ b/bindata/network/ovn-kubernetes/managed/ovnkube-control-plane.yaml @@ -249,7 +249,6 @@ spec: - name: KUBECONFIG value: "/etc/kubernetes/kubeconfig" terminationMessagePolicy: FallbackToLogsOnError - {{ if .HCPNodeSelector }} nodeSelector: {{ range $key, $value := .HCPNodeSelector }} @@ -286,6 +285,11 @@ spec: - key: ca.crt path: ca.crt tolerations: + {{- if .HCPTolerations }} + {{- range $t := .HCPTolerations }} + {{ $t }} + {{- end }} + {{- end }} - key: "hypershift.openshift.io/control-plane" operator: "Equal" value: "true" diff --git a/pkg/bootstrap/types.go b/pkg/bootstrap/types.go index c0a47d0e08..a15dd4e130 100644 --- a/pkg/bootstrap/types.go +++ b/pkg/bootstrap/types.go @@ -11,6 +11,7 @@ type OVNHyperShiftBootstrapResult struct { ClusterID string Namespace string HCPNodeSelector map[string]string + HCPTolerations []string ControlPlaneReplicas int ReleaseImage string ControlPlaneImage string diff --git a/pkg/hypershift/hypershift.go b/pkg/hypershift/hypershift.go index 4b9b4446c3..b6c1740e8b 100644 --- a/pkg/hypershift/hypershift.go +++ b/pkg/hypershift/hypershift.go @@ -10,6 +10,8 @@ import ( configv1 "github.com/openshift/api/config/v1" operv1 "github.com/openshift/api/operator/v1" + "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -56,6 +58,7 @@ type HostedControlPlane struct { ClusterID string ControllerAvailabilityPolicy AvailabilityPolicy NodeSelector map[string]string + Tolerations []string AdvertiseAddress string AdvertisePort int PriorityClass string @@ -152,6 +155,64 @@ func ParseHostedControlPlane(hcp *unstructured.Unstructured) (*HostedControlPlan return nil, fmt.Errorf("failed extract nodeSelector: %v", err) } + var tolerations []corev1.Toleration + var tolerationsYaml []string + tolerationsArray, tolerationsArrayFound, err := unstructured.NestedFieldCopy(hcp.UnstructuredContent(), "spec", "tolerations") + if err != nil { + return nil, fmt.Errorf("failed extract tolerations: %v", err) + } + if tolerationsArrayFound { + tolerationsArrayConverted, hasConverted := tolerationsArray.([]interface{}) + if hasConverted { + for _, entry := range tolerationsArrayConverted { + tolerationConverted, hasConverted := entry.(map[string]interface{}) + if hasConverted { + toleration := corev1.Toleration{} + raw, ok := tolerationConverted["key"] + if ok { + str, isString := raw.(string) + if isString { + toleration.Key = str + } + } + raw, ok = tolerationConverted["operator"] + if ok { + op, isOperator := raw.(string) + if isOperator { + toleration.Operator = corev1.TolerationOperator(op) + } + } + raw, ok = tolerationConverted["value"] + if ok { + str, isString := raw.(string) + if isString { + toleration.Value = str + } + } + raw, ok = tolerationConverted["effect"] + if ok { + effect, isEffect := raw.(string) + if isEffect { + toleration.Effect = corev1.TaintEffect(effect) + } + } + raw, ok = tolerationConverted["tolerationSeconds"] + if ok { + seconds, isSeconds := raw.(*int64) + if isSeconds { + toleration.TolerationSeconds = seconds + } + } + tolerations = append(tolerations, toleration) + } + } + } + tolerationsYaml, err = tolerationsToStringSliceYaml(tolerations) + if err != nil { + return nil, fmt.Errorf("failed to yaml marshal tolerations: %v", err) + } + } + advertiseAddress, valueFound, err := unstructured.NestedString(hcp.UnstructuredContent(), "spec", "networking", "apiServer", "advertiseAddress") if err != nil { return nil, fmt.Errorf("failed extract advertiseAddress: %v", err) @@ -192,6 +253,7 @@ func ParseHostedControlPlane(hcp *unstructured.Unstructured) (*HostedControlPlan ControllerAvailabilityPolicy: AvailabilityPolicy(controllerAvailabilityPolicy), ClusterID: clusterID, NodeSelector: nodeSelector, + Tolerations: tolerationsYaml, AdvertiseAddress: advertiseAddress, AdvertisePort: int(advertisePort), PriorityClass: controlPlanePriorityClassAnnotation, @@ -252,3 +314,29 @@ func SetHostedControlPlaneConditions(hcp *unstructured.Unstructured, operStatus hcp.Object["status"].(map[string]interface{})["conditions"] = conditions return conditions, nil } + +// tolerationsToStringSliceYaml converts a slice of tolerations into a slice of +// strings that represent the toleration in yaml syntax where each string +// is a line of yaml. The resulting string slice can be easily used in +// yaml manifest templating. +func tolerationsToStringSliceYaml(tolerations []corev1.Toleration) ([]string, error) { + if len(tolerations) == 0 { + return nil, nil + } + + yamlBytes, err := yaml.Marshal(tolerations) + if err != nil { + return nil, err + } + + yamlStrs := []string{} + for _, arg := range strings.Split(string(yamlBytes), "\n") { + + // filter out null and empty strings + if strings.Contains(arg, ": null") || strings.Contains(arg, ": \"\"") { + continue + } + yamlStrs = append(yamlStrs, arg) + } + return yamlStrs, nil +} diff --git a/pkg/network/cloud_network.go b/pkg/network/cloud_network.go index 2ade9a29af..db94fc2085 100644 --- a/pkg/network/cloud_network.go +++ b/pkg/network/cloud_network.go @@ -90,6 +90,7 @@ func renderCloudNetworkConfigController(conf *operv1.NetworkSpec, bootstrapResul data.Data["HostedClusterNamespace"] = hcpCfg.Namespace data.Data["ReleaseImage"] = hcpCfg.ReleaseImage data.Data["HCPNodeSelector"] = cloudBootstrapResult.HostedControlPlane.NodeSelector + data.Data["HCPTolerations"] = cloudBootstrapResult.HostedControlPlane.Tolerations data.Data["RunAsUser"] = hcpCfg.RunAsUser // In HyperShift CloudNetworkConfigController is deployed as a part of the hosted cluster controlplane // which means that it is created in the management cluster. diff --git a/pkg/network/multus_admission_controller.go b/pkg/network/multus_admission_controller.go index a1b5a92c93..6ce7ee6a5f 100644 --- a/pkg/network/multus_admission_controller.go +++ b/pkg/network/multus_admission_controller.go @@ -110,6 +110,7 @@ func renderMultusAdmissonControllerConfig(manifestDir string, externalControlPla data.Data["ClusterIDLabel"] = hypershift.ClusterIDLabel data.Data["ClusterID"] = bootstrapResult.Infra.HostedControlPlane.ClusterID data.Data["HCPNodeSelector"] = bootstrapResult.Infra.HostedControlPlane.NodeSelector + data.Data["HCPTolerations"] = bootstrapResult.Infra.HostedControlPlane.Tolerations data.Data["PriorityClass"] = bootstrapResult.Infra.HostedControlPlane.PriorityClass // Preserve any existing multus container resource requests which may have been modified by an external source diff --git a/pkg/network/node_identity.go b/pkg/network/node_identity.go index 000100757e..cc43aae3d1 100644 --- a/pkg/network/node_identity.go +++ b/pkg/network/node_identity.go @@ -97,6 +97,8 @@ func renderNetworkNodeIdentity(conf *operv1.NetworkSpec, bootstrapResult *bootst data.Data["TokenMinterImage"] = os.Getenv("TOKEN_MINTER_IMAGE") data.Data["TokenAudience"] = os.Getenv("TOKEN_AUDIENCE") data.Data["HCPNodeSelector"] = bootstrapResult.Infra.HostedControlPlane.NodeSelector + data.Data["HCPTolerations"] = bootstrapResult.Infra.HostedControlPlane.Tolerations + data.Data["NetworkNodeIdentityImage"] = hcpCfg.ControlPlaneImage // OVN_CONTROL_PLANE_IMAGE localAPIServer := bootstrapResult.Infra.APIServers[bootstrap.APIServerDefaultLocal] data.Data["K8S_LOCAL_APISERVER"] = "https://" + net.JoinHostPort(localAPIServer.Host, localAPIServer.Port) diff --git a/pkg/network/ovn_kubernetes.go b/pkg/network/ovn_kubernetes.go index 592a069ed2..56cf2f343c 100644 --- a/pkg/network/ovn_kubernetes.go +++ b/pkg/network/ovn_kubernetes.go @@ -218,6 +218,7 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo data.Data["ClusterID"] = bootstrapResult.OVN.OVNKubernetesConfig.HyperShiftConfig.ClusterID data.Data["ClusterIDLabel"] = hypershift.ClusterIDLabel data.Data["HCPNodeSelector"] = bootstrapResult.OVN.OVNKubernetesConfig.HyperShiftConfig.HCPNodeSelector + data.Data["HCPTolerations"] = bootstrapResult.OVN.OVNKubernetesConfig.HyperShiftConfig.HCPTolerations data.Data["OVN_NB_INACTIVITY_PROBE"] = nb_inactivity_probe data.Data["OVN_CERT_CN"] = OVN_CERT_CN data.Data["OVN_NORTHD_PROBE_INTERVAL"] = os.Getenv("OVN_NORTHD_PROBE_INTERVAL") @@ -710,6 +711,8 @@ func bootstrapOVNHyperShiftConfig(hc *hypershift.HyperShiftConfig, kubeClient cn ovnHypershiftResult.ClusterID = hcp.ClusterID ovnHypershiftResult.HCPNodeSelector = hcp.NodeSelector + ovnHypershiftResult.HCPTolerations = hcp.Tolerations + switch hcp.ControllerAvailabilityPolicy { case hypershift.HighlyAvailable: ovnHypershiftResult.ControlPlaneReplicas = 3