diff --git a/bindata/cloud-network-config-controller/managed/controller.yaml b/bindata/cloud-network-config-controller/managed/controller.yaml index 744fb3e094..be9883e00d 100644 --- a/bindata/cloud-network-config-controller/managed/controller.yaml +++ b/bindata/cloud-network-config-controller/managed/controller.yaml @@ -202,6 +202,11 @@ spec: readOnly: true terminationMessagePolicy: FallbackToLogsOnError tolerations: + {{- if .HCPTolerations }} + {{- range $t := .HCPTolerations }} + {{ $t }} + {{- end }} + {{- end }} - key: "hypershift.openshift.io/control-plane" operator: "Equal" value: "true" diff --git a/bindata/network/multus-admission-controller/admission-controller.yaml b/bindata/network/multus-admission-controller/admission-controller.yaml index 95ec3f1a7d..a03733bb24 100644 --- a/bindata/network/multus-admission-controller/admission-controller.yaml +++ b/bindata/network/multus-admission-controller/admission-controller.yaml @@ -259,6 +259,11 @@ spec: runAsUser: {{.RunAsUser}} {{- end }} tolerations: +{{- if .HCPTolerations }} + {{- range $t := .HCPTolerations }} + {{ $t }} + {{- end }} +{{- end }} - key: "hypershift.openshift.io/control-plane" operator: "Equal" value: "true" diff --git a/bindata/network/node-identity/managed/node-identity.yaml b/bindata/network/node-identity/managed/node-identity.yaml index 0c67159f89..a8336f7f6d 100644 --- a/bindata/network/node-identity/managed/node-identity.yaml +++ b/bindata/network/node-identity/managed/node-identity.yaml @@ -273,6 +273,11 @@ spec: - key: additional-pod-admission-cond.json path: additional-pod-admission-cond.json tolerations: + {{- if .HCPTolerations }} + {{- range $t := .HCPTolerations }} + {{ $t }} + {{- end }} + {{- end }} - key: "hypershift.openshift.io/control-plane" operator: "Equal" value: "true" diff --git a/bindata/network/ovn-kubernetes/managed/ovnkube-control-plane.yaml b/bindata/network/ovn-kubernetes/managed/ovnkube-control-plane.yaml index e9a7c5c730..f35c8a9b19 100644 --- a/bindata/network/ovn-kubernetes/managed/ovnkube-control-plane.yaml +++ b/bindata/network/ovn-kubernetes/managed/ovnkube-control-plane.yaml @@ -264,7 +264,6 @@ spec: - name: KUBECONFIG value: "/etc/kubernetes/kubeconfig" terminationMessagePolicy: FallbackToLogsOnError - {{ if .HCPNodeSelector }} nodeSelector: {{ range $key, $value := .HCPNodeSelector }} @@ -301,6 +300,11 @@ spec: - key: ca.crt path: ca.crt tolerations: + {{- if .HCPTolerations }} + {{- range $t := .HCPTolerations }} + {{ $t }} + {{- end }} + {{- end }} - key: "hypershift.openshift.io/control-plane" operator: "Equal" value: "true" diff --git a/pkg/bootstrap/types.go b/pkg/bootstrap/types.go index 9742abda26..a9adc16d61 100644 --- a/pkg/bootstrap/types.go +++ b/pkg/bootstrap/types.go @@ -11,6 +11,7 @@ type OVNHyperShiftBootstrapResult struct { ClusterID string Namespace string HCPNodeSelector map[string]string + HCPTolerations []string ControlPlaneReplicas int ReleaseImage string ControlPlaneImage string diff --git a/pkg/hypershift/hypershift.go b/pkg/hypershift/hypershift.go index 4b9b4446c3..76ef72022a 100644 --- a/pkg/hypershift/hypershift.go +++ b/pkg/hypershift/hypershift.go @@ -10,6 +10,8 @@ import ( configv1 "github.com/openshift/api/config/v1" operv1 "github.com/openshift/api/operator/v1" + "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -56,6 +58,7 @@ type HostedControlPlane struct { ClusterID string ControllerAvailabilityPolicy AvailabilityPolicy NodeSelector map[string]string + Tolerations []corev1.Toleration AdvertiseAddress string AdvertisePort int PriorityClass string @@ -152,6 +155,56 @@ func ParseHostedControlPlane(hcp *unstructured.Unstructured) (*HostedControlPlan return nil, fmt.Errorf("failed extract nodeSelector: %v", err) } + var tolerations []corev1.Toleration + tolerationsArray, tolerationsArrayFound, err := unstructured.NestedFieldCopy(hcp.UnstructuredContent(), "spec", "tolerations") + if tolerationsArrayFound { + tolerationsArrayConverted, hasConverted := tolerationsArray.([]interface{}) + if hasConverted { + for _, entry := range tolerationsArrayConverted { + tolerationConverted, hasConverted := entry.(map[string]interface{}) + if hasConverted { + toleration := corev1.Toleration{} + raw, ok := tolerationConverted["key"] + if ok { + str, isString := raw.(string) + if isString { + toleration.Key = str + } + } + raw, ok = tolerationConverted["operator"] + if ok { + op, isOperator := raw.(string) + if isOperator { + toleration.Operator = corev1.TolerationOperator(op) + } + } + raw, ok = tolerationConverted["value"] + if ok { + str, isString := raw.(string) + if isString { + toleration.Value = str + } + } + raw, ok = tolerationConverted["effect"] + if ok { + effect, isEffect := raw.(string) + if isEffect { + toleration.Effect = corev1.TaintEffect(effect) + } + } + raw, ok = tolerationConverted["tolerationSeconds"] + if ok { + seconds, isSeconds := raw.(*int64) + if isSeconds { + toleration.TolerationSeconds = seconds + } + } + tolerations = append(tolerations, toleration) + } + } + } + } + advertiseAddress, valueFound, err := unstructured.NestedString(hcp.UnstructuredContent(), "spec", "networking", "apiServer", "advertiseAddress") if err != nil { return nil, fmt.Errorf("failed extract advertiseAddress: %v", err) @@ -192,6 +245,7 @@ func ParseHostedControlPlane(hcp *unstructured.Unstructured) (*HostedControlPlan ControllerAvailabilityPolicy: AvailabilityPolicy(controllerAvailabilityPolicy), ClusterID: clusterID, NodeSelector: nodeSelector, + Tolerations: tolerations, AdvertiseAddress: advertiseAddress, AdvertisePort: int(advertisePort), PriorityClass: controlPlanePriorityClassAnnotation, @@ -252,3 +306,21 @@ func SetHostedControlPlaneConditions(hcp *unstructured.Unstructured, operStatus hcp.Object["status"].(map[string]interface{})["conditions"] = conditions return conditions, nil } + +func TolerationsToStringArray(tolerations []corev1.Toleration) ([]string, error) { + yamlBytes, err := yaml.Marshal(tolerations) + if err != nil { + return nil, err + } + + yamlStrs := []string{} + for _, arg := range strings.Split(string(yamlBytes), "\n") { + + // filter out null and empty strings + if strings.Contains(arg, ": null") || strings.Contains(arg, ": \"\"") { + continue + } + yamlStrs = append(yamlStrs, arg) + } + return yamlStrs, nil +} diff --git a/pkg/network/cloud_network.go b/pkg/network/cloud_network.go index 2ade9a29af..1b24ff43c8 100644 --- a/pkg/network/cloud_network.go +++ b/pkg/network/cloud_network.go @@ -83,6 +83,16 @@ func renderCloudNetworkConfigController(conf *operv1.NetworkSpec, bootstrapResul manifestDirs := make([]string, 0, 2) manifestDirs = append(manifestDirs, filepath.Join(manifestDir, "cloud-network-config-controller/common")) if hcpCfg := hypershift.NewHyperShiftConfig(); hcpCfg.Enabled { + if len(cloudBootstrapResult.HostedControlPlane.Tolerations) != 0 { + tolerations, err := hypershift.TolerationsToStringArray(cloudBootstrapResult.HostedControlPlane.Tolerations) + if err != nil { + return nil, err + } + data.Data["HCPTolerations"] = tolerations + } else { + data.Data["HCPTolerations"] = "" + } + data.Data["CLIImage"] = os.Getenv("CLI_IMAGE") data.Data["TokenMinterImage"] = os.Getenv("TOKEN_MINTER_IMAGE") data.Data["TokenAudience"] = os.Getenv("TOKEN_AUDIENCE") diff --git a/pkg/network/multus_admission_controller.go b/pkg/network/multus_admission_controller.go index a1b5a92c93..6a76b3cad7 100644 --- a/pkg/network/multus_admission_controller.go +++ b/pkg/network/multus_admission_controller.go @@ -112,6 +112,16 @@ func renderMultusAdmissonControllerConfig(manifestDir string, externalControlPla data.Data["HCPNodeSelector"] = bootstrapResult.Infra.HostedControlPlane.NodeSelector data.Data["PriorityClass"] = bootstrapResult.Infra.HostedControlPlane.PriorityClass + if len(bootstrapResult.Infra.HostedControlPlane.Tolerations) != 0 { + tolerations, err := hypershift.TolerationsToStringArray(bootstrapResult.Infra.HostedControlPlane.Tolerations) + if err != nil { + return nil, err + } + data.Data["HCPTolerations"] = tolerations + } else { + data.Data["HCPTolerations"] = "" + } + // Preserve any existing multus container resource requests which may have been modified by an external source multusDeploy := &appsv1.Deployment{} err = client.ClientFor(clientName).CRClient().Get( diff --git a/pkg/network/node_identity.go b/pkg/network/node_identity.go index 000100757e..21e2bddc38 100644 --- a/pkg/network/node_identity.go +++ b/pkg/network/node_identity.go @@ -97,6 +97,17 @@ func renderNetworkNodeIdentity(conf *operv1.NetworkSpec, bootstrapResult *bootst data.Data["TokenMinterImage"] = os.Getenv("TOKEN_MINTER_IMAGE") data.Data["TokenAudience"] = os.Getenv("TOKEN_AUDIENCE") data.Data["HCPNodeSelector"] = bootstrapResult.Infra.HostedControlPlane.NodeSelector + + if len(bootstrapResult.Infra.HostedControlPlane.Tolerations) != 0 { + tolerations, err := hypershift.TolerationsToStringArray(bootstrapResult.Infra.HostedControlPlane.Tolerations) + if err != nil { + return nil, err + } + data.Data["HCPTolerations"] = tolerations + } else { + data.Data["HCPTolerations"] = "" + } + data.Data["NetworkNodeIdentityImage"] = hcpCfg.ControlPlaneImage // OVN_CONTROL_PLANE_IMAGE localAPIServer := bootstrapResult.Infra.APIServers[bootstrap.APIServerDefaultLocal] data.Data["K8S_LOCAL_APISERVER"] = "https://" + net.JoinHostPort(localAPIServer.Host, localAPIServer.Port) diff --git a/pkg/network/ovn_kubernetes.go b/pkg/network/ovn_kubernetes.go index 4309023c12..b16407b437 100644 --- a/pkg/network/ovn_kubernetes.go +++ b/pkg/network/ovn_kubernetes.go @@ -215,6 +215,11 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo data.Data["ClusterID"] = bootstrapResult.OVN.OVNKubernetesConfig.HyperShiftConfig.ClusterID data.Data["ClusterIDLabel"] = hypershift.ClusterIDLabel data.Data["HCPNodeSelector"] = bootstrapResult.OVN.OVNKubernetesConfig.HyperShiftConfig.HCPNodeSelector + if len(bootstrapResult.OVN.OVNKubernetesConfig.HyperShiftConfig.HCPTolerations) != 0 { + data.Data["HCPTolerations"] = bootstrapResult.OVN.OVNKubernetesConfig.HyperShiftConfig.HCPTolerations + } else { + data.Data["HCPTolerations"] = "" + } data.Data["OVN_NB_INACTIVITY_PROBE"] = nb_inactivity_probe data.Data["OVN_CERT_CN"] = OVN_CERT_CN data.Data["OVN_NORTHD_PROBE_INTERVAL"] = os.Getenv("OVN_NORTHD_PROBE_INTERVAL") @@ -703,6 +708,13 @@ func bootstrapOVNHyperShiftConfig(hc *hypershift.HyperShiftConfig, kubeClient cn ovnHypershiftResult.ClusterID = hcp.ClusterID ovnHypershiftResult.HCPNodeSelector = hcp.NodeSelector + + tolerations, err := hypershift.TolerationsToStringArray(hcp.Tolerations) + if err != nil { + return nil, err + } + ovnHypershiftResult.HCPTolerations = tolerations + switch hcp.ControllerAvailabilityPolicy { case hypershift.HighlyAvailable: ovnHypershiftResult.ControlPlaneReplicas = 3 diff --git a/pkg/network/render_test.go b/pkg/network/render_test.go index 592358459a..a3de473388 100644 --- a/pkg/network/render_test.go +++ b/pkg/network/render_test.go @@ -2,6 +2,9 @@ package network import ( "fmt" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" "reflect" "strings" @@ -81,6 +84,133 @@ func TestClusterNetworkChangeOkOnMigration(t *testing.T) { } +// HostedControlPlane tolerations +// ================================= +func TestHyperShiftTolerations(t *testing.T) { + g := NewGomegaWithT(t) + + config := operv1.Network{ + Spec: operv1.NetworkSpec{ + ServiceNetwork: []string{"172.30.0.0/16"}, + ClusterNetwork: []operv1.ClusterNetworkEntry{ + { + CIDR: "10.128.0.0/15", + HostPrefix: 23, + }, + { + CIDR: "10.0.0.0/14", + HostPrefix: 24, + }, + }, + DefaultNetwork: operv1.DefaultNetworkDefinition{ + Type: "MyAwesomeThirdPartyPlugin", + }, + }, + } + + // Bootstrap a client with an infrastructure object + if err := configv1.AddToScheme(scheme.Scheme); err != nil { + t.Fatalf("failed to add configv1 to scheme: %v", err) + } + infrastructure := &configv1.Infrastructure{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, + Status: configv1.InfrastructureStatus{ + PlatformStatus: &configv1.PlatformStatus{}, + }, + } + + client := fake.NewFakeClient(infrastructure) + err := createProxy(client) + g.Expect(err).NotTo(HaveOccurred()) + + err = Validate(&config.Spec) + g.Expect(err).NotTo(HaveOccurred()) + + prev := config.Spec.DeepCopy() + fillDefaults(prev, nil) + next := config.Spec.DeepCopy() + fillDefaults(next, nil) + + err = IsChangeSafe(prev, next, &fakeBootstrapResult().Infra) + g.Expect(err).NotTo(HaveOccurred()) + + featureGatesCNO := featuregates.NewFeatureGate([]configv1.FeatureGateName{}, []configv1.FeatureGateName{}) + + hcpTolerations := []corev1.Toleration{ + { + Key: "node.kubernetes.io/not-ready", + Operator: "Exists", + Value: "", + Effect: "NoExecute", + TolerationSeconds: nil, + }, + { + Key: "node.kubernetes.io/unreachable", + Operator: "Exists", + Value: "", + Effect: "NoExecute", + TolerationSeconds: nil, + }, + { + Key: "node.kubernetes.io/memory-pressure", + Operator: "Exists", + Value: "", + Effect: "NoSchedule", + TolerationSeconds: nil, + }, + { + Key: "key1", + Operator: "Equal", + Value: "value1", + Effect: "NoSchedule", + TolerationSeconds: nil, + }, + { + Key: "key1", + Operator: "Exists", + Value: "", + Effect: "NoSchedule", + TolerationSeconds: nil, + }, + } + + // fake that we are in HyperShift hosted cluster + bootstrapResult := fakeBootstrapResult() + bootstrapResult.Infra = bootstrap.InfraStatus{} + bootstrapResult.Infra.HostedControlPlane = &hypershift.HostedControlPlane{ + ClusterID: "", + ControllerAvailabilityPolicy: "", + NodeSelector: map[string]string{ + "kubernetes.io/os": "linux", + }, + Tolerations: hcpTolerations, + AdvertiseAddress: "", + AdvertisePort: 0, + PriorityClass: "", + } + + err = Validate(&config.Spec) + g.Expect(err).NotTo(HaveOccurred()) + + objs, _, err := Render(prev, &configv1.NetworkSpec{}, manifestDir, client, featureGatesCNO, bootstrapResult) + g.Expect(err).NotTo(HaveOccurred()) + + for _, obj := range objs { + if obj.GetKind() == "Deployment" { + deployment := &appsv1.Deployment{} + err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, deployment) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(deployment.Spec.Template.Spec.Tolerations).To(ContainElements(hcpTolerations)) + } else if obj.GetKind() == "DaemonSet" && obj.GetName() != "multus" { + daemonset := &appsv1.DaemonSet{} + err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, daemonset) + g.Expect(daemonset.Spec.Template.Spec.Tolerations).To(ContainElements(hcpTolerations)) + } + + } + +} + // Invalid miscellaneous migration validation // ================================= func TestServiceNetworkChangeNotOkOnMigration(t *testing.T) {