diff --git a/api/controlplane/kubeadm/v1beta1/conversion.go b/api/controlplane/kubeadm/v1beta1/conversion.go index df956c8d9410..29bac3e8cd87 100644 --- a/api/controlplane/kubeadm/v1beta1/conversion.go +++ b/api/controlplane/kubeadm/v1beta1/conversion.go @@ -78,6 +78,7 @@ func (src *KubeadmControlPlane) ConvertTo(dstRaw conversion.Hub) error { // Recover other values if ok { bootstrapv1beta1.RestoreKubeadmConfigSpec(&restored.Spec.KubeadmConfigSpec, &dst.Spec.KubeadmConfigSpec) + dst.Spec.MachineTemplate.Spec.Taints = restored.Spec.MachineTemplate.Spec.Taints } if src.Spec.RemediationStrategy != nil { @@ -134,6 +135,7 @@ func (src *KubeadmControlPlaneTemplate) ConvertTo(dstRaw conversion.Hub) error { // Recover other values if ok { bootstrapv1beta1.RestoreKubeadmConfigSpec(&restored.Spec.Template.Spec.KubeadmConfigSpec, &dst.Spec.Template.Spec.KubeadmConfigSpec) + dst.Spec.Template.Spec.MachineTemplate.Spec.Taints = restored.Spec.Template.Spec.MachineTemplate.Spec.Taints } if src.Spec.Template.Spec.RemediationStrategy != nil { diff --git a/api/controlplane/kubeadm/v1beta2/kubeadm_control_plane_types.go b/api/controlplane/kubeadm/v1beta2/kubeadm_control_plane_types.go index e3622270b490..a02f810f0146 100644 --- a/api/controlplane/kubeadm/v1beta2/kubeadm_control_plane_types.go +++ b/api/controlplane/kubeadm/v1beta2/kubeadm_control_plane_types.go @@ -491,6 +491,23 @@ type KubeadmControlPlaneMachineTemplateSpec struct { // deletion contains configuration options for Machine deletion. // +optional Deletion KubeadmControlPlaneMachineTemplateDeletionSpec `json:"deletion,omitempty,omitzero"` + + // taints are the node taints that Cluster API will manage. + // This list is not necessarily complete: other Kubernetes components may add or remove other taints from nodes, + // e.g. the node controller might add the node.kubernetes.io/not-ready taint. + // Only those taints defined in this list will be added or removed by core Cluster API controllers. + // + // There can be at most 64 taints. + // A pod would have to tolerate all existing taints to run on the corresponding node. + // + // NOTE: This list is implemented as a "map" type, meaning that individual elements can be managed by different owners. + // +optional + // +listType=map + // +listMapKey=key + // +listMapKey=effect + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=64 + Taints []clusterv1.MachineTaint `json:"taints,omitempty"` } // KubeadmControlPlaneMachineTemplateDeletionSpec contains configuration options for Machine deletion. diff --git a/api/controlplane/kubeadm/v1beta2/kubeadmcontrolplanetemplate_types.go b/api/controlplane/kubeadm/v1beta2/kubeadmcontrolplanetemplate_types.go index 3c51600e4e79..7daf03d03708 100644 --- a/api/controlplane/kubeadm/v1beta2/kubeadmcontrolplanetemplate_types.go +++ b/api/controlplane/kubeadm/v1beta2/kubeadmcontrolplanetemplate_types.go @@ -139,6 +139,23 @@ type KubeadmControlPlaneTemplateMachineTemplateSpec struct { // deletion contains configuration options for Machine deletion. // +optional Deletion KubeadmControlPlaneTemplateMachineTemplateDeletionSpec `json:"deletion,omitempty,omitzero"` + + // taints are the node taints that Cluster API will manage. + // This list is not necessarily complete: other Kubernetes components may add or remove other taints from nodes, + // e.g. the node controller might add the node.kubernetes.io/not-ready taint. + // Only those taints defined in this list will be added or removed by core Cluster API controllers. + // + // There can be at most 64 taints. + // A pod would have to tolerate all existing taints to run on the corresponding node. + // + // NOTE: This list is implemented as a "map" type, meaning that individual elements can be managed by different owners. + // +optional + // +listType=map + // +listMapKey=key + // +listMapKey=effect + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=64 + Taints []clusterv1.MachineTaint `json:"taints,omitempty"` } // KubeadmControlPlaneTemplateMachineTemplateDeletionSpec contains configuration options for Machine deletion. diff --git a/api/controlplane/kubeadm/v1beta2/zz_generated.deepcopy.go b/api/controlplane/kubeadm/v1beta2/zz_generated.deepcopy.go index b485fa0a810d..5c96f4ac60b6 100644 --- a/api/controlplane/kubeadm/v1beta2/zz_generated.deepcopy.go +++ b/api/controlplane/kubeadm/v1beta2/zz_generated.deepcopy.go @@ -183,6 +183,11 @@ func (in *KubeadmControlPlaneMachineTemplateSpec) DeepCopyInto(out *KubeadmContr copy(*out, *in) } in.Deletion.DeepCopyInto(&out.Deletion) + if in.Taints != nil { + in, out := &in.Taints, &out.Taints + *out = make([]corev1beta2.MachineTaint, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeadmControlPlaneMachineTemplateSpec. @@ -477,6 +482,11 @@ func (in *KubeadmControlPlaneTemplateMachineTemplateDeletionSpec) DeepCopy() *Ku func (in *KubeadmControlPlaneTemplateMachineTemplateSpec) DeepCopyInto(out *KubeadmControlPlaneTemplateMachineTemplateSpec) { *out = *in in.Deletion.DeepCopyInto(&out.Deletion) + if in.Taints != nil { + in, out := &in.Taints, &out.Taints + *out = make([]corev1beta2.MachineTaint, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeadmControlPlaneTemplateMachineTemplateSpec. diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml index 5fc53686c597..b8426fd7fbf4 100644 --- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml +++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml @@ -7680,6 +7680,77 @@ spec: x-kubernetes-list-map-keys: - conditionType x-kubernetes-list-type: map + taints: + description: |- + taints are the node taints that Cluster API will manage. + This list is not necessarily complete: other Kubernetes components may add or remove other taints from nodes, + e.g. the node controller might add the node.kubernetes.io/not-ready taint. + Only those taints defined in this list will be added or removed by core Cluster API controllers. + + There can be at most 64 taints. + A pod would have to tolerate all existing taints to run on the corresponding node. + + NOTE: This list is implemented as a "map" type, meaning that individual elements can be managed by different owners. + items: + description: MachineTaint defines a taint equivalent to + corev1.Taint, but additionally having a propagation field. + properties: + effect: + description: effect is the effect for the taint. Valid + values are NoSchedule, PreferNoSchedule and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + type: string + key: + description: |- + key is the taint key to be applied to a node. + Must be a valid qualified name of maximum size 63 characters + with an optional subdomain prefix of maximum size 253 characters, + separated by a `/`. + maxLength: 317 + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/)?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + x-kubernetes-validations: + - message: key must be a valid qualified name of max + size 63 characters with an optional subdomain prefix + of max size 253 characters + rule: 'self.contains(''/'') ? ( self.split(''/'') + [0].size() <= 253 && self.split(''/'') [1].size() + <= 63 && self.split(''/'').size() == 2 ) : self.size() + <= 63' + propagation: + description: |- + propagation defines how this taint should be propagated to nodes. + Valid values are 'Always' and 'OnInitialization'. + Always: The taint will be continuously reconciled. If it is not set for a node, it will be added during reconciliation. + OnInitialization: The taint will be added during node initialization. If it gets removed from the node later on it will not get added again. + enum: + - Always + - OnInitialization + type: string + value: + description: |- + value is the taint value corresponding to the taint key. + It must be a valid label value of maximum size 63 characters. + maxLength: 63 + minLength: 1 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + type: string + required: + - effect + - key + - propagation + type: object + maxItems: 64 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - key + - effect + x-kubernetes-list-type: map required: - infrastructureRef type: object diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml index 694318b53ed9..8c60c3741b30 100644 --- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml +++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml @@ -6028,6 +6028,79 @@ spec: minimum: 0 type: integer type: object + taints: + description: |- + taints are the node taints that Cluster API will manage. + This list is not necessarily complete: other Kubernetes components may add or remove other taints from nodes, + e.g. the node controller might add the node.kubernetes.io/not-ready taint. + Only those taints defined in this list will be added or removed by core Cluster API controllers. + + There can be at most 64 taints. + A pod would have to tolerate all existing taints to run on the corresponding node. + + NOTE: This list is implemented as a "map" type, meaning that individual elements can be managed by different owners. + items: + description: MachineTaint defines a taint equivalent + to corev1.Taint, but additionally having a propagation + field. + properties: + effect: + description: effect is the effect for the taint. + Valid values are NoSchedule, PreferNoSchedule + and NoExecute. + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + type: string + key: + description: |- + key is the taint key to be applied to a node. + Must be a valid qualified name of maximum size 63 characters + with an optional subdomain prefix of maximum size 253 characters, + separated by a `/`. + maxLength: 317 + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/)?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + type: string + x-kubernetes-validations: + - message: key must be a valid qualified name + of max size 63 characters with an optional + subdomain prefix of max size 253 characters + rule: 'self.contains(''/'') ? ( self.split(''/'') + [0].size() <= 253 && self.split(''/'') [1].size() + <= 63 && self.split(''/'').size() == 2 ) + : self.size() <= 63' + propagation: + description: |- + propagation defines how this taint should be propagated to nodes. + Valid values are 'Always' and 'OnInitialization'. + Always: The taint will be continuously reconciled. If it is not set for a node, it will be added during reconciliation. + OnInitialization: The taint will be added during node initialization. If it gets removed from the node later on it will not get added again. + enum: + - Always + - OnInitialization + type: string + value: + description: |- + value is the taint value corresponding to the taint key. + It must be a valid label value of maximum size 63 characters. + maxLength: 63 + minLength: 1 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + type: string + required: + - effect + - key + - propagation + type: object + maxItems: 64 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - key + - effect + x-kubernetes-list-type: map type: object type: object remediation: diff --git a/controlplane/kubeadm/config/manager/manager.yaml b/controlplane/kubeadm/config/manager/manager.yaml index b0a027874483..5ed57fc15451 100644 --- a/controlplane/kubeadm/config/manager/manager.yaml +++ b/controlplane/kubeadm/config/manager/manager.yaml @@ -22,7 +22,7 @@ spec: - "--leader-elect" - "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}" - "--insecure-diagnostics=${CAPI_INSECURE_DIAGNOSTICS:=false}" - - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},ClusterTopology=${CLUSTER_TOPOLOGY:=false},KubeadmBootstrapFormatIgnition=${EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION:=false},PriorityQueue=${EXP_PRIORITY_QUEUE:=true},ReconcilerRateLimiting=${EXP_RECONCILER_RATE_LIMITING:=false},InPlaceUpdates=${EXP_IN_PLACE_UPDATES:=false}" + - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},ClusterTopology=${CLUSTER_TOPOLOGY:=false},KubeadmBootstrapFormatIgnition=${EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION:=false},PriorityQueue=${EXP_PRIORITY_QUEUE:=true},ReconcilerRateLimiting=${EXP_RECONCILER_RATE_LIMITING:=false},InPlaceUpdates=${EXP_IN_PLACE_UPDATES:=false},MachineTaintPropagation=${EXP_MACHINE_TAINT_PROPAGATION:=false}" image: controller:latest name: manager env: diff --git a/controlplane/kubeadm/internal/controllers/controller.go b/controlplane/kubeadm/internal/controllers/controller.go index 57ede403acd6..c89d09f7908b 100644 --- a/controlplane/kubeadm/internal/controllers/controller.go +++ b/controlplane/kubeadm/internal/controllers/controller.go @@ -822,6 +822,7 @@ func (r *KubeadmControlPlaneReconciler) syncMachines(ctx context.Context, contro m.Spec.Deletion.NodeDrainTimeoutSeconds = controlPlane.KCP.Spec.MachineTemplate.Spec.Deletion.NodeDrainTimeoutSeconds m.Spec.Deletion.NodeDeletionTimeoutSeconds = controlPlane.KCP.Spec.MachineTemplate.Spec.Deletion.NodeDeletionTimeoutSeconds m.Spec.Deletion.NodeVolumeDetachTimeoutSeconds = controlPlane.KCP.Spec.MachineTemplate.Spec.Deletion.NodeVolumeDetachTimeoutSeconds + m.Spec.Taints = controlPlane.KCP.Spec.MachineTemplate.Spec.Taints // Note: We intentionally don't set "minReadySeconds" on Machines because we consider it enough to have machine availability driven by readiness of control plane components. if err := patchHelper.Patch(ctx, m); err != nil { diff --git a/controlplane/kubeadm/internal/desiredstate/desired_state.go b/controlplane/kubeadm/internal/desiredstate/desired_state.go index 2997c4b85625..657fdc183e3d 100644 --- a/controlplane/kubeadm/internal/desiredstate/desired_state.go +++ b/controlplane/kubeadm/internal/desiredstate/desired_state.go @@ -166,6 +166,7 @@ func ComputeDesiredMachine(kcp *controlplanev1.KubeadmControlPlane, cluster *clu desiredMachine.Spec.Deletion.NodeDrainTimeoutSeconds = kcp.Spec.MachineTemplate.Spec.Deletion.NodeDrainTimeoutSeconds desiredMachine.Spec.Deletion.NodeDeletionTimeoutSeconds = kcp.Spec.MachineTemplate.Spec.Deletion.NodeDeletionTimeoutSeconds desiredMachine.Spec.Deletion.NodeVolumeDetachTimeoutSeconds = kcp.Spec.MachineTemplate.Spec.Deletion.NodeVolumeDetachTimeoutSeconds + desiredMachine.Spec.Taints = kcp.Spec.MachineTemplate.Spec.Taints // Note: We intentionally don't set "minReadySeconds" on Machines because we consider it enough to have machine availability driven by readiness of control plane components. if existingMachine != nil { diff --git a/controlplane/kubeadm/internal/desiredstate/desired_state_test.go b/controlplane/kubeadm/internal/desiredstate/desired_state_test.go index b7d5de7f8091..39cba7744f26 100644 --- a/controlplane/kubeadm/internal/desiredstate/desired_state_test.go +++ b/controlplane/kubeadm/internal/desiredstate/desired_state_test.go @@ -286,6 +286,47 @@ func Test_ComputeDesiredMachine(t *testing.T) { isUpdatingExistingMachine: false, wantErr: false, }, + { + name: "should return the correct Machine object when creating a new Machine with taints", + kcp: &controlplanev1.KubeadmControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: kcpName, + Namespace: cluster.Namespace, + }, + Spec: controlplanev1.KubeadmControlPlaneSpec{ + Version: "v1.16.6", + MachineTemplate: controlplanev1.KubeadmControlPlaneMachineTemplate{ + ObjectMeta: kcpMachineTemplateObjectMeta, + Spec: controlplanev1.KubeadmControlPlaneMachineTemplateSpec{ + Taints: []clusterv1.MachineTaint{ + { + Key: "foo", + Effect: "NoSchedule", + Propagation: clusterv1.MachineTaintPropagationAlways, + }, + { + Key: "bar", + Effect: "NoExecute", + Propagation: clusterv1.MachineTaintPropagationOnInitialization, + }, + }, + Deletion: controlplanev1.KubeadmControlPlaneMachineTemplateDeletionSpec{ + NodeDrainTimeoutSeconds: duration5s, + NodeDeletionTimeoutSeconds: duration5s, + NodeVolumeDetachTimeoutSeconds: duration5s, + }, + }, + }, + KubeadmConfigSpec: bootstrapv1.KubeadmConfigSpec{ + ClusterConfiguration: bootstrapv1.ClusterConfiguration{ + CertificatesDir: "foo", + }, + }, + }, + }, + isUpdatingExistingMachine: false, + wantErr: false, + }, { name: "should return the correct Machine object when updating an existing Machine (empty ClusterConfiguration annotation)", kcp: &controlplanev1.KubeadmControlPlane{ @@ -497,6 +538,7 @@ func Test_ComputeDesiredMachine(t *testing.T) { NodeVolumeDetachTimeoutSeconds: tt.kcp.Spec.MachineTemplate.Spec.Deletion.NodeVolumeDetachTimeoutSeconds, }, ReadinessGates: append(append(MandatoryMachineReadinessGates, etcdMandatoryMachineReadinessGates...), tt.kcp.Spec.MachineTemplate.Spec.ReadinessGates...), + Taints: tt.kcp.Spec.MachineTemplate.Spec.Taints, } // Verify Name. for _, matcher := range tt.want { diff --git a/controlplane/kubeadm/internal/webhooks/kubeadmcontrolplane.go b/controlplane/kubeadm/internal/webhooks/kubeadmcontrolplane.go index 390268c57857..ba309d4328a1 100644 --- a/controlplane/kubeadm/internal/webhooks/kubeadmcontrolplane.go +++ b/controlplane/kubeadm/internal/webhooks/kubeadmcontrolplane.go @@ -40,6 +40,7 @@ import ( controlplanev1 "sigs.k8s.io/cluster-api/api/controlplane/kubeadm/v1beta2" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" topologynames "sigs.k8s.io/cluster-api/internal/topology/names" + "sigs.k8s.io/cluster-api/internal/util/taints" "sigs.k8s.io/cluster-api/util/container" "sigs.k8s.io/cluster-api/util/secret" "sigs.k8s.io/cluster-api/util/version" @@ -339,6 +340,8 @@ func validateKubeadmControlPlaneSpec(s controlplanev1.KubeadmControlPlaneSpec, p ) } + allErrs = append(allErrs, taints.ValidateMachineTaints(s.MachineTemplate.Spec.Taints, pathPrefix.Child("machineTemplate", "spec", "taints"))...) + // Validate the metadata of the MachineTemplate allErrs = append(allErrs, s.MachineTemplate.ObjectMeta.Validate(pathPrefix.Child("machineTemplate", "metadata"))...) diff --git a/controlplane/kubeadm/internal/webhooks/kubeadmcontrolplanetemplate.go b/controlplane/kubeadm/internal/webhooks/kubeadmcontrolplanetemplate.go index 2b31c5f884f1..8135c67edfa9 100644 --- a/controlplane/kubeadm/internal/webhooks/kubeadmcontrolplanetemplate.go +++ b/controlplane/kubeadm/internal/webhooks/kubeadmcontrolplanetemplate.go @@ -34,6 +34,7 @@ import ( "sigs.k8s.io/cluster-api/bootstrap/kubeadm/defaulting" "sigs.k8s.io/cluster-api/feature" "sigs.k8s.io/cluster-api/internal/util/compare" + "sigs.k8s.io/cluster-api/internal/util/taints" ) func (webhook *KubeadmControlPlaneTemplate) SetupWebhookWithManager(mgr ctrl.Manager) error { @@ -136,6 +137,7 @@ func validateKubeadmControlPlaneTemplateResourceSpec(s controlplanev1.KubeadmCon allErrs = append(allErrs, validateRolloutAndCertValidityFields(s.Rollout, s.KubeadmConfigSpec.ClusterConfiguration, nil, pathPrefix)...) allErrs = append(allErrs, validateNaming(s.MachineNaming, pathPrefix.Child("machineNaming"))...) + allErrs = append(allErrs, taints.ValidateMachineTaints(s.MachineTemplate.Spec.Taints, pathPrefix.Child("machineTemplate", "spec", "taints"))...) // Validate the metadata of the MachineTemplate allErrs = append(allErrs, s.MachineTemplate.ObjectMeta.Validate(pathPrefix.Child("machineTemplate", "metadata"))...) diff --git a/docs/book/src/developer/providers/contracts/control-plane.md b/docs/book/src/developer/providers/contracts/control-plane.md index 35e455c4f03e..672ece782f9f 100644 --- a/docs/book/src/developer/providers/contracts/control-plane.md +++ b/docs/book/src/developer/providers/contracts/control-plane.md @@ -545,6 +545,33 @@ type FooControlPlaneMachineTemplateSpec struct { } ``` +In case you are developing a control plane provider that allows definition of machine taints, you SHOULD also implement +the following `spec.machineTemplate.spec` field. + +```go +type FooControlPlaneMachineTemplateSpec struct { + // taints are the node taints that Cluster API will manage. + // This list is not necessarily complete: other Kubernetes components may add or remove other taints from nodes, + // e.g. the node controller might add the node.kubernetes.io/not-ready taint. + // Only those taints defined in this list will be added or removed by core Cluster API controllers. + // + // There can be at most 64 taints. + // A pod would have to tolerate all existing taints to run on the corresponding node. + // + // NOTE: This list is implemented as a "map" type, meaning that individual elements can be managed by different owners. + // +optional + // +listType=map + // +listMapKey=key + // +listMapKey=effect + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=64 + Taints []MachineTaint `json:"taints,omitempty"` + + // See other rules for more details about mandatory/optional fields in ControlPlane spec. + // Other fields SHOULD be added based on the needs of your provider. +} +``` + NOTE: In the v1beta1 contract the `readinessGates` field was located directly in the `spec.machineTemplate` field. In case you are developing a control plane provider where control plane instances uses a Cluster API Machine diff --git a/docs/book/src/developer/providers/migrations/v1.12-to-v1.13.md b/docs/book/src/developer/providers/migrations/v1.12-to-v1.13.md index 9b56b58c08c1..f0f4acae091d 100644 --- a/docs/book/src/developer/providers/migrations/v1.12-to-v1.13.md +++ b/docs/book/src/developer/providers/migrations/v1.12-to-v1.13.md @@ -43,7 +43,8 @@ Any feedback or contributions to improve following documentation is welcome! ## Cluster API Contract changes -- +- A new, optional rule has been added to the control plane contract, defining what is required for implementing support + for taints. ## Deprecation @@ -55,7 +56,7 @@ Any feedback or contributions to improve following documentation is welcome! ## Suggested changes for providers -- +- If you are developing a control plane provider with support for machines, please consider adding `spec.machineTemplate.spec.taints` (see [contract](../contracts/control-plane.md#controlplane-machines)) ## Removals scheduled for future releases diff --git a/internal/api/controlplane/kubeadm/v1alpha3/conversion.go b/internal/api/controlplane/kubeadm/v1alpha3/conversion.go index dd117d6412b8..97d6e18b1fec 100644 --- a/internal/api/controlplane/kubeadm/v1alpha3/conversion.go +++ b/internal/api/controlplane/kubeadm/v1alpha3/conversion.go @@ -98,6 +98,7 @@ func (src *KubeadmControlPlane) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.MachineTemplate.Spec.ReadinessGates = restored.Spec.MachineTemplate.Spec.ReadinessGates dst.Spec.MachineTemplate.Spec.Deletion.NodeDeletionTimeoutSeconds = restored.Spec.MachineTemplate.Spec.Deletion.NodeDeletionTimeoutSeconds dst.Spec.MachineTemplate.Spec.Deletion.NodeVolumeDetachTimeoutSeconds = restored.Spec.MachineTemplate.Spec.Deletion.NodeVolumeDetachTimeoutSeconds + dst.Spec.MachineTemplate.Spec.Taints = restored.Spec.MachineTemplate.Spec.Taints dst.Spec.Rollout = restored.Spec.Rollout dst.Spec.Remediation = restored.Spec.Remediation diff --git a/internal/api/controlplane/kubeadm/v1alpha4/conversion.go b/internal/api/controlplane/kubeadm/v1alpha4/conversion.go index dfe7624ae9a7..6e2405a9291f 100644 --- a/internal/api/controlplane/kubeadm/v1alpha4/conversion.go +++ b/internal/api/controlplane/kubeadm/v1alpha4/conversion.go @@ -97,6 +97,7 @@ func (src *KubeadmControlPlane) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.MachineTemplate.Spec.ReadinessGates = restored.Spec.MachineTemplate.Spec.ReadinessGates dst.Spec.MachineTemplate.Spec.Deletion.NodeDeletionTimeoutSeconds = restored.Spec.MachineTemplate.Spec.Deletion.NodeDeletionTimeoutSeconds dst.Spec.MachineTemplate.Spec.Deletion.NodeVolumeDetachTimeoutSeconds = restored.Spec.MachineTemplate.Spec.Deletion.NodeVolumeDetachTimeoutSeconds + dst.Spec.MachineTemplate.Spec.Taints = restored.Spec.MachineTemplate.Spec.Taints dst.Spec.Rollout = restored.Spec.Rollout dst.Spec.Remediation = restored.Spec.Remediation diff --git a/internal/contract/controlplane.go b/internal/contract/controlplane.go index e98092547534..34441a375c2e 100644 --- a/internal/contract/controlplane.go +++ b/internal/contract/controlplane.go @@ -540,3 +540,64 @@ func (m *ReadinessGates) Set(obj *unstructured.Unstructured, readinessGates []cl } return nil } + +// Taints provides access to control plane's Taints. +func (c *ControlPlaneMachineTemplate) Taints() *Taints { + return &Taints{ + path: []string{"spec", "machineTemplate", "spec", "taints"}, + } +} + +// Taints provides a helper struct for working with Taints. +type Taints struct { + path Path +} + +// Path returns the path of the Taints. +func (m *Taints) Path() Path { + return m.path +} + +// Get gets the Taints object. +func (m *Taints) Get(obj *unstructured.Unstructured) ([]clusterv1.MachineTaint, error) { + unstructuredValue, ok, err := unstructured.NestedSlice(obj.UnstructuredContent(), m.Path()...) + if err != nil { + return nil, errors.Wrapf(err, "failed to retrieve control plane %s", "."+m.Path().String()) + } + if !ok { + return nil, errors.Wrapf(ErrFieldNotFound, "path %s", "."+m.Path().String()) + } + + var taints []clusterv1.MachineTaint + jsonValue, err := json.Marshal(unstructuredValue) + if err != nil { + return nil, errors.Wrapf(err, "failed to Marshal control plane %s", "."+m.Path().String()) + } + if err := json.Unmarshal(jsonValue, &taints); err != nil { + return nil, errors.Wrapf(err, "failed to Unmarshal control plane %s", "."+m.Path().String()) + } + + return taints, nil +} + +// Set sets the Taints value. +// Note: in case the value is nil, the system assumes that the control plane do not implement the optional list of taints. +func (m *Taints) Set(obj *unstructured.Unstructured, taints []clusterv1.MachineTaint) error { + unstructured.RemoveNestedField(obj.UnstructuredContent(), m.Path()...) + if taints == nil { + return nil + } + + jsonValue, err := json.Marshal(taints) + if err != nil { + return errors.Wrapf(err, "failed to Marshal control plane %s", "."+m.Path().String()) + } + var unstructuredValue []interface{} + if err := json.Unmarshal(jsonValue, &unstructuredValue); err != nil { + return errors.Wrapf(err, "failed to Unmarshal control plane %s", "."+m.Path().String()) + } + if err := unstructured.SetNestedSlice(obj.UnstructuredContent(), unstructuredValue, m.Path()...); err != nil { + return errors.Wrapf(err, "failed to set control plane %s", "."+m.Path().String()) + } + return nil +} diff --git a/internal/contract/controlplane_test.go b/internal/contract/controlplane_test.go index ef1ece6eabc1..6795b891bc01 100644 --- a/internal/contract/controlplane_test.go +++ b/internal/contract/controlplane_test.go @@ -485,6 +485,51 @@ func TestControlPlane(t *testing.T) { g.Expect(got).ToNot(BeNil()) g.Expect(got).To(BeComparableTo(readinessGates)) }) + + t.Run("Manages spec.machineTemplate.taints (v1beta2 contract)", func(t *testing.T) { + g := NewWithT(t) + + taints := []clusterv1.MachineTaint{ + {Key: "foo", Effect: "NoSchedule", Propagation: clusterv1.MachineTaintPropagationAlways}, + {Key: "bar", Effect: "NoExecute", Propagation: clusterv1.MachineTaintPropagationOnInitialization}, + } + + g.Expect(ControlPlane().MachineTemplate().Taints().Path()).To(Equal(Path{"spec", "machineTemplate", "spec", "taints"})) + + err := ControlPlane().MachineTemplate().Taints().Set(obj, taints) + g.Expect(err).ToNot(HaveOccurred()) + + got, err := ControlPlane().MachineTemplate().Taints().Get(obj) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).ToNot(BeNil()) + g.Expect(got).To(BeComparableTo(taints)) + + // Nil readinessGates are not set. + obj2 := &unstructured.Unstructured{Object: map[string]interface{}{}} + taints = nil + + err = ControlPlane().MachineTemplate().Taints().Set(obj2, taints) + g.Expect(err).ToNot(HaveOccurred()) + + _, ok, err := unstructured.NestedSlice(obj2.UnstructuredContent(), ControlPlane().MachineTemplate().Taints().Path()...) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(ok).To(BeFalse()) + + _, err = ControlPlane().MachineTemplate().Taints().Get(obj2) + g.Expect(err).To(HaveOccurred()) + + // Empty taints are set. + obj3 := &unstructured.Unstructured{Object: map[string]interface{}{}} + taints = []clusterv1.MachineTaint{} + + err = ControlPlane().MachineTemplate().Taints().Set(obj3, taints) + g.Expect(err).ToNot(HaveOccurred()) + + got, err = ControlPlane().MachineTemplate().Taints().Get(obj3) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).ToNot(BeNil()) + g.Expect(got).To(BeComparableTo(taints)) + }) } func TestControlPlaneEndpoints(t *testing.T) { diff --git a/internal/util/taints/validation.go b/internal/util/taints/validation.go new file mode 100644 index 000000000000..696061b548ec --- /dev/null +++ b/internal/util/taints/validation.go @@ -0,0 +1,62 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package taints + +import ( + "strings" + + "k8s.io/apimachinery/pkg/util/validation/field" + + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/cluster-api/feature" +) + +// ValidateMachineTaints validates MachineTaints. +func ValidateMachineTaints(taints []clusterv1.MachineTaint, taintsPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + if !feature.Gates.Enabled(feature.MachineTaintPropagation) { + if len(taints) > 0 { + allErrs = append(allErrs, field.Forbidden(taintsPath, "taints are not allowed to be set when the feature gate MachineTaintPropagation is disabled")) + } + } + + for i, taint := range taints { + idxPath := taintsPath.Index(i) + + // The following validations uses a switch statement, because if one of them matches, then the others won't. + + switch { + // Validate for keys which are reserved for usage by the cluster-api or providers. + case taint.Key == clusterv1.NodeUninitializedTaint.Key: + allErrs = append(allErrs, field.Invalid(idxPath.Child("key"), taint.Key, "taint key is not allowed")) + case taint.Key == clusterv1.NodeOutdatedRevisionTaint.Key: + allErrs = append(allErrs, field.Invalid(idxPath.Child("key"), taint.Key, "taint key is not allowed")) + // Validate for keys which are reserved for usage by the node or node-lifecycle-controller, but allow `node.kubernetes.io/out-of-service`. + case strings.HasPrefix(taint.Key, "node.kubernetes.io/") && taint.Key != "node.kubernetes.io/out-of-service": + allErrs = append(allErrs, field.Invalid(idxPath.Child("key"), taint.Key, "taint key must not have the prefix node.kubernetes.io/, except for node.kubernetes.io/out-of-service")) + // Validate for keys which are reserved for usage by the cloud-controller-manager or kubelet. + case strings.HasPrefix(taint.Key, "node.cloudprovider.kubernetes.io/"): + allErrs = append(allErrs, field.Invalid(idxPath.Child("key"), taint.Key, "taint key must not have the prefix node.cloudprovider.kubernetes.io/")) + // Validate for the deprecated kubeadm node-role taint. + case taint.Key == "node-role.kubernetes.io/master": + allErrs = append(allErrs, field.Invalid(idxPath.Child("key"), taint.Key, "taint is deprecated since 1.24 and should not be used anymore")) + } + } + + return allErrs +} diff --git a/internal/webhooks/machine.go b/internal/webhooks/machine.go index f7863180cc94..a45d5e97c466 100644 --- a/internal/webhooks/machine.go +++ b/internal/webhooks/machine.go @@ -31,6 +31,7 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" "sigs.k8s.io/cluster-api/feature" + "sigs.k8s.io/cluster-api/internal/util/taints" "sigs.k8s.io/cluster-api/util/labels" "sigs.k8s.io/cluster-api/util/version" ) @@ -137,7 +138,7 @@ func (webhook *Machine) validate(oldM, newM *clusterv1.Machine) error { } } - allErrs = append(allErrs, validateMachineTaints(newM.Spec.Taints, specPath.Child("taints"))...) + allErrs = append(allErrs, taints.ValidateMachineTaints(newM.Spec.Taints, specPath.Child("taints"))...) allErrs = append(allErrs, validateMachineTaintsForWorkers(newM.Spec.Taints, newM, specPath.Child("taints"))...) if len(allErrs) == 0 { @@ -146,41 +147,6 @@ func (webhook *Machine) validate(oldM, newM *clusterv1.Machine) error { return apierrors.NewInvalid(clusterv1.GroupVersion.WithKind("Machine").GroupKind(), newM.Name, allErrs) } -func validateMachineTaints(taints []clusterv1.MachineTaint, taintsPath *field.Path) field.ErrorList { - var allErrs field.ErrorList - - if !feature.Gates.Enabled(feature.MachineTaintPropagation) { - if len(taints) > 0 { - allErrs = append(allErrs, field.Forbidden(taintsPath, "taints are not allowed to be set when the feature gate MachineTaintPropagation is disabled")) - } - } - - for i, taint := range taints { - idxPath := taintsPath.Index(i) - - // The following validations uses a switch statement, because if one of them matches, then the others won't. - - switch { - // Validate for keys which are reserved for usage by the cluster-api or providers. - case taint.Key == clusterv1.NodeUninitializedTaint.Key: - allErrs = append(allErrs, field.Invalid(idxPath.Child("key"), taint.Key, "taint key is not allowed")) - case taint.Key == clusterv1.NodeOutdatedRevisionTaint.Key: - allErrs = append(allErrs, field.Invalid(idxPath.Child("key"), taint.Key, "taint key is not allowed")) - // Validate for keys which are reserved for usage by the node or node-lifecycle-controller, but allow `node.kubernetes.io/out-of-service`. - case strings.HasPrefix(taint.Key, "node.kubernetes.io/") && taint.Key != "node.kubernetes.io/out-of-service": - allErrs = append(allErrs, field.Invalid(idxPath.Child("key"), taint.Key, "taint key must not have the prefix node.kubernetes.io/, except for node.kubernetes.io/out-of-service")) - // Validate for keys which are reserved for usage by the cloud-controller-manager or kubelet. - case strings.HasPrefix(taint.Key, "node.cloudprovider.kubernetes.io/"): - allErrs = append(allErrs, field.Invalid(idxPath.Child("key"), taint.Key, "taint key must not have the prefix node.cloudprovider.kubernetes.io/")) - // Validate for the deprecated kubeadm node-role taint. - case taint.Key == "node-role.kubernetes.io/master": - allErrs = append(allErrs, field.Invalid(idxPath.Child("key"), taint.Key, "taint is deprecated since 1.24 and should not be used anymore")) - } - } - - return allErrs -} - func validateMachineTaintsForWorkers(taints []clusterv1.MachineTaint, machine *clusterv1.Machine, taintsPath *field.Path) field.ErrorList { var allErrs field.ErrorList diff --git a/internal/webhooks/machinedeployment.go b/internal/webhooks/machinedeployment.go index 40e3ef90f5a3..266c600f069e 100644 --- a/internal/webhooks/machinedeployment.go +++ b/internal/webhooks/machinedeployment.go @@ -39,6 +39,7 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" "sigs.k8s.io/cluster-api/feature" topologynames "sigs.k8s.io/cluster-api/internal/topology/names" + "sigs.k8s.io/cluster-api/internal/util/taints" "sigs.k8s.io/cluster-api/util/version" ) @@ -254,7 +255,7 @@ func (webhook *MachineDeployment) validate(oldMD, newMD *clusterv1.MachineDeploy allErrs = append(allErrs, validateMDMachineNaming(newMD.Spec.MachineNaming, specPath.Child("machineNaming"))...) - allErrs = append(allErrs, validateMachineTaints(newMD.Spec.Template.Spec.Taints, specPath.Child("template", "spec", "taints"))...) + allErrs = append(allErrs, taints.ValidateMachineTaints(newMD.Spec.Template.Spec.Taints, specPath.Child("template", "spec", "taints"))...) allErrs = append(allErrs, validateMachineTaintsForWorkers(newMD.Spec.Template.Spec.Taints, nil, specPath.Child("template", "spec", "taints"))...) // Validate the metadata of the template. diff --git a/internal/webhooks/machineset.go b/internal/webhooks/machineset.go index 75d011a2d209..fba19e94fb8d 100644 --- a/internal/webhooks/machineset.go +++ b/internal/webhooks/machineset.go @@ -40,6 +40,7 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" "sigs.k8s.io/cluster-api/feature" topologynames "sigs.k8s.io/cluster-api/internal/topology/names" + "sigs.k8s.io/cluster-api/internal/util/taints" "sigs.k8s.io/cluster-api/util/labels/format" "sigs.k8s.io/cluster-api/util/version" ) @@ -229,7 +230,7 @@ func (webhook *MachineSet) validate(oldMS, newMS *clusterv1.MachineSet) error { } } - allErrs = append(allErrs, validateMachineTaints(newMS.Spec.Template.Spec.Taints, specPath.Child("template", "spec", "taints"))...) + allErrs = append(allErrs, taints.ValidateMachineTaints(newMS.Spec.Template.Spec.Taints, specPath.Child("template", "spec", "taints"))...) allErrs = append(allErrs, validateMachineTaintsForWorkers(newMS.Spec.Template.Spec.Taints, nil, specPath.Child("template", "spec", "taints"))...) allErrs = append(allErrs, validateMSMachineNaming(newMS.Spec.MachineNaming, specPath.Child("machineNaming"))...) diff --git a/test/e2e/data/infrastructure-docker/main/cluster-template-md-taints/kcp-taints.yaml b/test/e2e/data/infrastructure-docker/main/cluster-template-md-taints/kcp-taints.yaml new file mode 100644 index 000000000000..0c82cfcda1a6 --- /dev/null +++ b/test/e2e/data/infrastructure-docker/main/cluster-template-md-taints/kcp-taints.yaml @@ -0,0 +1,16 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + machineTemplate: + spec: + taints: + - effect: PreferNoSchedule + key: pre-existing-on-initialization-taint + propagation: OnInitialization + value: on-initialization-value + - effect: PreferNoSchedule + key: pre-existing-always-taint + propagation: Always + value: always-value \ No newline at end of file diff --git a/test/e2e/data/infrastructure-docker/main/cluster-template-md-taints/kustomization.yaml b/test/e2e/data/infrastructure-docker/main/cluster-template-md-taints/kustomization.yaml index 75efc26cec88..0e43bec25353 100644 --- a/test/e2e/data/infrastructure-docker/main/cluster-template-md-taints/kustomization.yaml +++ b/test/e2e/data/infrastructure-docker/main/cluster-template-md-taints/kustomization.yaml @@ -5,3 +5,4 @@ resources: patches: - path: md-taints.yaml +- path: kcp-taints.yaml diff --git a/test/e2e/md_rollout.go b/test/e2e/md_rollout.go index afd5c848e4dd..282e2e70b562 100644 --- a/test/e2e/md_rollout.go +++ b/test/e2e/md_rollout.go @@ -120,6 +120,14 @@ func MachineDeploymentRolloutSpec(ctx context.Context, inputGetter func() Machin MachineDeployments: clusterResources.MachineDeployments, }) + By("Upgrade ControlPlane and MachineDeployment in-place mutable taints field and wait for in-place propagation") + + additionalControlPlaneNodeTaint := corev1.Taint{ + Key: "node-role.kubernetes.io/control-plane", + Value: "", + Effect: "NoSchedule", + } + preExistingAlwaysTaint := clusterv1.MachineTaint{ Key: "pre-existing-always-taint", Value: "always-value", @@ -156,9 +164,21 @@ func MachineDeploymentRolloutSpec(ctx context.Context, inputGetter func() Machin preExistingOnInitializationTaint, ) - Byf("Verify MachineDeployment Machines and Nodes have the correct taints") wlClient := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, clusterResources.Cluster.Namespace, clusterResources.Cluster.Name).GetClient() - verifyMachineAndNodeTaints(ctx, verifyMachineAndNodeTaintsInput{ + + Byf("Verify ControlPlane Machines and Nodes have the correct taints") + verifyControlPlaneMachineAndNodeTaints(ctx, verifyControlPlaneMachineAndNodeTaintsInput{ + BootstrapClusterClient: input.BootstrapClusterProxy.GetClient(), + WorkloadClusterClient: wlClient, + ClusterName: clusterResources.Cluster.Name, + Namespace: clusterResources.Cluster.Namespace, + ControlPlaneReplicas: clusterResources.ControlPlane.Spec.Replicas, + MachineTaints: wantMachineTaints, + NodeTaints: append(wantNodeTaints, additionalControlPlaneNodeTaint), + }) + + Byf("Verify MachineDeployment Machines and Nodes have the correct taints") + verifyMachineDeploymentMachineAndNodeTaints(ctx, verifyMachineDeploymentMachineAndNodeTaintsInput{ BootstrapClusterClient: input.BootstrapClusterProxy.GetClient(), WorkloadClusterClient: wlClient, ClusterName: clusterResources.Cluster.Name, @@ -167,7 +187,7 @@ func MachineDeploymentRolloutSpec(ctx context.Context, inputGetter func() Machin NodeTaints: wantNodeTaints, }) - Byf("Verify in-place propagation by adding new taints to the MachineDeployment") + Byf("Verify in-place propagation by adding new taints to the ControlPlane and MachineDeployment") wantMachineTaints = []clusterv1.MachineTaint{ preExistingAlwaysTaint, preExistingOnInitializationTaint, @@ -179,6 +199,12 @@ func MachineDeploymentRolloutSpec(ctx context.Context, inputGetter func() Machin preExistingOnInitializationTaint, addingAlwaysTaint, ) + + patchHelper, err := patch.NewHelper(clusterResources.ControlPlane, input.BootstrapClusterProxy.GetClient()) + Expect(err).ToNot(HaveOccurred()) + clusterResources.ControlPlane.Spec.MachineTemplate.Spec.Taints = wantMachineTaints + Expect(patchHelper.Patch(ctx, clusterResources.ControlPlane)).To(Succeed()) + for _, md := range clusterResources.MachineDeployments { patchHelper, err := patch.NewHelper(md, input.BootstrapClusterProxy.GetClient()) Expect(err).ToNot(HaveOccurred()) @@ -186,7 +212,17 @@ func MachineDeploymentRolloutSpec(ctx context.Context, inputGetter func() Machin Expect(patchHelper.Patch(ctx, md)).To(Succeed()) } - verifyMachineAndNodeTaints(ctx, verifyMachineAndNodeTaintsInput{ + verifyControlPlaneMachineAndNodeTaints(ctx, verifyControlPlaneMachineAndNodeTaintsInput{ + BootstrapClusterClient: input.BootstrapClusterProxy.GetClient(), + WorkloadClusterClient: wlClient, + ClusterName: clusterResources.Cluster.Name, + Namespace: clusterResources.Cluster.Namespace, + ControlPlaneReplicas: clusterResources.ControlPlane.Spec.Replicas, + MachineTaints: wantMachineTaints, + NodeTaints: append(wantNodeTaints, additionalControlPlaneNodeTaint), + }) + + verifyMachineDeploymentMachineAndNodeTaints(ctx, verifyMachineDeploymentMachineAndNodeTaintsInput{ BootstrapClusterClient: input.BootstrapClusterProxy.GetClient(), WorkloadClusterClient: wlClient, ClusterName: clusterResources.Cluster.Name, @@ -221,7 +257,17 @@ func MachineDeploymentRolloutSpec(ctx context.Context, inputGetter func() Machin addingAlwaysTaint, ) - verifyMachineAndNodeTaints(ctx, verifyMachineAndNodeTaintsInput{ + verifyControlPlaneMachineAndNodeTaints(ctx, verifyControlPlaneMachineAndNodeTaintsInput{ + BootstrapClusterClient: input.BootstrapClusterProxy.GetClient(), + WorkloadClusterClient: wlClient, + ClusterName: clusterResources.Cluster.Name, + Namespace: clusterResources.Cluster.Namespace, + ControlPlaneReplicas: clusterResources.ControlPlane.Spec.Replicas, + MachineTaints: wantMachineTaints, + NodeTaints: append(wantNodeTaints, additionalControlPlaneNodeTaint), + }) + + verifyMachineDeploymentMachineAndNodeTaints(ctx, verifyMachineDeploymentMachineAndNodeTaintsInput{ BootstrapClusterClient: input.BootstrapClusterProxy.GetClient(), WorkloadClusterClient: wlClient, ClusterName: clusterResources.Cluster.Name, @@ -230,12 +276,18 @@ func MachineDeploymentRolloutSpec(ctx context.Context, inputGetter func() Machin NodeTaints: wantNodeTaints, }) - Byf("Verify in-place propagation by removing taints from the MachineDeployment") + Byf("Verify in-place propagation by removing taints from ControlPlane and the MachineDeployment") wantMachineTaints = []clusterv1.MachineTaint{ preExistingOnInitializationTaint, addingOnInitializationTaint, } wantNodeTaints = toCoreV1Taints() + + patchHelper, err = patch.NewHelper(clusterResources.ControlPlane, input.BootstrapClusterProxy.GetClient()) + Expect(err).ToNot(HaveOccurred()) + clusterResources.ControlPlane.Spec.MachineTemplate.Spec.Taints = wantMachineTaints + Expect(patchHelper.Patch(ctx, clusterResources.ControlPlane)).To(Succeed()) + for _, md := range clusterResources.MachineDeployments { patchHelper, err := patch.NewHelper(md, input.BootstrapClusterProxy.GetClient()) Expect(err).ToNot(HaveOccurred()) @@ -243,7 +295,17 @@ func MachineDeploymentRolloutSpec(ctx context.Context, inputGetter func() Machin Expect(patchHelper.Patch(ctx, md)).To(Succeed()) } - verifyMachineAndNodeTaints(ctx, verifyMachineAndNodeTaintsInput{ + verifyControlPlaneMachineAndNodeTaints(ctx, verifyControlPlaneMachineAndNodeTaintsInput{ + BootstrapClusterClient: input.BootstrapClusterProxy.GetClient(), + WorkloadClusterClient: wlClient, + ClusterName: clusterResources.Cluster.Name, + Namespace: clusterResources.Cluster.Namespace, + ControlPlaneReplicas: clusterResources.ControlPlane.Spec.Replicas, + MachineTaints: wantMachineTaints, + NodeTaints: append(wantNodeTaints, additionalControlPlaneNodeTaint), + }) + + verifyMachineDeploymentMachineAndNodeTaints(ctx, verifyMachineDeploymentMachineAndNodeTaintsInput{ BootstrapClusterClient: input.BootstrapClusterProxy.GetClient(), WorkloadClusterClient: wlClient, ClusterName: clusterResources.Cluster.Name, @@ -289,7 +351,46 @@ func MachineDeploymentRolloutSpec(ctx context.Context, inputGetter func() Machin }) } -type verifyMachineAndNodeTaintsInput struct { +type verifyControlPlaneMachineAndNodeTaintsInput struct { + BootstrapClusterClient client.Client + WorkloadClusterClient client.Client + ClusterName string + Namespace string + ControlPlaneReplicas *int32 + MachineTaints []clusterv1.MachineTaint + NodeTaints []corev1.Taint +} + +func verifyControlPlaneMachineAndNodeTaints(ctx context.Context, input verifyControlPlaneMachineAndNodeTaintsInput) { + Expect(ctx).NotTo(BeNil(), "ctx is required for verifyControlPlaneMachineAndNodeTaints") + Expect(input.BootstrapClusterClient).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterClient can't be nil when calling verifyControlPlaneMachineAndNodeTaints") + Expect(input.WorkloadClusterClient).ToNot(BeNil(), "Invalid argument. input.WorkloadClusterClient can't be nil when calling verifyControlPlaneMachineAndNodeTaints") + Expect(input.ClusterName).NotTo(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling verifyControlPlaneMachineAndNodeTaints") + Expect(input.Namespace).NotTo(BeEmpty(), "Invalid argument. input.Namespace can't be empty when calling verifyControlPlaneMachineAndNodeTaints") + Expect(input.ControlPlaneReplicas).NotTo(BeNil(), "Invalid argument. input.ControlPlaneReplicas can't be nil when calling verifyControlPlaneMachineAndNodeTaints") + + Eventually(func(g Gomega) { + if input.ControlPlaneReplicas != nil { + machines := framework.GetControlPlaneMachinesByCluster(ctx, framework.GetControlPlaneMachinesByClusterInput{ + Lister: input.BootstrapClusterClient, + ClusterName: input.ClusterName, + Namespace: input.Namespace, + }) + + g.Expect(machines).To(HaveLen(int(ptr.Deref(input.ControlPlaneReplicas, 0)))) + for _, machine := range machines { + g.Expect(machine.Spec.Taints).To(ConsistOf(input.MachineTaints)) + g.Expect(machine.Status.NodeRef.IsDefined()).To(BeTrue()) + + node := &corev1.Node{} + g.Expect(input.WorkloadClusterClient.Get(ctx, client.ObjectKey{Name: machine.Status.NodeRef.Name}, node)).To(Succeed()) + g.Expect(node.Spec.Taints).To(ConsistOf(input.NodeTaints)) + } + } + }, "1m").Should(Succeed()) +} + +type verifyMachineDeploymentMachineAndNodeTaintsInput struct { BootstrapClusterClient client.Client WorkloadClusterClient client.Client ClusterName string @@ -298,12 +399,12 @@ type verifyMachineAndNodeTaintsInput struct { NodeTaints []corev1.Taint } -func verifyMachineAndNodeTaints(ctx context.Context, input verifyMachineAndNodeTaintsInput) { - Expect(ctx).NotTo(BeNil(), "ctx is required for verifyMachineAndNodeTaints") - Expect(input.BootstrapClusterClient).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterClient can't be nil when calling verifyMachineAndNodeTaints") - Expect(input.WorkloadClusterClient).ToNot(BeNil(), "Invalid argument. input.WorkloadClusterClient can't be nil when calling verifyMachineAndNodeTaints") - Expect(input.ClusterName).NotTo(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling verifyMachineAndNodeTaints") - Expect(input.MachineDeployments).NotTo(BeNil(), "Invalid argument. input.MachineDeployments can't be nil when calling verifyMachineAndNodeTaints") +func verifyMachineDeploymentMachineAndNodeTaints(ctx context.Context, input verifyMachineDeploymentMachineAndNodeTaintsInput) { + Expect(ctx).NotTo(BeNil(), "ctx is required for verifyMachineDeploymentMachineAndNodeTaints") + Expect(input.BootstrapClusterClient).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterClient can't be nil when calling verifyMachineDeploymentMachineAndNodeTaints") + Expect(input.WorkloadClusterClient).ToNot(BeNil(), "Invalid argument. input.WorkloadClusterClient can't be nil when calling verifyMachineDeploymentMachineAndNodeTaints") + Expect(input.ClusterName).NotTo(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling verifyMachineDeploymentMachineAndNodeTaints") + Expect(input.MachineDeployments).NotTo(BeNil(), "Invalid argument. input.MachineDeployments can't be nil when calling verifyMachineDeploymentMachineAndNodeTaints") Eventually(func(g Gomega) { for _, md := range input.MachineDeployments {