feat: support TalosControlPlane rolling upgrade

Track Kubernetes version changes in `TalosControlPlane` resource.
When changed recreate nodes one by one until all nodes have the desired
Kubernetes version.
The same happens when `infrastructureTemplate` is updated.

Added new parameters to the `TalosControlPlane` resource which specify
controlplanes rollout strategy and strategy configuration.

Enable `Defaulter` webhook to populate all created/updated `TalosControlPlane`
resources rollout strategy.

Signed-off-by: Artem Chernyshev <[email protected]>

Author: Unix4ever

Dockerfile

@@ -40,8 +40,7 @@ ARG NAME
 RUN --mount=type=cache,target=/.cache controller-gen crd:crdVersions=v1 paths="./api/..." output:crd:dir=config/crd/bases output:webhook:dir=config/webhook webhook
 RUN --mount=type=cache,target=/.cache controller-gen rbac:roleName=manager-role paths="./controllers/..." output:rbac:dir=config/rbac
 FROM scratch AS manifests
-COPY --from=manifests-build /src/config/crd /config/crd
-COPY --from=manifests-build /src/config/rbac /config/rbac
+COPY --from=manifests-build /src/config /config
 
 FROM build AS generate-build
 RUN --mount=type=cache,target=/.cache controller-gen object:headerFile=./hack/boilerplate.go.txt paths="./..."

api/v1alpha3/conditions.go

@@ -38,6 +38,16 @@ const (
 	InvalidControlPlaneConfigReason = "InvalidControlPlaneConfig"
 )
 
+const (
+	// MachinesSpecUpToDateCondition documents that the spec of the machines controlled by the TalosControlPlane
+	// is up to date. When this condition is false, the TalosControlPlane is executing a rolling upgrade.
+	MachinesSpecUpToDateCondition clusterv1.ConditionType = "MachinesSpecUpToDate"
+
+	// RollingUpdateInProgressReason (Severity=Warning) documents a TalosControlPlane object executing a
+	// rolling upgrade for aligning the machines spec to the desired state.
+	RollingUpdateInProgressReason = "RollingUpdateInProgress"
+)
+
 const (
 	// ResizedCondition documents a TalosControlPlane that is resizing the set of controlled machines.
 	ResizedCondition clusterv1.ConditionType = "Resized"

api/v1alpha3/taloscontrolplane_types.go

@@ -8,6 +8,7 @@ import (
 	cabptv1 "github.com/talos-systems/cluster-api-bootstrap-provider-talos/api/v1alpha3"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 )
 
@@ -21,6 +22,15 @@ type ControlPlaneConfig struct {
 	ControlPlaneConfig cabptv1.TalosConfigSpec `json:"controlplane"`
 }
 
+// RolloutStrategyType defines the rollout strategies for a KubeadmControlPlane.
+type RolloutStrategyType string
+
+const (
+	// RollingUpdateStrategyType replaces the old control planes by new one using rolling update
+	// i.e. gradually scale up or down the old control planes and scale up or down the new one.
+	RollingUpdateStrategyType RolloutStrategyType = "RollingUpdate"
+)
+
 // TalosControlPlaneSpec defines the desired state of TalosControlPlane
 type TalosControlPlaneSpec struct {
 	// Number of desired machines. Defaults to 1. When stacked etcd is used only
@@ -41,6 +51,39 @@ type TalosControlPlaneSpec struct {
 	// ControlPlaneConfig is a two TalosConfigSpecs
 	// to use for initializing and joining machines to the control plane.
 	ControlPlaneConfig ControlPlaneConfig `json:"controlPlaneConfig"`
+
+	// The RolloutStrategy to use to replace control plane machines with
+	// new ones.
+	// +optional
+	// +kubebuilder:default={type: "RollingUpdate", rollingUpdate: {maxSurge: 1}}
+	RolloutStrategy *RolloutStrategy `json:"rolloutStrategy,omitempty"`
+}
+
+// RolloutStrategy describes how to replace existing machines
+// with new ones.
+type RolloutStrategy struct {
+	// Rolling update config params. Present only if
+	// RolloutStrategyType = RollingUpdate.
+	// +optional
+	RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"`
+
+	// Type of rollout. Currently the only supported strategy is
+	// "RollingUpdate".
+	// Default is RollingUpdate.
+	// +optional
+	Type RolloutStrategyType `json:"type,omitempty"`
+}
+
+// RollingUpdate is used to control the desired behavior of rolling update.
+type RollingUpdate struct {
+	// The maximum number of control planes that can be scheduled above or under the
+	// desired number of control planes.
+	// Value can be an absolute number 1 or 0.
+	// Defaults to 1.
+	// Example: when this is set to 1, the control plane can be scaled
+	// up immediately when the rolling update starts.
+	// +optional
+	MaxSurge *intstr.IntOrString `json:"maxSurge,omitempty"`
 }
 
 // TalosControlPlaneStatus defines the observed state of TalosControlPlane
@@ -126,13 +169,13 @@ type TalosControlPlane struct {
 }
 
 // GetConditions returns the set of conditions for this object.
-func (in *TalosControlPlane) GetConditions() clusterv1.Conditions {
-	return in.Status.Conditions
+func (r *TalosControlPlane) GetConditions() clusterv1.Conditions {
+	return r.Status.Conditions
 }
 
 // SetConditions sets the conditions on this object.
-func (in *TalosControlPlane) SetConditions(conditions clusterv1.Conditions) {
-	in.Status.Conditions = conditions
+func (r *TalosControlPlane) SetConditions(conditions clusterv1.Conditions) {
+	r.Status.Conditions = conditions
 }
 
 // +kubebuilder:object:root=true

api/v1alpha3/taloscontrolplane_webhook.go

@@ -1,11 +1,61 @@
 package v1alpha3
 
 import (
+	"strings"
+
+	"k8s.io/apimachinery/pkg/util/intstr"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
 )
 
+// SetupWebhookWithManager implements webhook methods.
 func (r *TalosControlPlane) SetupWebhookWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewWebhookManagedBy(mgr).
 		For(r).
 		Complete()
 }
+
+// +kubebuilder:webhook:verbs=create;update,path=/mutate-controlplane-cluster-x-k8s-io-v1beta1-kubeadmcontrolplane,mutating=true,failurePolicy=fail,matchPolicy=Equivalent,groups=controlplane.cluster.x-k8s.io,resources=kubeadmcontrolplanes,versions=v1beta1,name=default.kubeadmcontrolplane.controlplane.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1
+
+var _ webhook.Defaulter = &TalosControlPlane{}
+
+// Default implements webhook.Defaulter so a webhook will be registered for the type.
+func (r *TalosControlPlane) Default() {
+	defaultTalosControlPlaneSpec(&r.Spec, r.Namespace)
+}
+
+func defaultTalosControlPlaneSpec(s *TalosControlPlaneSpec, namespace string) {
+	if s.Replicas == nil {
+		replicas := int32(1)
+		s.Replicas = &replicas
+	}
+
+	if !strings.HasPrefix(s.Version, "v") {
+		s.Version = "v" + s.Version
+	}
+
+	s.RolloutStrategy = defaultRolloutStrategy(s.RolloutStrategy)
+}
+
+func defaultRolloutStrategy(rolloutStrategy *RolloutStrategy) *RolloutStrategy {
+	ios1 := intstr.FromInt(1)
+
+	if rolloutStrategy == nil {
+		rolloutStrategy = &RolloutStrategy{}
+	}
+
+	// Enforce RollingUpdate strategy and default MaxSurge if not set.
+	if rolloutStrategy != nil {
+		if len(rolloutStrategy.Type) == 0 {
+			rolloutStrategy.Type = RollingUpdateStrategyType
+		}
+		if rolloutStrategy.Type == RollingUpdateStrategyType {
+			if rolloutStrategy.RollingUpdate == nil {
+				rolloutStrategy.RollingUpdate = &RollingUpdate{}
+			}
+			rolloutStrategy.RollingUpdate.MaxSurge = intstr.ValueOrDefault(rolloutStrategy.RollingUpdate.MaxSurge, ios1)
+		}
+	}
+
+	return rolloutStrategy
+}

api/v1alpha3/zz_generated.deepcopy.go

@@ -183,6 +183,34 @@ spec:
                   This is a pointer to distinguish between explicit zero and not specified.
                 format: int32
                 type: integer
+              rolloutStrategy:
+                default:
+                  rollingUpdate:
+                    maxSurge: 1
+                  type: RollingUpdate
+                description: The RolloutStrategy to use to replace control plane machines
+                  with new ones.
+                properties:
+                  rollingUpdate:
+                    description: Rolling update config params. Present only if RolloutStrategyType
+                      = RollingUpdate.
+                    properties:
+                      maxSurge:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        description: 'The maximum number of control planes that can
+                          be scheduled above or under the desired number of control
+                          planes. Value can be an absolute number 1 or 0. Defaults
+                          to 1. Example: when this is set to 1, the control plane
+                          can be scaled up immediately when the rolling update starts.'
+                        x-kubernetes-int-or-string: true
+                    type: object
+                  type:
+                    description: Type of rollout. Currently the only supported strategy
+                      is "RollingUpdate". Default is RollingUpdate.
+                    type: string
+                type: object
               version:
                 description: Version defines the desired Kubernetes version.
                 minLength: 2

config/crd/bases/controlplane.cluster.x-k8s.io_taloscontrolplanes.yaml

@@ -18,7 +18,7 @@ bases:
 
 patchesStrategicMerge:
   - manager_webhook_patch.yaml
-#  - webhookcainjection_patch.yaml
+  - webhookcainjection_patch.yaml
 
 vars:
   - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR

config/default/kustomization.yaml

@@ -1,15 +1,8 @@
 # This patch add annotation to admission webhook config and
 # the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize.
-apiVersion: admissionregistration.k8s.io/v1beta1
+apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
   name: mutating-webhook-configuration
   annotations:
     cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
----
-apiVersion: admissionregistration.k8s.io/v1beta1
-kind: ValidatingWebhookConfiguration
-metadata:
-  name: validating-webhook-configuration
-  annotations:
-    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)

config/default/webhookcainjection_patch.yaml

@@ -0,0 +1,29 @@
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  creationTimestamp: null
+  name: mutating-webhook-configuration
+webhooks:
+- admissionReviewVersions:
+  - v1
+  - v1beta1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-controlplane-cluster-x-k8s-io-v1beta1-kubeadmcontrolplane
+  failurePolicy: Fail
+  matchPolicy: Equivalent
+  name: default.kubeadmcontrolplane.controlplane.cluster.x-k8s.io
+  rules:
+  - apiGroups:
+    - controlplane.cluster.x-k8s.io
+    apiVersions:
+    - v1beta1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - kubeadmcontrolplanes
+  sideEffects: None

config/webhook/manifests.yaml

@@ -1,4 +1,4 @@
-
+---
 apiVersion: v1
 kind: Service
 metadata: