Skip to content

Commit 9dc5f13

Browse files
sbueringersbueringer
committed
ClusterClass: Allow fine-granular control of MachineDeployment upgrades
Signed-off-by: Stefan Büringer [email protected]
1 parent 329753c commit 9dc5f13

File tree

10 files changed

+430
-72
lines changed

10 files changed

+430
-72
lines changed

‎api/v1beta1/common_types.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,26 @@ const (
3333
// to track the name of the MachineDeployment topology it represents.
3434
ClusterTopologyMachineDeploymentNameLabel = "topology.cluster.x-k8s.io/deployment-name"
3535

36+
// ClusterTopologyHoldUpgradeSequenceAnnotation can be used to hold the entire MachineDeployment upgrade sequence.
37+
// If the annotation is set on a MachineDeployment topology in Cluster.spec.topology.workers, the Kubernetes upgrade
38+
// for this MachineDeployment topology and all subsequent ones is deferred.
39+
// Examples:
40+
// - If you want to pause upgrade after CP upgrade, this annotation should be applied to the first MachineDeployment
41+
// in the list of MachineDeployments in Cluster.spec.topology. The upgrade will not be completed until the annotation
42+
// is removed and all MachineDeployments are upgraded.
43+
// - If you want to pause upgrade after the 50th MachineDeployment, this annotation should be applied to the 51st
44+
// MachineDeployment in the list.
45+
ClusterTopologyHoldUpgradeSequenceAnnotation = "topology.cluster.x-k8s.io/hold-upgrade-sequence"
46+
47+
// ClusterTopologyDeferUpgradeAnnotation can be used to defer the Kubernetes upgrade of a single MachineDeployment topology.
48+
// If the annotation is set on a MachineDeployment topology in Cluster.spec.topology.workers, the Kubernetes upgrade
49+
// for this MachineDeployment topology is deferred. It doesn't affect other MachineDeployment topologies.
50+
// Example:
51+
// - If you want to defer the upgrades of the 3rd and 5th MachineDeployments of the list, set the annotation on them.
52+
// The upgrade process will upgrade MachineDeployment in position 1,2, (skip 3), 4, (skip 5), 6 etc. The upgrade
53+
// will not be completed until the annotation is removed and all MachineDeployments are upgraded.
54+
ClusterTopologyDeferUpgradeAnnotation = "topology.cluster.x-k8s.io/defer-upgrade"
55+
3656
// ClusterTopologyUnsafeUpdateClassNameAnnotation can be used to disable the webhook check on
3757
// update that disallows a pre-existing Cluster to be populated with Topology information and Class.
3858
ClusterTopologyUnsafeUpdateClassNameAnnotation = "unsafe.topology.cluster.x-k8s.io/disable-update-class-name-check"

‎api/v1beta1/condition_consts.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,10 @@ const (
273273
// not yet completed because at least one of the MachineDeployments is not yet updated to match the desired topology spec.
274274
TopologyReconciledMachineDeploymentsUpgradePendingReason = "MachineDeploymentsUpgradePending"
275275

276+
// TopologyReconciledMachineDeploymentsUpgradeDeferredReason (Severity=Info) documents reconciliation of a Cluster topology
277+
// not yet completed because the upgrade for at least one of the MachineDeployments has been deferred.
278+
TopologyReconciledMachineDeploymentsUpgradeDeferredReason = "MachineDeploymentsUpgradeDeferred"
279+
276280
// TopologyReconciledHookBlockingReason (Severity=Info) documents reconciliation of a Cluster topology
277281
// not yet completed because at least one of the lifecycle hooks is blocking.
278282
TopologyReconciledHookBlockingReason = "LifecycleHookBlocking"

‎docs/book/src/reference/labels_and_annotations.md

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
11
**Supported Labels:**
22

33

4-
| Label | Note |
5-
|:--------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
6-
| cluster.x-k8s.io/cluster-name | It is set on machines linked to a cluster and external objects(bootstrap and infrastructure providers). |
7-
| topology.cluster.x-k8s.io/owned | It is set on all the object which are managed as part of a ClusterTopology. |
8-
| topology.cluster.x-k8s.io/deployment-name | It is set on the generated MachineDeployment objects to track the name of the MachineDeployment topology it represents. |
9-
| cluster.x-k8s.io/provider | It is set on components in the provider manifest. The label allows one to easily identify all the components belonging to a provider. The clusterctl tool uses this label for implementing provider's lifecycle operations. |
10-
| cluster.x-k8s.io/watch-filter | It can be applied to any Cluster API object. Controllers which allow for selective reconciliation may check this label and proceed with reconciliation of the object only if this label and a configured value is present. |
11-
| cluster.x-k8s.io/interruptible | It is used to mark the nodes that run on interruptible instances. |
12-
| cluster.x-k8s.io/control-plane | It is set on machines or related objects that are part of a control plane. |
13-
| cluster.x-k8s.io/set-name | It is set on machines if they're controlled by MachineSet. The value of this label may be a hash if the MachineSet name is longer than 63 characters. |
14-
| cluster.x-k8s.io/control-plane-name | It is set on machines if they're controlled by a contorl plane. The value of this label may be a hash if the control plane name is longer than 63 characters. |
15-
| cluster.x-k8s.io/deployment-name | It is set on machines if they're controlled by a MachineDeployment. |
16-
| machine-template-hash | It is applied to Machines in a MachineDeployment containing the hash of the template. |
4+
| Label | Note |
5+
|:------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
6+
| cluster.x-k8s.io/cluster-name | It is set on machines linked to a cluster and external objects(bootstrap and infrastructure providers). |
7+
| topology.cluster.x-k8s.io/owned | It is set on all the object which are managed as part of a ClusterTopology. |
8+
| topology.cluster.x-k8s.io/deployment-name | It is set on the generated MachineDeployment objects to track the name of the MachineDeployment topology it represents. |
9+
| cluster.x-k8s.io/provider | It is set on components in the provider manifest. The label allows one to easily identify all the components belonging to a provider. The clusterctl tool uses this label for implementing provider's lifecycle operations. |
10+
| cluster.x-k8s.io/watch-filter | It can be applied to any Cluster API object. Controllers which allow for selective reconciliation may check this label and proceed with reconciliation of the object only if this label and a configured value is present. |
11+
| cluster.x-k8s.io/interruptible | It is used to mark the nodes that run on interruptible instances. |
12+
| cluster.x-k8s.io/control-plane | It is set on machines or related objects that are part of a control plane. |
13+
| cluster.x-k8s.io/set-name | It is set on machines if they're controlled by MachineSet. The value of this label may be a hash if the MachineSet name is longer than 63 characters. |
14+
| cluster.x-k8s.io/control-plane-name | It is set on machines if they're controlled by a control plane. The value of this label may be a hash if the control plane name is longer than 63 characters. |
15+
| cluster.x-k8s.io/deployment-name | It is set on machines if they're controlled by a MachineDeployment. |
16+
| machine-template-hash | It is applied to Machines in a MachineDeployment containing the hash of the template. |
1717
<br>
1818

1919

@@ -36,7 +36,9 @@
3636
| cluster.x-k8s.io/skip-remediation | It is used to mark the machines that should not be considered for remediation by MachineHealthCheck reconciler. |
3737
| cluster.x-k8s.io/managed-by | It can be applied to InfraCluster resources to signify that some external system is managing the cluster infrastructure. Provider InfraCluster controllers will ignore resources with this annotation. An external controller must fulfill the contract of the InfraCluster resource. External infrastructure providers should ensure that the annotation, once set, cannot be removed. |
3838
| cluster.x-k8s.io/replicas-managed-by | It can be applied to MachinePool resources to signify that some external system is managing infrastructure scaling for that pool. See [the MachinePool documentation](../developer/architecture/controllers/machine-pool.md#externally-managed-autoscaler) for more details. |
39+
| topology.cluster.x-k8s.io/defer-upgrade | It can be used to defer the Kubernetes upgrade of a single MachineDeployment topology. If the annotation is set on a MachineDeployment topology in Cluster.spec.topology.workers, the Kubernetes upgrade for this MachineDeployment topology is deferred. It doesn't affect other MachineDeployment topologies. |
3940
| topology.cluster.x-k8s.io/dry-run | It is an annotation that gets set on objects by the topology controller only during a server side dry run apply operation. It is used for validating update webhooks for objects which get updated by template rotation (e.g. InfrastructureMachineTemplate). When the annotation is set and the admission request is a dry run, the webhook should deny validation due to immutability. By that the request will succeed (without any changes to the actual object because it is a dry run) and the topology controller will receive the resulting object. |
41+
| topology.cluster.x-k8s.io/hold-upgrade-sequence | It can be used to hold the entire MachineDeployment upgrade sequence. If the annotation is set on a MachineDeployment topology in Cluster.spec.topology.workers, the Kubernetes upgrade for this MachineDeployment topology and all subsequent ones is deferred. |
4042
| machine.cluster.x-k8s.io/certificates-expiry | It captures the expiry date of the machine certificates in RFC3339 format. It is used to trigger rollout of control plane machines before certificates expire. It can be set on BootstrapConfig and Machine objects. The value set on Machine object takes precedence. The annotation is only used by control plane machines. |
4143
| machine.cluster.x-k8s.io/exclude-node-draining | It explicitly skips node draining if set. |
4244
| machine.cluster.x-k8s.io/exclude-wait-for-node-volume-detach | It explicitly skips the waiting for node volume detaching if set. |

‎internal/controllers/topology/cluster/conditions.go

Lines changed: 36 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -91,40 +91,52 @@ func (r *Reconciler) reconcileTopologyReconciledCondition(s *scope.Scope, cluste
9191
return nil
9292
}
9393

94-
// If either the Control Plane or any of the MachineDeployments are still pending to pick up the new version (generally
95-
// happens when upgrading the cluster) then the topology is not considered as fully reconciled.
96-
if s.UpgradeTracker.ControlPlane.PendingUpgrade || s.UpgradeTracker.MachineDeployments.PendingUpgrade() {
94+
// The topology is not considered as fully reconciled if one of the following is true:
95+
// * either the Control Plane or any of the MachineDeployments are still pending to pick up the new version
96+
// (generally happens when upgrading the cluster)
97+
// * when there are MachineDeployments for which the upgrade has been deferred
98+
if s.UpgradeTracker.ControlPlane.PendingUpgrade ||
99+
s.UpgradeTracker.MachineDeployments.PendingUpgrade() ||
100+
s.UpgradeTracker.MachineDeployments.DeferredUpgrade() {
97101
msgBuilder := &strings.Builder{}
98102
var reason string
99-
if s.UpgradeTracker.ControlPlane.PendingUpgrade {
100-
msgBuilder.WriteString(fmt.Sprintf("Control plane upgrade to %s on hold. ", s.Blueprint.Topology.Version))
103+
104+
switch {
105+
case s.UpgradeTracker.ControlPlane.PendingUpgrade:
106+
msgBuilder.WriteString(fmt.Sprintf("Control plane upgrade to %s on hold.", s.Blueprint.Topology.Version))
101107
reason = clusterv1.TopologyReconciledControlPlaneUpgradePendingReason
102-
} else {
103-
msgBuilder.WriteString(fmt.Sprintf("MachineDeployment(s) %s upgrade to version %s on hold. ",
104-
strings.Join(s.UpgradeTracker.MachineDeployments.PendingUpgradeNames(), ", "),
108+
case s.UpgradeTracker.MachineDeployments.PendingUpgrade():
109+
msgBuilder.WriteString(fmt.Sprintf("MachineDeployment(s) %s upgrade to version %s on hold.",
110+
computeMachineDeploymentNameList(s.UpgradeTracker.MachineDeployments.PendingUpgradeNames()),
105111
s.Blueprint.Topology.Version,
106112
))
107113
reason = clusterv1.TopologyReconciledMachineDeploymentsUpgradePendingReason
114+
case s.UpgradeTracker.MachineDeployments.DeferredUpgrade():
115+
msgBuilder.WriteString(fmt.Sprintf("MachineDeployment(s) %s upgrade to version %s deferred.",
116+
computeMachineDeploymentNameList(s.UpgradeTracker.MachineDeployments.DeferredUpgradeNames()),
117+
s.Blueprint.Topology.Version,
118+
))
119+
reason = clusterv1.TopologyReconciledMachineDeploymentsUpgradeDeferredReason
108120
}
109121

110122
switch {
111123
case s.UpgradeTracker.ControlPlane.IsProvisioning:
112-
msgBuilder.WriteString("Control plane is completing initial provisioning")
124+
msgBuilder.WriteString(" Control plane is completing initial provisioning")
113125

114126
case s.UpgradeTracker.ControlPlane.IsUpgrading:
115127
cpVersion, err := contract.ControlPlane().Version().Get(s.Current.ControlPlane.Object)
116128
if err != nil {
117129
return errors.Wrap(err, "failed to get control plane spec version")
118130
}
119-
msgBuilder.WriteString(fmt.Sprintf("Control plane is upgrading to version %s", *cpVersion))
131+
msgBuilder.WriteString(fmt.Sprintf(" Control plane is upgrading to version %s", *cpVersion))
120132

121133
case s.UpgradeTracker.ControlPlane.IsScaling:
122-
msgBuilder.WriteString("Control plane is reconciling desired replicas")
134+
msgBuilder.WriteString(" Control plane is reconciling desired replicas")
123135

124136
case s.Current.MachineDeployments.IsAnyRollingOut():
125-
msgBuilder.WriteString(fmt.Sprintf("MachineDeployment(s) %s are rolling out", strings.Join(
126-
s.UpgradeTracker.MachineDeployments.RolloutNames(), ", ",
127-
)))
137+
msgBuilder.WriteString(fmt.Sprintf(" MachineDeployment(s) %s are rolling out",
138+
computeMachineDeploymentNameList(s.UpgradeTracker.MachineDeployments.RolloutNames()),
139+
))
128140
}
129141

130142
conditions.Set(
@@ -149,3 +161,13 @@ func (r *Reconciler) reconcileTopologyReconciledCondition(s *scope.Scope, cluste
149161

150162
return nil
151163
}
164+
165+
// computeMachineDeploymentNameList computes the MD name list to be shown in conditions.
166+
// It shortens the list to at most 5 MachineDeployment names.
167+
func computeMachineDeploymentNameList(mdList []string) any {
168+
if len(mdList) > 5 {
169+
mdList = append(mdList[:5], "...")
170+
}
171+
172+
return strings.Join(mdList, ", ")
173+
}

0 commit comments

Comments
 (0)