Skip to content

Commit 9cbf19e

Browse files
author
Yuvaraj Kakaraparthi
committed
add support or concurrent MD upgrades
1 parent a2318d1 commit 9cbf19e

File tree

10 files changed

+321
-69
lines changed

10 files changed

+321
-69
lines changed

‎api/v1beta1/common_types.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,10 @@ const (
5353
// will not be completed until the annotation is removed and all MachineDeployments are upgraded.
5454
ClusterTopologyDeferUpgradeAnnotation = "topology.cluster.x-k8s.io/defer-upgrade"
5555

56+
// ClusterTopologyUpgradeConcurrencyAnnotation can be set as top-level annotation on the Cluster object of
57+
// a classy Cluster to define the maximum concurrency while upgrading MachineDeployments.
58+
ClusterTopologyUpgradeConcurrencyAnnotation = "topology.cluster.x-k8s.io/upgrade-concurrency"
59+
5660
// ClusterTopologyUnsafeUpdateClassNameAnnotation can be used to disable the webhook check on
5761
// update that disallows a pre-existing Cluster to be populated with Topology information and Class.
5862
ClusterTopologyUnsafeUpdateClassNameAnnotation = "unsafe.topology.cluster.x-k8s.io/disable-update-class-name-check"

‎docs/book/src/reference/labels_and_annotations.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
| topology.cluster.x-k8s.io/defer-upgrade | It can be used to defer the Kubernetes upgrade of a single MachineDeployment topology. If the annotation is set on a MachineDeployment topology in Cluster.spec.topology.workers, the Kubernetes upgrade for this MachineDeployment topology is deferred. It doesn't affect other MachineDeployment topologies. |
4040
| topology.cluster.x-k8s.io/dry-run | It is an annotation that gets set on objects by the topology controller only during a server side dry run apply operation. It is used for validating update webhooks for objects which get updated by template rotation (e.g. InfrastructureMachineTemplate). When the annotation is set and the admission request is a dry run, the webhook should deny validation due to immutability. By that the request will succeed (without any changes to the actual object because it is a dry run) and the topology controller will receive the resulting object. |
4141
| topology.cluster.x-k8s.io/hold-upgrade-sequence | It can be used to hold the entire MachineDeployment upgrade sequence. If the annotation is set on a MachineDeployment topology in Cluster.spec.topology.workers, the Kubernetes upgrade for this MachineDeployment topology and all subsequent ones is deferred. |
42+
| topology.cluster.x-k8s.io/upgrade-concurrency | It can be used to configure the maximum concurrency while upgrading MachineDeployments of a classy Cluster. It is set as a top level annotation on the Cluster object. The value should be >= 1. If unspecified the upgrade concurrency will default to 1. |
4243
| machine.cluster.x-k8s.io/certificates-expiry | It captures the expiry date of the machine certificates in RFC3339 format. It is used to trigger rollout of control plane machines before certificates expire. It can be set on BootstrapConfig and Machine objects. The value set on Machine object takes precedence. The annotation is only used by control plane machines. |
4344
| machine.cluster.x-k8s.io/exclude-node-draining | It explicitly skips node draining if set. |
4445
| machine.cluster.x-k8s.io/exclude-wait-for-node-volume-detach | It explicitly skips the waiting for node volume detaching if set. |

‎internal/controllers/topology/cluster/desired_state.go

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -841,14 +841,6 @@ func computeMachineDeploymentVersion(s *scope.Scope, machineDeploymentTopology c
841841
return currentVersion, nil
842842
}
843843

844-
// At this point the control plane is stable (not scaling, not upgrading, not being upgraded).
845-
// Checking to see if the machine deployments are also stable.
846-
// If any of the MachineDeployments is rolling out, do not upgrade the machine deployment yet.
847-
if s.Current.MachineDeployments.IsAnyRollingOut() {
848-
s.UpgradeTracker.MachineDeployments.MarkPendingUpgrade(currentMDState.Object.Name)
849-
return currentVersion, nil
850-
}
851-
852844
// Control plane and machine deployments are stable.
853845
// Ready to pick up the topology version.
854846
s.UpgradeTracker.MachineDeployments.MarkRollingOut(currentMDState.Object.Name)

‎internal/controllers/topology/cluster/desired_state_test.go

Lines changed: 93 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1635,23 +1635,25 @@ func TestComputeMachineDeployment(t *testing.T) {
16351635
}).
16361636
Build()
16371637

1638-
machineDeploymentStable := builder.MachineDeployment("test-namespace", "md-1").
1638+
machineDeploymentStable := builder.MachineDeployment("test-namespace", "md-stable").
16391639
WithGeneration(1).
16401640
WithReplicas(2).
16411641
WithStatus(clusterv1.MachineDeploymentStatus{
16421642
ObservedGeneration: 2,
16431643
Replicas: 2,
1644+
ReadyReplicas: 2,
16441645
UpdatedReplicas: 2,
16451646
AvailableReplicas: 2,
16461647
}).
16471648
Build()
16481649

1649-
machineDeploymentRollingOut := builder.MachineDeployment("test-namespace", "md-1").
1650+
machineDeploymentRollingOut := builder.MachineDeployment("test-namespace", "md-rolling").
16501651
WithGeneration(1).
16511652
WithReplicas(2).
16521653
WithStatus(clusterv1.MachineDeploymentStatus{
16531654
ObservedGeneration: 2,
16541655
Replicas: 1,
1656+
ReadyReplicas: 1,
16551657
UpdatedReplicas: 1,
16561658
AvailableReplicas: 1,
16571659
}).
@@ -1669,28 +1671,46 @@ func TestComputeMachineDeployment(t *testing.T) {
16691671
name string
16701672
machineDeploymentsState scope.MachineDeploymentsStateMap
16711673
currentMDVersion *string
1674+
upgradeConcurrency string
16721675
topologyVersion string
16731676
expectedVersion string
16741677
}{
16751678
{
16761679
name: "use cluster.spec.topology.version if creating a new machine deployment",
16771680
machineDeploymentsState: nil,
1681+
upgradeConcurrency: "1",
16781682
currentMDVersion: nil,
16791683
topologyVersion: "v1.2.3",
16801684
expectedVersion: "v1.2.3",
16811685
},
16821686
{
1683-
name: "use machine deployment's spec.template.spec.version if one of the machine deployments is rolling out",
1687+
name: "use machine deployment's spec.template.spec.version if one of the machine deployments is rolling out, concurrency limit reached",
16841688
machineDeploymentsState: machineDeploymentsStateRollingOut,
1689+
upgradeConcurrency: "1",
16851690
currentMDVersion: pointer.String("v1.2.2"),
16861691
topologyVersion: "v1.2.3",
16871692
expectedVersion: "v1.2.2",
16881693
},
1694+
{
1695+
name: "use cluster.spec.topology.version if one of the machine deployments is rolling out, concurrency limit not reached",
1696+
machineDeploymentsState: machineDeploymentsStateRollingOut,
1697+
upgradeConcurrency: "2",
1698+
currentMDVersion: pointer.String("v1.2.2"),
1699+
topologyVersion: "v1.2.3",
1700+
expectedVersion: "v1.2.3",
1701+
},
16891702
}
16901703
for _, tt := range tests {
16911704
t.Run(tt.name, func(t *testing.T) {
16921705
g := NewWithT(t)
1693-
s := scope.New(cluster)
1706+
1707+
testCluster := cluster.DeepCopy()
1708+
if testCluster.Annotations == nil {
1709+
testCluster.Annotations = map[string]string{}
1710+
}
1711+
testCluster.Annotations[clusterv1.ClusterTopologyUpgradeConcurrencyAnnotation] = tt.upgradeConcurrency
1712+
1713+
s := scope.New(testCluster)
16941714
s.Blueprint = blueprint
16951715
s.Blueprint.Topology.Version = tt.topologyVersion
16961716
s.Blueprint.Topology.ControlPlane = clusterv1.ControlPlaneTopology{
@@ -1709,6 +1729,7 @@ func TestComputeMachineDeployment(t *testing.T) {
17091729
WithStatus(clusterv1.MachineDeploymentStatus{
17101730
ObservedGeneration: 2,
17111731
Replicas: 2,
1732+
ReadyReplicas: 2,
17121733
UpdatedReplicas: 2,
17131734
AvailableReplicas: 2,
17141735
}).
@@ -1722,6 +1743,7 @@ func TestComputeMachineDeployment(t *testing.T) {
17221743
s.Current.ControlPlane = &scope.ControlPlaneState{
17231744
Object: controlPlaneStable123,
17241745
}
1746+
s.UpgradeTracker.MachineDeployments.MarkRollingOut(s.Current.MachineDeployments.RollingOut()...)
17251747
desiredControlPlaneState := &scope.ControlPlaneState{
17261748
Object: controlPlaneStable123,
17271749
}
@@ -1828,45 +1850,55 @@ func TestComputeMachineDeploymentVersion(t *testing.T) {
18281850
//
18291851
// A machine deployment is considered upgrading if any of the above conditions
18301852
// is false.
1831-
machineDeploymentStable := builder.MachineDeployment("test-namespace", "md-1").
1832-
WithGeneration(1).
1833-
WithReplicas(2).
1834-
WithStatus(clusterv1.MachineDeploymentStatus{
1835-
ObservedGeneration: 2,
1836-
Replicas: 2,
1837-
UpdatedReplicas: 2,
1838-
AvailableReplicas: 2,
1839-
ReadyReplicas: 2,
1840-
UnavailableReplicas: 0,
1841-
}).
1842-
Build()
1843-
machineDeploymentRollingOut := builder.MachineDeployment("test-namespace", "md-2").
1844-
WithGeneration(1).
1845-
WithReplicas(2).
1846-
WithStatus(clusterv1.MachineDeploymentStatus{
1847-
ObservedGeneration: 2,
1848-
Replicas: 1,
1849-
UpdatedReplicas: 1,
1850-
AvailableReplicas: 1,
1851-
ReadyReplicas: 1,
1852-
UnavailableReplicas: 1,
1853-
}).
1854-
Build()
1853+
stableMachineDeployment := func(ns, name string) *clusterv1.MachineDeployment {
1854+
return builder.MachineDeployment(ns, name).
1855+
WithGeneration(1).
1856+
WithReplicas(2).
1857+
WithStatus(clusterv1.MachineDeploymentStatus{
1858+
ObservedGeneration: 2,
1859+
Replicas: 2,
1860+
UpdatedReplicas: 2,
1861+
AvailableReplicas: 2,
1862+
ReadyReplicas: 2,
1863+
UnavailableReplicas: 0,
1864+
}).
1865+
Build()
1866+
}
18551867

1856-
machineDeploymentsStateStable := scope.MachineDeploymentsStateMap{
1857-
"md1": &scope.MachineDeploymentState{Object: machineDeploymentStable},
1858-
"md2": &scope.MachineDeploymentState{Object: machineDeploymentStable},
1868+
rollingMachineDeployment := func(ns, name string) *clusterv1.MachineDeployment {
1869+
return builder.MachineDeployment(ns, name).
1870+
WithGeneration(1).
1871+
WithReplicas(2).
1872+
WithStatus(clusterv1.MachineDeploymentStatus{
1873+
ObservedGeneration: 2,
1874+
Replicas: 1,
1875+
UpdatedReplicas: 1,
1876+
AvailableReplicas: 1,
1877+
ReadyReplicas: 1,
1878+
UnavailableReplicas: 1,
1879+
}).
1880+
Build()
1881+
}
1882+
1883+
twoMachineDeploymentsStateStable := scope.MachineDeploymentsStateMap{
1884+
"md1": &scope.MachineDeploymentState{Object: stableMachineDeployment("test1", "md1")},
1885+
"md2": &scope.MachineDeploymentState{Object: stableMachineDeployment("test1", "md2")},
1886+
}
1887+
oneStableOneRollingMachineDeploymentState := scope.MachineDeploymentsStateMap{
1888+
"md1": &scope.MachineDeploymentState{Object: stableMachineDeployment("test1", "md1")},
1889+
"md2": &scope.MachineDeploymentState{Object: rollingMachineDeployment("test1", "md2")},
18591890
}
1860-
machineDeploymentsStateRollingOut := scope.MachineDeploymentsStateMap{
1861-
"md1": &scope.MachineDeploymentState{Object: machineDeploymentStable},
1862-
"md2": &scope.MachineDeploymentState{Object: machineDeploymentRollingOut},
1891+
twoRollingMachineDeploymentState := scope.MachineDeploymentsStateMap{
1892+
"md1": &scope.MachineDeploymentState{Object: rollingMachineDeployment("test1", "md1")},
1893+
"md2": &scope.MachineDeploymentState{Object: rollingMachineDeployment("test1", "md2")},
18631894
}
18641895

18651896
tests := []struct {
18661897
name string
18671898
machineDeploymentTopology clusterv1.MachineDeploymentTopology
18681899
currentMachineDeploymentState *scope.MachineDeploymentState
18691900
machineDeploymentsStateMap scope.MachineDeploymentsStateMap
1901+
upgradeConcurrency int
18701902
currentControlPlane *unstructured.Unstructured
18711903
desiredControlPlane *unstructured.Unstructured
18721904
topologyVersion string
@@ -1889,16 +1921,7 @@ func TestComputeMachineDeploymentVersion(t *testing.T) {
18891921
},
18901922
},
18911923
currentMachineDeploymentState: &scope.MachineDeploymentState{Object: builder.MachineDeployment("test1", "md-current").WithVersion("v1.2.2").Build()},
1892-
machineDeploymentsStateMap: machineDeploymentsStateStable,
1893-
currentControlPlane: controlPlaneStable123,
1894-
desiredControlPlane: controlPlaneDesired,
1895-
topologyVersion: "v1.2.3",
1896-
expectedVersion: "v1.2.2",
1897-
},
1898-
{
1899-
name: "should return machine deployment's spec.template.spec.version if any one of the machine deployments is rolling out",
1900-
currentMachineDeploymentState: &scope.MachineDeploymentState{Object: builder.MachineDeployment("test1", "md-current").WithVersion("v1.2.2").Build()},
1901-
machineDeploymentsStateMap: machineDeploymentsStateRollingOut,
1924+
machineDeploymentsStateMap: twoMachineDeploymentsStateStable,
19021925
currentControlPlane: controlPlaneStable123,
19031926
desiredControlPlane: controlPlaneDesired,
19041927
topologyVersion: "v1.2.3",
@@ -1908,7 +1931,7 @@ func TestComputeMachineDeploymentVersion(t *testing.T) {
19081931
// Control plane is considered upgrading if the control plane's spec.version and status.version is not equal.
19091932
name: "should return machine deployment's spec.template.spec.version if control plane is upgrading",
19101933
currentMachineDeploymentState: &scope.MachineDeploymentState{Object: builder.MachineDeployment("test1", "md-current").WithVersion("v1.2.2").Build()},
1911-
machineDeploymentsStateMap: machineDeploymentsStateStable,
1934+
machineDeploymentsStateMap: twoMachineDeploymentsStateStable,
19121935
currentControlPlane: controlPlaneUpgrading,
19131936
topologyVersion: "v1.2.3",
19141937
expectedVersion: "v1.2.2",
@@ -1917,7 +1940,7 @@ func TestComputeMachineDeploymentVersion(t *testing.T) {
19171940
// Control plane is considered ready to upgrade if spec.version of current and desired control planes are not equal.
19181941
name: "should return machine deployment's spec.template.spec.version if control plane is ready to upgrade",
19191942
currentMachineDeploymentState: &scope.MachineDeploymentState{Object: builder.MachineDeployment("test1", "md-current").WithVersion("v1.2.2").Build()},
1920-
machineDeploymentsStateMap: machineDeploymentsStateStable,
1943+
machineDeploymentsStateMap: twoMachineDeploymentsStateStable,
19211944
currentControlPlane: controlPlaneStable122,
19221945
desiredControlPlane: controlPlaneDesired,
19231946
topologyVersion: "v1.2.3",
@@ -1927,20 +1950,40 @@ func TestComputeMachineDeploymentVersion(t *testing.T) {
19271950
// Control plane is considered scaling if its spec.replicas is not equal to any of status.replicas, status.readyReplicas or status.updatedReplicas.
19281951
name: "should return machine deployment's spec.template.spec.version if control plane is scaling",
19291952
currentMachineDeploymentState: &scope.MachineDeploymentState{Object: builder.MachineDeployment("test1", "md-current").WithVersion("v1.2.2").Build()},
1930-
machineDeploymentsStateMap: machineDeploymentsStateStable,
1953+
machineDeploymentsStateMap: twoMachineDeploymentsStateStable,
19311954
currentControlPlane: controlPlaneScaling,
19321955
topologyVersion: "v1.2.3",
19331956
expectedVersion: "v1.2.2",
19341957
},
19351958
{
19361959
name: "should return cluster.spec.topology.version if the control plane is not upgrading, not scaling, not ready to upgrade and none of the machine deployments are rolling out",
19371960
currentMachineDeploymentState: &scope.MachineDeploymentState{Object: builder.MachineDeployment("test1", "md-current").WithVersion("v1.2.2").Build()},
1938-
machineDeploymentsStateMap: machineDeploymentsStateStable,
1961+
machineDeploymentsStateMap: twoMachineDeploymentsStateStable,
19391962
currentControlPlane: controlPlaneStable123,
19401963
desiredControlPlane: controlPlaneDesired,
19411964
topologyVersion: "v1.2.3",
19421965
expectedVersion: "v1.2.3",
19431966
},
1967+
{
1968+
name: "should return cluster.spec.topology.version if control plane is stable, other machine deployments are rolling out, concurrency limit not reached",
1969+
currentMachineDeploymentState: &scope.MachineDeploymentState{Object: builder.MachineDeployment("test1", "md-current").WithVersion("v1.2.2").Build()},
1970+
machineDeploymentsStateMap: oneStableOneRollingMachineDeploymentState,
1971+
upgradeConcurrency: 2,
1972+
currentControlPlane: controlPlaneStable123,
1973+
desiredControlPlane: controlPlaneDesired,
1974+
topologyVersion: "v1.2.3",
1975+
expectedVersion: "v1.2.3",
1976+
},
1977+
{
1978+
name: "should return machine deployment's spec.template.spec.version if control plane is stable, other machine deployments are rolling out, concurrency limit reached",
1979+
currentMachineDeploymentState: &scope.MachineDeploymentState{Object: builder.MachineDeployment("test1", "md-current").WithVersion("v1.2.2").Build()},
1980+
machineDeploymentsStateMap: twoRollingMachineDeploymentState,
1981+
upgradeConcurrency: 2,
1982+
currentControlPlane: controlPlaneStable123,
1983+
desiredControlPlane: controlPlaneDesired,
1984+
topologyVersion: "v1.2.3",
1985+
expectedVersion: "v1.2.2",
1986+
},
19441987
}
19451988

19461989
for _, tt := range tests {
@@ -1959,9 +2002,10 @@ func TestComputeMachineDeploymentVersion(t *testing.T) {
19592002
ControlPlane: &scope.ControlPlaneState{Object: tt.currentControlPlane},
19602003
MachineDeployments: tt.machineDeploymentsStateMap,
19612004
},
1962-
UpgradeTracker: scope.NewUpgradeTracker(),
2005+
UpgradeTracker: scope.NewUpgradeTracker(scope.MaxMDUpgradeConcurrency(tt.upgradeConcurrency)),
19632006
}
19642007
desiredControlPlaneState := &scope.ControlPlaneState{Object: tt.desiredControlPlane}
2008+
s.UpgradeTracker.MachineDeployments.MarkRollingOut(s.Current.MachineDeployments.RollingOut()...)
19652009
version, err := computeMachineDeploymentVersion(s, tt.machineDeploymentTopology, desiredControlPlaneState, tt.currentMachineDeploymentState)
19662010
g.Expect(err).NotTo(HaveOccurred())
19672011
g.Expect(version).To(Equal(tt.expectedVersion))

0 commit comments

Comments
 (0)