Skip to content

Commit 9db4c78

Browse files
fabriziopandinisbueringer
fabriziopandini
and
sbueringer
authored
✨ Implement core logic for chained upgrades (#12726)
* Chained upgrades # Conflicts: # exp/topology/desiredstate/desired_state_test.go * Fix review findings * Address comments * Fix some minor findings * Temporarily drop race from test-infrastructure --------- Co-authored-by: Stefan Bueringer <[email protected]>
1 parent 987c157 commit 9db4c78

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+4345
-925
lines changed

‎Makefile‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -938,30 +938,31 @@ test-cover: ## Run unit and integration tests and generate a coverage report
938938
go tool cover -func=out/coverage.out -o out/coverage.txt
939939
go tool cover -html=out/coverage.out -o out/coverage.html
940940

941-
.PHONY: test-docker-infrastructure
942-
test-docker-infrastructure: $(SETUP_ENVTEST) ## Run unit and integration tests with race detector for docker infrastructure provider
941+
.PHONY: test-infrastructure
942+
test-infrastructure: $(SETUP_ENVTEST) ## Run unit and integration tests with race detector for docker infrastructure provider
943943
# Note: Fuzz tests are not executed with race detector because they would just time out.
944944
# To achieve that, all files with fuzz tests have the "!race" build tag, to still run fuzz tests
945945
# we have an additional `go test` run that focuses on "TestFuzzyConversion".
946-
cd $(CAPD_DIR); KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -race ./... $(TEST_ARGS)
947-
$(MAKE) test-docker-infrastructure-conversions TEST_ARGS="$(TEST_ARGS)"
946+
cd test/infrastructure; KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test ./... $(TEST_ARGS)
947+
$(MAKE) test-infrastructure-conversions TEST_ARGS="$(TEST_ARGS)"
948948

949-
.PHONY: test-docker-infrastructure-conversions
950-
test-docker-infrastructure-conversions: $(SETUP_ENVTEST) ## Run conversions test for docker infrastructure provider
951-
cd $(CAPD_DIR); KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -run "^TestFuzzyConversion$$" ./... $(TEST_ARGS)
949+
.PHONY: test-infrastructure-conversions
950+
test-infrastructure-conversions: $(SETUP_ENVTEST) ## Run conversions test for docker infrastructure provider
951+
cd test/infrastructure; KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -run "^TestFuzzyConversion$$" ./... $(TEST_ARGS)
952952

953-
.PHONY: test-docker-infrastructure-verbose
954-
test-docker-infrastructure-verbose: ## Run unit and integration tests with race detector and with verbose flag for docker infrastructure provider
955-
$(MAKE) test-docker-infrastructure TEST_ARGS="$(TEST_ARGS) -v"
953+
.PHONY: test-infrastructure-verbose
954+
test-infrastructure-verbose: ## Run unit and integration tests with race detector and with verbose flag for docker infrastructure provider
955+
$(MAKE) test-infrastructure TEST_ARGS="$(TEST_ARGS) -v"
956956

957-
.PHONY: test-docker-infrastructure-junit
958-
test-docker-infrastructure-junit: $(SETUP_ENVTEST) $(GOTESTSUM) ## Run unit and integration tests with race detector and generate a junit report for docker infrastructure provider
959-
cd $(CAPD_DIR); set +o errexit; (KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -race -json ./... $(TEST_ARGS); echo $$? > $(ARTIFACTS)/junit.infra_docker.exitcode) | tee $(ARTIFACTS)/junit.infra_docker.stdout
957+
.PHONY: test-infrastructure-junit
958+
test-infrastructure-junit: $(SETUP_ENVTEST) $(GOTESTSUM) ## Run unit and integration tests with race detector and generate a junit report for docker infrastructure provider
959+
cd test/infrastructure; set +o errexit; (KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -json ./... $(TEST_ARGS); echo $$? > $(ARTIFACTS)/junit.infra_docker.exitcode) | tee $(ARTIFACTS)/junit.infra_docker.stdout
960960
$(GOTESTSUM) --junitfile $(ARTIFACTS)/junit.infra_docker.xml --raw-command cat $(ARTIFACTS)/junit.infra_docker.stdout
961961
exit $$(cat $(ARTIFACTS)/junit.infra_docker.exitcode)
962-
cd $(CAPD_DIR); set +o errexit; (KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -run "^TestFuzzyConversion$$" -json ./... $(TEST_ARGS); echo $$? > $(ARTIFACTS)/junit-fuzz.infra_docker.exitcode) | tee $(ARTIFACTS)/junit-fuzz.infra_docker.stdout
962+
cd test/infrastructure; set +o errexit; (KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -run "^TestFuzzyConversion$$" -json ./... $(TEST_ARGS); echo $$? > $(ARTIFACTS)/junit-fuzz.infra_docker.exitcode) | tee $(ARTIFACTS)/junit-fuzz.infra_docker.stdout
963963
$(GOTESTSUM) --junitfile $(ARTIFACTS)/junit-fuzz.infra_docker.xml --raw-command cat $(ARTIFACTS)/junit-fuzz.infra_docker.stdout
964964
exit $$(cat $(ARTIFACTS)/junit-fuzz.infra_docker.exitcode)
965+
965966
.PHONY: test-test-extension
966967
test-test-extension: $(SETUP_ENVTEST) ## Run unit and integration tests for the test extension
967968
cd $(TEST_EXTENSION_DIR); KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -race ./... $(TEST_ARGS)

‎api/core/v1beta1/conversion.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,8 @@ func (src *ClusterClass) ConvertTo(dstRaw conversion.Hub) error {
248248
dst.Status.Variables[i] = variable
249249
}
250250

251+
dst.Spec.KubernetesVersions = restored.Spec.KubernetesVersions
252+
251253
return nil
252254
}
253255

‎api/core/v1beta1/zz_generated.conversion.go‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/core/v1beta2/clusterclass_types.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,18 @@ type ClusterClassSpec struct {
135135
// +kubebuilder:validation:MinItems=1
136136
// +kubebuilder:validation:MaxItems=1000
137137
Patches []ClusterClassPatch `json:"patches,omitempty"`
138+
139+
// kubernetesVersions is the list of Kubernetes versions that can be
140+
// used for clusters using this ClusterClass.
141+
// The list of version must be ordered from the older to the newer version, and there should be
142+
// at least one version for every minor in between the first and the last version.
143+
// +optional
144+
// +listType=atomic
145+
// +kubebuilder:validation:MinItems=1
146+
// +kubebuilder:validation:MaxItems=100
147+
// +kubebuilder:validation:items:MinLength=1
148+
// +kubebuilder:validation:items:MaxLength=256
149+
KubernetesVersions []string `json:"kubernetesVersions,omitempty"`
138150
}
139151

140152
// InfrastructureClass defines the class for the infrastructure cluster.

‎api/core/v1beta2/common_types.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@ const (
3636
// to track the name of the MachineDeployment topology it represents.
3737
ClusterTopologyMachineDeploymentNameLabel = "topology.cluster.x-k8s.io/deployment-name"
3838

39+
// ClusterTopologyUpgradeStepAnnotation tracks the version of the current upgrade step.
40+
// It is only set when an upgrade is in progress, and it contains the control plane version computed by topology controller.
41+
ClusterTopologyUpgradeStepAnnotation = "topology.internal.cluster.x-k8s.io/upgrade-step"
42+
3943
// ClusterTopologyHoldUpgradeSequenceAnnotation can be used to hold the entire MachineDeployment upgrade sequence.
4044
// If the annotation is set on a MachineDeployment topology in Cluster.spec.topology.workers, the Kubernetes upgrade
4145
// for this MachineDeployment topology and all subsequent ones is deferred.

‎api/core/v1beta2/zz_generated.deepcopy.go‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/core/v1beta2/zz_generated.openapi.go‎

Lines changed: 20 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crd/bases/cluster.x-k8s.io_clusterclasses.yaml‎

Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎controlplane/kubeadm/internal/controllers/scale.go‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -185,8 +185,18 @@ func (r *KubeadmControlPlaneReconciler) preflightChecks(ctx context.Context, con
185185

186186
if feature.Gates.Enabled(feature.ClusterTopology) {
187187
// Block when we expect an upgrade to be propagated for topology clusters.
188-
if controlPlane.Cluster.Spec.Topology.IsDefined() && controlPlane.Cluster.Spec.Topology.Version != controlPlane.KCP.Spec.Version {
189-
logger.Info(fmt.Sprintf("Waiting for a version upgrade to %s to be propagated from Cluster.spec.topology", controlPlane.Cluster.Spec.Topology.Version))
188+
// NOTE: in case the cluster is performing an upgrade, allow creation of machines for the intermediate step.
189+
hasSameVersionOfCurrentUpgradeStep := false
190+
if version, ok := controlPlane.Cluster.GetAnnotations()[clusterv1.ClusterTopologyUpgradeStepAnnotation]; ok {
191+
hasSameVersionOfCurrentUpgradeStep = version == controlPlane.KCP.Spec.Version
192+
}
193+
194+
if controlPlane.Cluster.Spec.Topology.IsDefined() && controlPlane.Cluster.Spec.Topology.Version != controlPlane.KCP.Spec.Version && !hasSameVersionOfCurrentUpgradeStep {
195+
v := controlPlane.Cluster.Spec.Topology.Version
196+
if version, ok := controlPlane.Cluster.GetAnnotations()[clusterv1.ClusterTopologyUpgradeStepAnnotation]; ok {
197+
v = version
198+
}
199+
logger.Info(fmt.Sprintf("Waiting for a version upgrade to %s to be propagated", v))
190200
controlPlane.PreflightCheckResults.TopologyVersionMismatch = true
191201
return ctrl.Result{RequeueAfter: preflightFailedRequeueAfter}, nil
192202
}

‎controlplane/kubeadm/internal/controllers/scale_test.go‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -562,6 +562,37 @@ func TestPreflightChecks(t *testing.T) {
562562
TopologyVersionMismatch: true,
563563
},
564564
},
565+
{
566+
name: "control plane with a pending upgrade, but not yet at the current step of the upgrade plan, should requeue",
567+
cluster: &clusterv1.Cluster{
568+
ObjectMeta: metav1.ObjectMeta{
569+
Annotations: map[string]string{
570+
clusterv1.ClusterTopologyUpgradeStepAnnotation: "v1.32.0",
571+
},
572+
},
573+
Spec: clusterv1.ClusterSpec{
574+
Topology: clusterv1.Topology{
575+
Version: "v1.33.0",
576+
},
577+
},
578+
},
579+
kcp: &controlplanev1.KubeadmControlPlane{
580+
Spec: controlplanev1.KubeadmControlPlaneSpec{
581+
Version: "v1.31.0",
582+
},
583+
},
584+
machines: []*clusterv1.Machine{
585+
{},
586+
},
587+
588+
expectResult: ctrl.Result{RequeueAfter: preflightFailedRequeueAfter},
589+
expectPreflight: internal.PreflightCheckResults{
590+
HasDeletingMachine: false,
591+
ControlPlaneComponentsNotHealthy: false,
592+
EtcdClusterNotHealthy: false,
593+
TopologyVersionMismatch: true,
594+
},
595+
},
565596
{
566597
name: "control plane with a deleting machine should requeue",
567598
kcp: &controlplanev1.KubeadmControlPlane{},
@@ -687,6 +718,55 @@ func TestPreflightChecks(t *testing.T) {
687718
TopologyVersionMismatch: false,
688719
},
689720
},
721+
{
722+
name: "control plane with a pending upgrade, but already at the current step of the upgrade plan, should pass",
723+
cluster: &clusterv1.Cluster{
724+
ObjectMeta: metav1.ObjectMeta{
725+
Annotations: map[string]string{
726+
clusterv1.ClusterTopologyUpgradeStepAnnotation: "v1.32.0",
727+
},
728+
},
729+
Spec: clusterv1.ClusterSpec{
730+
Topology: clusterv1.Topology{
731+
Version: "v1.33.0",
732+
},
733+
},
734+
},
735+
kcp: &controlplanev1.KubeadmControlPlane{
736+
Spec: controlplanev1.KubeadmControlPlaneSpec{
737+
Version: "v1.32.0",
738+
}, Status: controlplanev1.KubeadmControlPlaneStatus{
739+
Conditions: []metav1.Condition{
740+
{Type: controlplanev1.KubeadmControlPlaneControlPlaneComponentsHealthyCondition, Status: metav1.ConditionTrue},
741+
{Type: controlplanev1.KubeadmControlPlaneEtcdClusterHealthyCondition, Status: metav1.ConditionTrue},
742+
},
743+
},
744+
},
745+
machines: []*clusterv1.Machine{
746+
{
747+
Status: clusterv1.MachineStatus{
748+
NodeRef: clusterv1.MachineNodeReference{
749+
Name: "node-1",
750+
},
751+
Conditions: []metav1.Condition{
752+
{Type: controlplanev1.KubeadmControlPlaneMachineAPIServerPodHealthyCondition, Status: metav1.ConditionTrue},
753+
{Type: controlplanev1.KubeadmControlPlaneMachineControllerManagerPodHealthyCondition, Status: metav1.ConditionTrue},
754+
{Type: controlplanev1.KubeadmControlPlaneMachineSchedulerPodHealthyCondition, Status: metav1.ConditionTrue},
755+
{Type: controlplanev1.KubeadmControlPlaneMachineEtcdPodHealthyCondition, Status: metav1.ConditionTrue},
756+
{Type: controlplanev1.KubeadmControlPlaneMachineEtcdMemberHealthyCondition, Status: metav1.ConditionTrue},
757+
},
758+
},
759+
},
760+
},
761+
762+
expectResult: ctrl.Result{},
763+
expectPreflight: internal.PreflightCheckResults{
764+
HasDeletingMachine: false,
765+
ControlPlaneComponentsNotHealthy: false,
766+
EtcdClusterNotHealthy: false,
767+
TopologyVersionMismatch: false,
768+
},
769+
},
690770
}
691771

692772
for _, tt := range testCases {

0 commit comments

Comments
 (0)