Skip to content

Commit ee24244

Browse files
imdegaraeapedriza
imdegarae
and
apedriza
authored
Report controlplane status for InPlace upgrades (#923)
Signed-off-by: Adrian Pedriza <[email protected]> Co-authored-by: Adrian Pedriza <[email protected]>
1 parent cff2277 commit ee24244

File tree

13 files changed

+882
-138
lines changed

13 files changed

+882
-138
lines changed

‎.github/workflows/go.yml‎

Lines changed: 32 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -171,31 +171,38 @@ jobs:
171171
uses: ./.github/workflows/capi-smoke-tests.yml
172172
with:
173173
smoke-suite: ${{ matrix.smoke-suite }}
174-
e2e-migration:
175-
name: E2E
176-
needs: build
177174

178-
runs-on: ubuntu-latest
175+
e2e:
176+
name: E2E test
177+
needs: build
178+
runs-on: ubuntu-22.04-8core
179+
strategy:
180+
fail-fast: false
181+
matrix:
182+
e2e-suite:
183+
- workload-cluster-inplace-upgrade
184+
- workload-cluster-recreate-upgrade
179185

180186
steps:
181-
- name: Check out code into the Go module directory
182-
uses: actions/checkout@v4
183-
184-
- name: Set up Go
185-
uses: actions/setup-go@v5
186-
with:
187-
go-version-file: go.mod
188-
189-
- name: Build e2e images
190-
run: make docker-build
191-
192-
- name: Run e2e tests
193-
run: make e2e
194-
195-
- name: Archive artifacts
196-
if: failure()
197-
uses: actions/[email protected]
198-
with:
199-
name: e2e-artifacts
200-
path: _artifacts
201-
if-no-files-found: ignore
187+
- name: Check out code into the Go module directory
188+
uses: actions/checkout@v4
189+
190+
- name: Set up Go
191+
uses: actions/setup-go@v5
192+
with:
193+
go-version-file: go.mod
194+
195+
- name: Run e2e test
196+
run: |
197+
export TEST_NAME=Test$(echo "${{ matrix.e2e-suite }}" | awk -F'-' '{for(i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) substr($i,2)}1' OFS='')
198+
echo "Running E2E tests with TEST_NAME=$TEST_NAME"
199+
make release
200+
make e2e TEST_NAME="$TEST_NAME"
201+
202+
- name: Archive artifacts
203+
if: failure()
204+
uses: actions/[email protected]
205+
with:
206+
name: e2e-artifacts
207+
path: _artifacts
208+
if-no-files-found: ignore

‎Makefile‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -106,9 +106,8 @@ vet: ## Run go vet against code.
106106
test: manifests generate fmt vet envtest ## Run tests.
107107
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $(GO_TEST_DIRS) -coverprofile cover.out
108108

109-
.PHONY: e2e
110-
e2e: ## Run the end-to-end tests
111-
go test -v ./e2e -tags e2e \
109+
e2e: k0smotron-image-bundle.tar install.yaml
110+
go test -v -tags e2e -run '$(TEST_NAME)' ./e2e \
112111
-artifacts-folder="$(ARTIFACTS)" \
113112
-config="$(E2E_CONF_FILE)" \
114113
-skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \

‎Makefile.variables‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,7 @@ CONTROLLER_TOOLS_VERSION ?= v0.14.0
33
GO_VERSION ?= 1.22.6
44
GOLANGCILINT_VERSION ?= 1.52.2
55
CRDOC_VERSION ?= v0.6.2
6+
7+
e2esuite := \
8+
workload-cluster-inplace-upgrade \
9+
workload-cluster-recreate-upgrade \

‎e2e/common.go‎

Lines changed: 0 additions & 29 deletions
This file was deleted.

‎e2e/data/infrastructure-docker/cluster-template-out-of-cluster.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ metadata:
4141
spec:
4242
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
4343
version: v1.30.1+k0s.0
44-
updateStrategy: Recreate
44+
updateStrategy: ${UPDATE_STRATEGY}
4545
k0sConfigSpec:
4646
args:
4747
- --enable-worker

‎e2e/suite_test.go‎ renamed to ‎e2e/setup.go‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,16 @@ import (
4343
ctrl "sigs.k8s.io/controller-runtime"
4444
)
4545

46+
// Test suite constants for e2e config variables.
47+
const (
48+
KubernetesVersion = "KUBERNETES_VERSION"
49+
KubernetesVersionManagement = "KUBERNETES_VERSION_MANAGEMENT"
50+
KubernetesVersionFirstUpgradeTo = "KUBERNETES_VERSION_FIRST_UPGRADE_TO"
51+
KubernetesVersionSecondUpgradeTo = "KUBERNETES_VERSION_SECOND_UPGRADE_TO"
52+
ControlPlaneMachineCount = "CONTROL_PLANE_MACHINE_COUNT"
53+
IPFamily = "IP_FAMILY"
54+
)
55+
4656
var (
4757
ctx = ctrl.SetupSignalHandler()
4858

@@ -86,7 +96,7 @@ func init() {
8696
flag.BoolVar(&useExistingCluster, "use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)")
8797
}
8898

89-
func TestMain(m *testing.M) {
99+
func setup(t *testing.T, test func(t *testing.T)) {
90100
ctrl.SetLogger(klog.Background())
91101
flag.Parse()
92102

@@ -102,13 +112,13 @@ func TestMain(m *testing.M) {
102112
panic(err)
103113
}
104114

105-
code := m.Run()
106-
107-
if !skipCleanup {
108-
tearDown(managementClusterProvider, managementClusterProxy)
109-
}
115+
defer func() {
116+
if !skipCleanup {
117+
tearDown(managementClusterProvider, managementClusterProxy)
118+
}
119+
}()
110120

111-
os.Exit(code)
121+
test(t)
112122
}
113123

114124
func setupMothership() error {

‎e2e/util/controlplane.go‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,16 @@ import (
3737
crclient "sigs.k8s.io/controller-runtime/pkg/client"
3838
)
3939

40-
func WaitForControlPlaneToBeReady(ctx context.Context, client crclient.Client, cpObjectKey crclient.ObjectKey, interval Interval) error {
40+
func WaitForControlPlaneToBeReady(ctx context.Context, client crclient.Client, cp *cpv1beta1.K0sControlPlane, interval Interval) error {
4141
fmt.Println("Waiting for the control plane to be ready")
42+
43+
controlplaneObjectKey := crclient.ObjectKey{
44+
Name: cp.Name,
45+
Namespace: cp.Namespace,
46+
}
4247
controlplane := &cpv1beta1.K0sControlPlane{}
4348
err := wait.PollUntilContextTimeout(ctx, interval.tick, interval.timeout, true, func(ctx context.Context) (done bool, err error) {
44-
if err := client.Get(ctx, cpObjectKey, controlplane); err != nil {
49+
if err := client.Get(ctx, controlplaneObjectKey, controlplane); err != nil {
4550
return false, errors.Wrapf(err, "failed to get controlplane")
4651
}
4752

@@ -98,11 +103,7 @@ func UpgradeControlPlaneAndWaitForReadyUpgrade(ctx context.Context, input Upgrad
98103
return fmt.Errorf("failed to patch the new kubernetes version to controlplane %s: %w", klog.KObj(input.ControlPlane), err)
99104
}
100105

101-
controlplaneObjectKey := crclient.ObjectKey{
102-
Name: input.ControlPlane.Name,
103-
Namespace: input.ControlPlane.Namespace,
104-
}
105-
err = WaitForControlPlaneToBeReady(ctx, input.ClusterProxy.GetClient(), controlplaneObjectKey, input.WaitForControlPlaneReadyInterval)
106+
err = WaitForControlPlaneToBeReady(ctx, input.ClusterProxy.GetClient(), input.ControlPlane, input.WaitForControlPlaneReadyInterval)
106107
if err != nil {
107108
return err
108109
}

‎e2e/workload_cluster_inplace_upgrade_test.go‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
//go:build e2e
2+
3+
/*
4+
Copyright 2025.
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
*/
18+
19+
package e2e
20+
21+
import (
22+
"fmt"
23+
"path/filepath"
24+
"testing"
25+
"time"
26+
27+
"github.com/k0smotron/k0smotron/e2e/util"
28+
"github.com/stretchr/testify/require"
29+
"k8s.io/utils/ptr"
30+
capiframework "sigs.k8s.io/cluster-api/test/framework"
31+
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
32+
capiutil "sigs.k8s.io/cluster-api/util"
33+
)
34+
35+
func TestWorkloadClusterInplaceUpgrade(t *testing.T) {
36+
setup(t, workloadClusterInplaceUpgradeSpec)
37+
}
38+
39+
// Validation of the correct operation of k0smotron when the
40+
// K0sControlPlane object is updated. It simulates a typical user workflow that includes:
41+
//
42+
// 1. Creation of a workload cluster.
43+
// - Ensures the cluster becomes operational.
44+
//
45+
// 2. Updating the control plane version using Inplace upgrade strategy.
46+
// - Verifies the cluster status aligns with the expected state after the update.
47+
//
48+
// 3. Performing a subsequent control plane version upgrade using Inplace upgrade strategy.
49+
// - Confirms the cluster status is consistent and desired post-update.
50+
func workloadClusterInplaceUpgradeSpec(t *testing.T) {
51+
testName := "workload-inplace-upgrade"
52+
53+
// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
54+
namespace, _ := util.SetupSpecNamespace(ctx, testName, managementClusterProxy, artifactFolder)
55+
56+
clusterName := fmt.Sprintf("%s-%s", testName, capiutil.RandomString(6))
57+
58+
workloadClusterTemplate := clusterctl.ConfigCluster(ctx, clusterctl.ConfigClusterInput{
59+
ClusterctlConfigPath: clusterctlConfigPath,
60+
KubeconfigPath: managementClusterProxy.GetKubeconfigPath(),
61+
// select cluster templates
62+
Flavor: "out-of-cluster",
63+
64+
Namespace: namespace.Name,
65+
ClusterName: clusterName,
66+
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion),
67+
ControlPlaneMachineCount: ptr.To[int64](3),
68+
// TODO: make infra provider configurable
69+
InfrastructureProvider: "docker",
70+
LogFolder: filepath.Join(artifactFolder, "clusters", managementClusterProxy.GetName()),
71+
ClusterctlVariables: map[string]string{
72+
"CLUSTER_NAME": clusterName,
73+
"NAMESPACE": namespace.Name,
74+
"UPDATE_STRATEGY": "InPlace",
75+
},
76+
})
77+
require.NotNil(t, workloadClusterTemplate)
78+
79+
require.Eventually(t, func() bool {
80+
return managementClusterProxy.CreateOrUpdate(ctx, workloadClusterTemplate) == nil
81+
}, 10*time.Second, 1*time.Second, "Failed to apply the cluster template")
82+
83+
cluster, err := util.DiscoveryAndWaitForCluster(ctx, capiframework.DiscoveryAndWaitForClusterInput{
84+
Getter: managementClusterProxy.GetClient(),
85+
Namespace: namespace.Name,
86+
Name: clusterName,
87+
}, util.GetInterval(e2eConfig, testName, "wait-cluster"))
88+
require.NoError(t, err)
89+
90+
defer func() {
91+
util.DumpSpecResourcesAndCleanup(
92+
ctx,
93+
testName,
94+
managementClusterProxy,
95+
artifactFolder,
96+
namespace,
97+
cancelWatches,
98+
cluster,
99+
util.GetInterval(e2eConfig, testName, "wait-delete-cluster"),
100+
skipCleanup,
101+
)
102+
}()
103+
104+
controlPlane, err := util.DiscoveryAndWaitForControlPlaneInitialized(ctx, capiframework.DiscoveryAndWaitForControlPlaneInitializedInput{
105+
Lister: managementClusterProxy.GetClient(),
106+
Cluster: cluster,
107+
}, util.GetInterval(e2eConfig, testName, "wait-controllers"))
108+
require.NoError(t, err)
109+
// For Inplace upgrades we need to wait for the controlplane to have all the replicas ready before upgrading it again.
110+
err = util.WaitForControlPlaneToBeReady(ctx, managementClusterProxy.GetClient(), controlPlane, util.GetInterval(e2eConfig, testName, "wait-kube-proxy-upgrade"))
111+
require.NoError(t, err)
112+
113+
fmt.Println("Upgrading the Kubernetes control-plane version")
114+
err = util.UpgradeControlPlaneAndWaitForReadyUpgrade(ctx, util.UpgradeControlPlaneAndWaitForUpgradeInput{
115+
ClusterProxy: managementClusterProxy,
116+
Cluster: cluster,
117+
ControlPlane: controlPlane,
118+
KubernetesUpgradeVersion: e2eConfig.GetVariable(KubernetesVersionFirstUpgradeTo),
119+
WaitForKubeProxyUpgradeInterval: util.GetInterval(e2eConfig, testName, "wait-kube-proxy-upgrade"),
120+
WaitForControlPlaneReadyInterval: util.GetInterval(e2eConfig, testName, "wait-control-plane"),
121+
})
122+
require.NoError(t, err)
123+
124+
fmt.Println("Upgrading the Kubernetes control-plane version again")
125+
err = util.UpgradeControlPlaneAndWaitForReadyUpgrade(ctx, util.UpgradeControlPlaneAndWaitForUpgradeInput{
126+
ClusterProxy: managementClusterProxy,
127+
Cluster: cluster,
128+
ControlPlane: controlPlane,
129+
KubernetesUpgradeVersion: e2eConfig.GetVariable(KubernetesVersionSecondUpgradeTo),
130+
WaitForKubeProxyUpgradeInterval: util.GetInterval(e2eConfig, testName, "wait-kube-proxy-upgrade"),
131+
WaitForControlPlaneReadyInterval: util.GetInterval(e2eConfig, testName, "wait-control-plane"),
132+
})
133+
require.NoError(t, err)
134+
}

‎e2e/workload_cluster_upgrade_test.go‎ renamed to ‎e2e/workload_cluster_recreate_upgrade_test.go‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,20 +32,23 @@ import (
3232
capiutil "sigs.k8s.io/cluster-api/util"
3333
)
3434

35+
func TestWorkloadClusterRecreateUpgrade(t *testing.T) {
36+
setup(t, workloadClusterRecreateUpgradeSpec)
37+
}
38+
3539
// Validation of the correct operation of k0smotron when the
3640
// K0sControlPlane object is updated. It simulates a typical user workflow that includes:
3741
//
3842
// 1. Creation of a workload cluster.
3943
// - Ensures the cluster becomes operational.
4044
//
41-
// 2. Updating the control plane version.
45+
// 2. Updating the control plane version using Recreate upgrade strategy.
4246
// - Verifies the cluster status aligns with the expected state after the update.
4347
//
44-
// 3. Performing a subsequent control plane version update.
48+
// 3. Performing a subsequent control plane version upgrade using Inplace upgrade strategy.
4549
// - Confirms the cluster status is consistent and desired post-update.
46-
func TestWorkloadClusterUpgrade(t *testing.T) {
47-
48-
testName := "workload-upgrade"
50+
func workloadClusterRecreateUpgradeSpec(t *testing.T) {
51+
testName := "workload-recreate-upgrade"
4952

5053
// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
5154
namespace, _ := util.SetupSpecNamespace(ctx, testName, managementClusterProxy, artifactFolder)
@@ -66,8 +69,9 @@ func TestWorkloadClusterUpgrade(t *testing.T) {
6669
InfrastructureProvider: "docker",
6770
LogFolder: filepath.Join(artifactFolder, "clusters", managementClusterProxy.GetName()),
6871
ClusterctlVariables: map[string]string{
69-
"CLUSTER_NAME": clusterName,
70-
"NAMESPACE": namespace.Name,
72+
"CLUSTER_NAME": clusterName,
73+
"NAMESPACE": namespace.Name,
74+
"UPDATE_STRATEGY": "Recreate",
7175
},
7276
})
7377
require.NotNil(t, workloadClusterTemplate)

‎internal/controller/controlplane/k0s_controlplane_controller.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,8 +143,14 @@ func (c *K0sController) Reconcile(ctx context.Context, req ctrl.Request) (res ct
143143
// Separate var for status update errors to avoid shadowing err
144144
derr := c.updateStatus(ctx, kcp, cluster)
145145
if derr != nil {
146-
log.Error(derr, "Failed to update status")
147-
return
146+
if !errors.Is(derr, errUpgradeNotCompleted) {
147+
log.Error(derr, "Failed to update status")
148+
return
149+
}
150+
151+
if res.IsZero() {
152+
res = ctrl.Result{RequeueAfter: 10 * time.Second}
153+
}
148154
}
149155

150156
if errors.Is(err, ErrNotReady) || reflect.DeepEqual(existingStatus, kcp.Status) {

0 commit comments

Comments
 (0)