Skip to content

Commit 8c11f10

Browse files
simcodsimcod
committed
Use flavor for upgrade
1 parent 40bfd1b commit 8c11f10

File tree

3 files changed

+29
-194
lines changed

3 files changed

+29
-194
lines changed

‎test/e2e/frmwrk/cluster_upgrade_kubernetes_test.go‎

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,37 +3,53 @@ package frmwrk
33
import (
44
"context"
55
"fmt"
6+
. "github.com/onsi/ginkgo/v2" //nolint:staticcheck
67
"k8s.io/utils/ptr"
78
"os"
8-
"strings"
9-
10-
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
119

12-
. "github.com/onsi/ginkgo/v2" //nolint:staticcheck
10+
capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
1311
)
1412

1513
var _ = Describe("Upgrade Kubernetes Cluster Version", Label("upgrade"), func() {
1614
ctx := context.TODO()
17-
fromKubernetesVersion := strings.Split(os.Getenv("KUBERNETES_VERSION_UPGRADE_FROM"), ",")
18-
toKubernetesVersion := strings.Split(os.Getenv("KUBERNETES_VERSION_UPGRADE_TO"), ",")
15+
fromKubernetesVersion := os.Getenv("KUBERNETES_VERSION_UPGRADE_FROM")
16+
toKubernetesVersion := os.Getenv("KUBERNETES_VERSION_UPGRADE_TO")
17+
18+
ec := e2eCtx.NewE2ECluster(ClusterConfig{
19+
SpecName: "cluster-upgrade",
20+
NamespaceName: "cluster-upgrade",
21+
ClusterName: "cluster-upgrade",
22+
KubernetesVersion: fromKubernetesVersion, //TBD
23+
ControlPlaneMachineImage: os.Getenv("E2E_CONTROL_PLANE_MACHINE_IMAGE_PREFIX") + fromKubernetesVersion,
24+
ControlPlaneMachineCount: 1,
25+
WorkerMachineImage: os.Getenv("E2E_WORKER_MACHINE_IMAGE_PREFIX") + fromKubernetesVersion,
26+
WorkerMachineCount: 1,
27+
})
28+
29+
ec.SetupMetalStackPreconditions(ctx)
30+
ec.SetupNamespace(ctx)
31+
32+
DeferCleanup(func() {
33+
ec.Dump(context.Background())
34+
})
1935

20-
It(fmt.Sprintf("from %d to %d", fromKubernetesVersion, toKubernetesVersion), func() {
36+
It(fmt.Sprintf("from %s to %s", fromKubernetesVersion, toKubernetesVersion), func() {
2137
cfg := e2eCtx.E2EConfig.DeepCopy()
2238
cfg.Variables["KUBERNETES_VERSION_UPGRADE_FROM"] = e2eCtx.envOrVar("KUBERNETES_VERSION_UPGRADE_FROM")
2339
cfg.Variables["KUBERNETES_VERSION_UPGRADE_TO"] = e2eCtx.envOrVar("KUBERNETES_VERSION_UPGRADE_TO")
2440
cfg.Variables["KUBETEST_CONFIGURATION"] = e2eCtx.envOrVar("KUBETEST_CONFIGURATION")
2541

26-
clusterUpgrade(ctx, e2eCtx, func() ClusterUpgradeSpecInput {
27-
return ClusterUpgradeSpecInput{
42+
capi_e2e.ClusterUpgradeConformanceSpec(ctx, func() capi_e2e.ClusterUpgradeConformanceSpecInput {
43+
return capi_e2e.ClusterUpgradeConformanceSpecInput{
2844
E2EConfig: cfg,
2945
ClusterctlConfigPath: e2eCtx.Environment.ClusterctlConfigPath,
3046
BootstrapClusterProxy: e2eCtx.Environment.Bootstrap,
3147
ArtifactFolder: e2eCtx.Environment.artifactsPath,
32-
SkipCleanup: true,
48+
SkipCleanup: false,
3349
SkipConformanceTests: true,
34-
ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{},
3550
ControlPlaneMachineCount: ptr.To[int64](1),
3651
WorkerMachineCount: ptr.To[int64](1),
52+
Flavor: ptr.To(e2eCtx.Environment.Flavor),
3753
}
3854
})
3955
})

‎test/e2e/frmwrk/config/capi-e2e-config.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -144,8 +144,8 @@ variables:
144144
CONTROL_PLANE_MACHINE_COUNT: "1"
145145
WORKER_MACHINE_COUNT: "1"
146146
# K8s Upgrade variables
147-
KUBERNETES_VERSION_UPGRADE_FROM: "v1.31.13"
148-
KUBERNETES_VERSION_UPGRADE_TO: "v1.32.9"
147+
KUBERNETES_VERSION_UPGRADE_FROM: "v1.33.5"
148+
KUBERNETES_VERSION_UPGRADE_TO: "v1.34.1"
149149

150150
intervals:
151151
default/wait-controllers: ["4m", "10s"]

‎test/e2e/frmwrk/shared_cases.go‎

Lines changed: 0 additions & 181 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,11 @@ package frmwrk
22

33
import (
44
"context"
5-
"fmt"
6-
corev1 "k8s.io/api/core/v1"
7-
"k8s.io/utils/ptr"
8-
"os"
9-
"path"
10-
"path/filepath"
11-
125
. "github.com/onsi/ginkgo/v2" //nolint:staticcheck
136
. "github.com/onsi/gomega" //nolint:staticcheck
147

158
"sigs.k8s.io/cluster-api/test/framework"
169
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
17-
"sigs.k8s.io/cluster-api/util"
18-
19-
"github.com/drone/envsubst/v2"
2010
)
2111

2212
type ClusterUpgradeSpecInput struct {
@@ -82,174 +72,3 @@ func createE2ECluster(ctx context.Context, e2eCtx *E2EContext, cfg ClusterConfig
8272

8373
return ec
8474
}
85-
86-
// Reference: https://github.com/kubernetes-sigs/cluster-api/blob/76697d5bf2a746aa1bf942c88170692666bfcc5c/test/e2e/cluster_upgrade.go#L37
87-
func clusterUpgrade(ctx context.Context, e2eCtx *E2EContext, inputGetter func() ClusterUpgradeSpecInput) {
88-
89-
var (
90-
specName = "cluster-upgrade"
91-
input ClusterUpgradeSpecInput
92-
namespace *corev1.Namespace
93-
//cancelWatches context.CancelFunc
94-
95-
KubernetesVersionUpgradeFrom = "KUBERNETES_VERSION_UPGRADE_FROM"
96-
KubernetesVersionUpgradeTo = "KUBERNETES_VERSION_UPGRADE_TO"
97-
CPMachineTemplateUpgradeTo = "CONTROL_PLANE_MACHINE_TEMPLATE_UPGRADE_TO"
98-
WorkersMachineTemplateUpgradeTo = "WORKERS_MACHINE_TEMPLATE_UPGRADE_TO"
99-
100-
controlPlaneMachineCount int64
101-
workerMachineCount int64
102-
103-
clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult
104-
105-
clusterName string
106-
)
107-
108-
ec := e2eCtx.NewE2ECluster(ClusterConfig{
109-
SpecName: specName,
110-
NamespaceName: "cluster-upgrade",
111-
ClusterName: "e2e-upgrade",
112-
})
113-
ec.SetupMetalStackPreconditions(ctx)
114-
//ec.SetupNamespace(ctx)
115-
116-
BeforeEach(func() {
117-
Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
118-
input = inputGetter()
119-
Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
120-
Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName)
121-
Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)
122-
Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)
123-
124-
Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersionUpgradeFrom))
125-
Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersionUpgradeTo))
126-
127-
if input.ControlPlaneMachineCount == nil {
128-
controlPlaneMachineCount = 1
129-
} else {
130-
controlPlaneMachineCount = *input.ControlPlaneMachineCount
131-
}
132-
133-
if input.WorkerMachineCount == nil {
134-
workerMachineCount = 2
135-
} else {
136-
workerMachineCount = *input.WorkerMachineCount
137-
}
138-
139-
clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)
140-
})
141-
142-
It("Should create and upgrade a workload cluster", func() {
143-
By("Creating a workload cluster")
144-
145-
infrastructureProvider := clusterctl.DefaultInfrastructureProvider
146-
if input.InfrastructureProvider != nil {
147-
infrastructureProvider = *input.InfrastructureProvider
148-
}
149-
150-
clusterName = fmt.Sprintf("%s-%s", specName, util.RandomString(6))
151-
152-
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
153-
ClusterProxy: input.BootstrapClusterProxy,
154-
ConfigCluster: clusterctl.ConfigClusterInput{
155-
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
156-
ClusterctlConfigPath: input.ClusterctlConfigPath,
157-
KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(),
158-
InfrastructureProvider: infrastructureProvider,
159-
Namespace: namespace.Name,
160-
ClusterName: clusterName,
161-
KubernetesVersion: input.E2EConfig.MustGetVariable(KubernetesVersionUpgradeFrom),
162-
ControlPlaneMachineCount: ptr.To[int64](controlPlaneMachineCount),
163-
WorkerMachineCount: ptr.To[int64](workerMachineCount),
164-
},
165-
ControlPlaneWaiters: input.ControlPlaneWaiters,
166-
WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"),
167-
WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
168-
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
169-
WaitForMachinePools: input.E2EConfig.GetIntervals(specName, "wait-machine-pool-nodes"),
170-
}, clusterResources)
171-
172-
By("Wait for CNI and CCM")
173-
targetTemplate, err := os.ReadFile(path.Join(e2eCtx.Environment.artifactsPath, "config", "target", "base.yaml"))
174-
Expect(err).ToNot(HaveOccurred())
175-
176-
vars := ec.Variables()
177-
targetResources, err := envsubst.Eval(string(targetTemplate), func(varName string) string {
178-
return vars[varName]
179-
})
180-
Expect(err).ToNot(HaveOccurred())
181-
182-
Eventually(func() error {
183-
return ec.Refs.Workload.CreateOrUpdate(ctx, []byte(targetResources))
184-
}, "10m", "15s").Should(Succeed()) // currently this long delay is required as machines might not be ready yet
185-
186-
By("Upgrading the Kubernetes control-plane")
187-
var (
188-
upgradeCPMachineTemplateTo *string
189-
upgradeWorkersMachineTemplateTo *string
190-
)
191-
192-
if input.E2EConfig.HasVariable(CPMachineTemplateUpgradeTo) {
193-
upgradeCPMachineTemplateTo = ptr.To(input.E2EConfig.MustGetVariable(CPMachineTemplateUpgradeTo))
194-
}
195-
196-
if input.E2EConfig.HasVariable(WorkersMachineTemplateUpgradeTo) {
197-
upgradeWorkersMachineTemplateTo = ptr.To(input.E2EConfig.MustGetVariable(WorkersMachineTemplateUpgradeTo))
198-
}
199-
200-
framework.UpgradeControlPlaneAndWaitForUpgrade(ctx, framework.UpgradeControlPlaneAndWaitForUpgradeInput{
201-
ClusterProxy: input.BootstrapClusterProxy,
202-
Cluster: clusterResources.Cluster,
203-
ControlPlane: clusterResources.ControlPlane,
204-
EtcdImageTag: "",
205-
DNSImageTag: "",
206-
KubernetesUpgradeVersion: input.E2EConfig.MustGetVariable(KubernetesVersionUpgradeTo),
207-
UpgradeMachineTemplate: upgradeCPMachineTemplateTo,
208-
WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
209-
WaitForKubeProxyUpgrade: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
210-
WaitForDNSUpgrade: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
211-
WaitForEtcdUpgrade: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
212-
PreWaitForControlPlaneToBeUpgraded: func() {},
213-
})
214-
215-
if workerMachineCount > 0 {
216-
By("Upgrading the machine deployment")
217-
framework.UpgradeMachineDeploymentsAndWait(ctx, framework.UpgradeMachineDeploymentsAndWaitInput{
218-
ClusterProxy: input.BootstrapClusterProxy,
219-
Cluster: clusterResources.Cluster,
220-
UpgradeVersion: input.E2EConfig.MustGetVariable(KubernetesVersionUpgradeTo),
221-
UpgradeMachineTemplate: upgradeWorkersMachineTemplateTo,
222-
MachineDeployments: clusterResources.MachineDeployments,
223-
WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
224-
})
225-
226-
if len(clusterResources.MachinePools) > 0 {
227-
By("Upgrading the machinepool instances")
228-
framework.UpgradeMachinePoolAndWait(ctx, framework.UpgradeMachinePoolAndWaitInput{
229-
ClusterProxy: input.BootstrapClusterProxy,
230-
Cluster: clusterResources.Cluster,
231-
UpgradeVersion: input.E2EConfig.MustGetVariable(KubernetesVersionUpgradeTo),
232-
WaitForMachinePoolToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-pool-upgrade"),
233-
MachinePools: clusterResources.MachinePools,
234-
})
235-
}
236-
}
237-
238-
By("Waiting until nodes are ready")
239-
workloadProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, clusterResources.Cluster.Name)
240-
workloadClient := workloadProxy.GetClient()
241-
framework.WaitForNodesReady(ctx, framework.WaitForNodesReadyInput{
242-
Lister: workloadClient,
243-
KubernetesVersion: input.E2EConfig.MustGetVariable(KubernetesVersionUpgradeTo),
244-
Count: int(clusterResources.ExpectedTotalNodes()),
245-
WaitForNodesReady: input.E2EConfig.GetIntervals(specName, "wait-nodes-ready"),
246-
})
247-
248-
By("PASSED!")
249-
})
250-
251-
//AfterEach(func() {
252-
// // Dumps all the resources in the spec Namespace, then cleanups the cluster object and the spec Namespace itself.
253-
// framework.DumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ClusterctlConfigPath, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
254-
//})
255-
}

0 commit comments

Comments
 (0)