Skip to content

Commit aa250fb

Browse files
vishesh92vishesh92
committed
Move CKS integration to a separate controller
1 parent 4e29479 commit aa250fb

17 files changed

+517
-90
lines changed

‎api/v1beta1/conversion.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ func Convert_v1beta1_CloudStackCluster_To_v1beta3_CloudStackCluster(in *CloudSta
5151
//nolint:golint,revive,stylecheck
5252
func Convert_v1beta3_CloudStackCluster_To_v1beta1_CloudStackCluster(in *v1beta3.CloudStackCluster, out *CloudStackCluster, scope conv.Scope) error {
5353
if len(in.Spec.FailureDomains) < 1 {
54-
return fmt.Errorf("v1beta3 to v1beta1 conversion not supported when < 1 failure domain is provided. Input CloudStackCluster spec %s", fmt.Sprintf("%v", in.Spec))
54+
return fmt.Errorf("v1beta3 to v1beta1 conversion not supported when < 1 failure domain is provided. Input CloudStackCluster spec %v", in.Spec)
5555
}
5656
out.ObjectMeta = in.ObjectMeta
5757
out.Spec = CloudStackClusterSpec{

‎controllers/cks_cluster_controller.go

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controllers
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"strings"
23+
24+
ctrl "sigs.k8s.io/controller-runtime"
25+
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
26+
27+
infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3"
28+
csCtrlrUtils "sigs.k8s.io/cluster-api-provider-cloudstack/controllers/utils"
29+
)
30+
31+
const CksClusterFinalizer = "ckscluster.infrastructure.cluster.x-k8s.io"
32+
33+
// RBAC permissions for CloudStackCluster.
34+
// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackclusters,verbs=get;list;watch;update;patch
35+
// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackclusters/status,verbs=create;get;update;patch
36+
// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackclusters/finalizers,verbs=update
37+
38+
// CksClusterReconciliationRunner is a ReconciliationRunner with extensions specific to CloudStackClusters.
39+
// The runner does the actual reconciliation.
40+
type CksClusterReconciliationRunner struct {
41+
*csCtrlrUtils.ReconciliationRunner
42+
FailureDomains *infrav1.CloudStackFailureDomainList
43+
ReconciliationSubject *infrav1.CloudStackCluster
44+
}
45+
46+
// CksClusterReconciler is the k8s controller manager's interface to reconcile a CloudStackCluster.
47+
// This is primarily to adapt to k8s.
48+
type CksClusterReconciler struct {
49+
csCtrlrUtils.ReconcilerBase
50+
}
51+
52+
// Initialize a new CloudStackCluster reconciliation runner with concrete types and initialized member fields.
53+
func NewCksClusterReconciliationRunner() *CksClusterReconciliationRunner {
54+
// Set concrete type and init pointers.
55+
runner := &CksClusterReconciliationRunner{ReconciliationSubject: &infrav1.CloudStackCluster{}}
56+
runner.FailureDomains = &infrav1.CloudStackFailureDomainList{}
57+
// Setup the base runner. Initializes pointers and links reconciliation methods.
58+
runner.ReconciliationRunner = csCtrlrUtils.NewRunner(runner, runner.ReconciliationSubject, "CKSClusterController")
59+
runner.CSCluster = runner.ReconciliationSubject
60+
return runner
61+
}
62+
63+
// Reconcile is the method k8s will call upon a reconciliation request.
64+
func (reconciler *CksClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (retRes ctrl.Result, retErr error) {
65+
r := NewCksClusterReconciliationRunner()
66+
r.UsingBaseReconciler(reconciler.ReconcilerBase).ForRequest(req).WithRequestCtx(ctx)
67+
r.WithAdditionalCommonStages(r.GetFailureDomains(r.FailureDomains))
68+
return r.RunBaseReconciliationStages()
69+
}
70+
71+
// Reconcile actually reconciles the CloudStackCluster.
72+
func (r *CksClusterReconciliationRunner) Reconcile() (res ctrl.Result, reterr error) {
73+
if !r.CSCluster.Spec.SyncWithACS {
74+
return ctrl.Result{}, nil
75+
}
76+
// Prevent premature deletion.
77+
controllerutil.AddFinalizer(r.ReconciliationSubject, CksClusterFinalizer)
78+
79+
if len(r.FailureDomains.Items) == 0 {
80+
return r.RequeueWithMessage("No failure domains found")
81+
}
82+
83+
res, err := r.AsFailureDomainUser(&r.FailureDomains.Items[0].Spec)()
84+
if r.ShouldReturn(res, err) {
85+
return res, err
86+
}
87+
88+
r.Log.Info("Creating entry with CKS")
89+
err = r.CSUser.GetOrCreateCksCluster(r.CAPICluster, r.ReconciliationSubject, &r.FailureDomains.Items[0].Spec)
90+
if err != nil {
91+
return r.RequeueWithMessage(fmt.Sprintf("Failed creating ExternalManaged CKS cluster on CloudStack. error: %s", err.Error()))
92+
}
93+
return ctrl.Result{}, nil
94+
}
95+
96+
// ReconcileDelete cleans up resources used by the cluster and finally removes the CloudStackCluster's finalizers.
97+
func (r *CksClusterReconciliationRunner) ReconcileDelete() (ctrl.Result, error) {
98+
if r.ReconciliationSubject.Status.CloudStackClusterID != "" {
99+
if len(r.FailureDomains.Items) == 0 {
100+
return ctrl.Result{}, fmt.Errorf("no failure domains found")
101+
}
102+
res, err := r.AsFailureDomainUser(&r.FailureDomains.Items[0].Spec)()
103+
if r.ShouldReturn(res, err) {
104+
return res, err
105+
}
106+
err = r.CSUser.DeleteCksCluster(r.ReconciliationSubject)
107+
if err != nil && !strings.Contains(err.Error(), " not found") {
108+
return r.RequeueWithMessage(fmt.Sprintf("Deleting cks cluster on CloudStack failed. error: %s", err.Error()))
109+
}
110+
}
111+
controllerutil.RemoveFinalizer(r.ReconciliationSubject, CksClusterFinalizer)
112+
return ctrl.Result{}, nil
113+
}
114+
115+
// SetupWithManager sets up the controller with the Manager.
116+
func (reconciler *CksClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
117+
return ctrl.NewControllerManagedBy(mgr).
118+
For(&infrav1.CloudStackCluster{}).
119+
Complete(reconciler)
120+
}

‎controllers/cks_cluster_controller_test.go

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controllers_test
18+
19+
import (
20+
"github.com/golang/mock/gomock"
21+
. "github.com/onsi/ginkgo/v2"
22+
. "github.com/onsi/gomega"
23+
"k8s.io/apimachinery/pkg/api/errors"
24+
infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3"
25+
"sigs.k8s.io/cluster-api-provider-cloudstack/controllers"
26+
"sigs.k8s.io/cluster-api-provider-cloudstack/pkg/cloud"
27+
dummies "sigs.k8s.io/cluster-api-provider-cloudstack/test/dummies/v1beta3"
28+
"sigs.k8s.io/controller-runtime/pkg/client"
29+
"sigs.k8s.io/controller-runtime/pkg/controller"
30+
)
31+
32+
var _ = Describe("CksCloudStackClusterReconciler", func() {
33+
Context("With k8s like test environment.", func() {
34+
BeforeEach(func() {
35+
dummies.SetDummyVars()
36+
SetupTestEnvironment()
37+
Ω(ClusterReconciler.SetupWithManager(ctx, k8sManager, controller.Options{})).Should(Succeed()) // Register CloudStack ClusterReconciler.
38+
Ω(FailureDomainReconciler.SetupWithManager(k8sManager, controller.Options{})).Should(Succeed()) // Register CloudStack FailureDomainReconciler.
39+
Ω(CksClusterReconciler.SetupWithManager(k8sManager)).Should(Succeed()) // Register CloudStack Cks ClusterReconciler.
40+
})
41+
42+
It("Should create a cluster in CKS.", func() {
43+
mockCloudClient.EXPECT().GetOrCreateCksCluster(gomock.Any(), gomock.Any(), gomock.Any()).Do(func(_, arg1, _ interface{}) {
44+
arg1.(*infrav1.CloudStackCluster).Status.CloudStackClusterID = "cluster-id-123"
45+
}).MinTimes(1).Return(nil)
46+
mockCloudClient.EXPECT().ResolveZone(gomock.Any()).AnyTimes()
47+
mockCloudClient.EXPECT().ResolveNetworkForZone(gomock.Any()).AnyTimes().Do(
48+
func(arg1 interface{}) {
49+
arg1.(*infrav1.CloudStackZoneSpec).Network.ID = "SomeID"
50+
arg1.(*infrav1.CloudStackZoneSpec).Network.Type = cloud.NetworkTypeShared
51+
}).MinTimes(1)
52+
53+
Eventually(func() string {
54+
key := client.ObjectKeyFromObject(dummies.CSCluster)
55+
if err := k8sClient.Get(ctx, key, dummies.CSCluster); err != nil {
56+
return ""
57+
}
58+
return dummies.CSCluster.Status.CloudStackClusterID
59+
}, timeout).WithPolling(pollInterval).Should(Equal("cluster-id-123"))
60+
61+
})
62+
})
63+
64+
Context("With k8s like test environment.", func() {
65+
BeforeEach(func() {
66+
dummies.SetDummyVars()
67+
dummies.CSCluster.Status.CloudStackClusterID = "cluster-id-123"
68+
SetupTestEnvironment()
69+
Ω(ClusterReconciler.SetupWithManager(ctx, k8sManager, controller.Options{})).Should(Succeed()) // Register CloudStack ClusterReconciler.
70+
Ω(FailureDomainReconciler.SetupWithManager(k8sManager, controller.Options{})).Should(Succeed()) // Register CloudStack FailureDomainReconciler.
71+
Ω(CksClusterReconciler.SetupWithManager(k8sManager)).Should(Succeed()) // Register CloudStack Cks ClusterReconciler.
72+
})
73+
74+
It("Should delete the cluster in CKS.", func() {
75+
76+
Ω(k8sClient.Delete(ctx, dummies.CSCluster)).Should(Succeed())
77+
78+
Eventually(func() bool {
79+
csCluster := &infrav1.CloudStackCluster{}
80+
csClusterKey := client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: dummies.CSCluster.Name}
81+
if err := k8sClient.Get(ctx, csClusterKey, csCluster); err != nil {
82+
return errors.IsNotFound(err)
83+
}
84+
return false
85+
}, timeout).WithPolling(pollInterval).Should(BeTrue())
86+
})
87+
88+
})
89+
90+
Context("Without a k8s test environment.", func() {
91+
It("Should create a reconciliation runner with a Cloudstack Cluster as the reconciliation subject.", func() {
92+
reconRunner := controllers.NewCksClusterReconciliationRunner()
93+
Ω(reconRunner.ReconciliationSubject).ShouldNot(BeNil())
94+
})
95+
})
96+
})

‎controllers/cks_machine_controller.go

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
/*
2+
Copyright 2022 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controllers
18+
19+
import (
20+
"context"
21+
"fmt"
22+
23+
ctrl "sigs.k8s.io/controller-runtime"
24+
25+
infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3"
26+
csCtrlrUtils "sigs.k8s.io/cluster-api-provider-cloudstack/controllers/utils"
27+
)
28+
29+
// RBAC permissions for CloudStackCluster.
30+
// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackmachines,verbs=get;list;watch
31+
// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackmachines/status,verbs=get
32+
33+
// CksMachineReconciliationRunner is a ReconciliationRunner with extensions specific to CloudStackClusters.
34+
// The runner does the actual reconciliation.
35+
type CksMachineReconciliationRunner struct {
36+
*csCtrlrUtils.ReconciliationRunner
37+
FailureDomain *infrav1.CloudStackFailureDomain
38+
ReconciliationSubject *infrav1.CloudStackMachine
39+
}
40+
41+
// CksMachineReconciler is the k8s controller manager's interface to reconcile a CloudStackCluster.
42+
// This is primarily to adapt to k8s.
43+
type CksMachineReconciler struct {
44+
csCtrlrUtils.ReconcilerBase
45+
}
46+
47+
// Initialize a new CloudStackCluster reconciliation runner with concrete types and initialized member fields.
48+
func NewCksMachineReconciliationRunner() *CksMachineReconciliationRunner {
49+
// Set concrete type and init pointers.
50+
runner := &CksMachineReconciliationRunner{ReconciliationSubject: &infrav1.CloudStackMachine{}}
51+
runner.FailureDomain = &infrav1.CloudStackFailureDomain{}
52+
// Setup the base runner. Initializes pointers and links reconciliation methods.
53+
runner.ReconciliationRunner = csCtrlrUtils.NewRunner(runner, runner.ReconciliationSubject, "CKSMachineController")
54+
return runner
55+
}
56+
57+
// Reconcile is the method k8s will call upon a reconciliation request.
58+
func (reconciler *CksMachineReconciler) Reconcile(ctx context.Context, req ctrl.Request) (retRes ctrl.Result, retErr error) {
59+
r := NewCksMachineReconciliationRunner()
60+
r.UsingBaseReconciler(reconciler.ReconcilerBase).ForRequest(req).WithRequestCtx(ctx)
61+
r.WithAdditionalCommonStages(
62+
r.GetFailureDomainByName(func() string { return r.ReconciliationSubject.Spec.FailureDomainName }, r.FailureDomain),
63+
r.AsFailureDomainUser(&r.FailureDomain.Spec))
64+
return r.RunBaseReconciliationStages()
65+
}
66+
67+
// Reconcile actually reconciles the CloudStackCluster.
68+
func (r *CksMachineReconciliationRunner) Reconcile() (res ctrl.Result, reterr error) {
69+
if !r.CSCluster.Spec.SyncWithACS {
70+
return ctrl.Result{}, nil
71+
}
72+
if r.CSCluster.Status.CloudStackClusterID == "" {
73+
return r.RequeueWithMessage("CloudStackClusterID is not set")
74+
}
75+
76+
if r.ReconciliationSubject.Spec.InstanceID == nil || *r.ReconciliationSubject.Spec.InstanceID == "" {
77+
return r.RequeueWithMessage("InstanceID is not set")
78+
}
79+
80+
res, err := r.AsFailureDomainUser(&r.FailureDomain.Spec)()
81+
if r.ShouldReturn(res, err) {
82+
return res, err
83+
}
84+
r.Log.Info("Assigning VM to CKS")
85+
err = r.CSUser.AddVMToCksCluster(r.CSCluster, r.ReconciliationSubject)
86+
if err != nil {
87+
return r.RequeueWithMessage(fmt.Sprintf("Adding VM to CloudStack CKS cluster failed. error: %s", err.Error()))
88+
}
89+
return ctrl.Result{}, nil
90+
91+
}
92+
93+
// ReconcileDelete cleans up resources used by the cluster and finally removes the CloudStackCluster's finalizers.
94+
func (r *CksMachineReconciliationRunner) ReconcileDelete() (ctrl.Result, error) {
95+
if r.ReconciliationSubject.Spec.InstanceID != nil && *r.ReconciliationSubject.Spec.InstanceID != "" {
96+
err := r.CSUser.RemoveVMFromCksCluster(r.CSCluster, r.ReconciliationSubject)
97+
if err != nil {
98+
return r.RequeueWithMessage(fmt.Sprintf("Removing VM from CloudStack CKS cluster failed. error: %s", err.Error()))
99+
}
100+
}
101+
return ctrl.Result{}, nil
102+
}
103+
104+
// SetupWithManager sets up the controller with the Manager.
105+
func (reconciler *CksMachineReconciler) SetupWithManager(mgr ctrl.Manager) error {
106+
return ctrl.NewControllerManagedBy(mgr).
107+
For(&infrav1.CloudStackMachine{}).
108+
Complete(reconciler)
109+
}

0 commit comments

Comments
 (0)