Skip to content

Commit e59f792

Browse files
timebertttimebertt
committed
Add integration test for controllerring controller
1 parent 0a74a85 commit e59f792

File tree

2 files changed

+327
-0
lines changed

2 files changed

+327
-0
lines changed

‎test/integration/sharder/controller/controllerring/controllerring_suite_test.go‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
/*
2+
Copyright 2025 Tim Ebert.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controllerring_test
18+
19+
import (
20+
"context"
21+
"testing"
22+
"time"
23+
24+
"github.com/go-logr/logr"
25+
. "github.com/onsi/ginkgo/v2"
26+
. "github.com/onsi/gomega"
27+
coordinationv1 "k8s.io/api/coordination/v1"
28+
corev1 "k8s.io/api/core/v1"
29+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
30+
"k8s.io/apimachinery/pkg/labels"
31+
testclock "k8s.io/utils/clock/testing"
32+
"sigs.k8s.io/controller-runtime/pkg/cache"
33+
"sigs.k8s.io/controller-runtime/pkg/client"
34+
"sigs.k8s.io/controller-runtime/pkg/envtest"
35+
"sigs.k8s.io/controller-runtime/pkg/envtest/komega"
36+
logf "sigs.k8s.io/controller-runtime/pkg/log"
37+
"sigs.k8s.io/controller-runtime/pkg/log/zap"
38+
"sigs.k8s.io/controller-runtime/pkg/manager"
39+
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
40+
41+
configv1alpha1 "github.com/timebertt/kubernetes-controller-sharding/pkg/apis/config/v1alpha1"
42+
shardingv1alpha1 "github.com/timebertt/kubernetes-controller-sharding/pkg/apis/sharding/v1alpha1"
43+
"github.com/timebertt/kubernetes-controller-sharding/pkg/controller/controllerring"
44+
utilclient "github.com/timebertt/kubernetes-controller-sharding/pkg/utils/client"
45+
"github.com/timebertt/kubernetes-controller-sharding/pkg/utils/test"
46+
. "github.com/timebertt/kubernetes-controller-sharding/pkg/utils/test/matchers"
47+
)
48+
49+
func TestControllerRing(t *testing.T) {
50+
RegisterFailHandler(Fail)
51+
RunSpecs(t, "Sharder ControllerRing Controller Integration Test Suite")
52+
}
53+
54+
const testID = "controllerring-test"
55+
56+
var (
57+
log logr.Logger
58+
59+
testClient client.Client
60+
61+
clock *testclock.FakePassiveClock
62+
63+
testNamespace *corev1.Namespace
64+
testRunID string
65+
testRunLabels map[string]string
66+
)
67+
68+
var _ = BeforeSuite(func(ctx SpecContext) {
69+
logf.SetLogger(zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter)))
70+
log = logf.Log.WithName(testID)
71+
72+
By("Start test environment")
73+
testEnv := &envtest.Environment{
74+
CRDInstallOptions: envtest.CRDInstallOptions{
75+
Paths: []string{test.PathShardingCRDs()},
76+
},
77+
ErrorIfCRDPathMissing: true,
78+
}
79+
80+
restConfig, err := testEnv.Start()
81+
Expect(err).NotTo(HaveOccurred())
82+
Expect(restConfig).NotTo(BeNil())
83+
84+
DeferCleanup(func() {
85+
By("Stop test environment")
86+
Expect(testEnv.Stop()).To(Succeed())
87+
})
88+
89+
By("Create test clients")
90+
testClient, err = client.New(restConfig, client.Options{Scheme: utilclient.SharderScheme})
91+
Expect(err).NotTo(HaveOccurred())
92+
Expect(testClient).NotTo(BeNil())
93+
94+
clientContext, clientCancel := context.WithCancel(context.Background())
95+
komega.SetClient(testClient)
96+
komega.SetContext(clientContext)
97+
DeferCleanup(clientCancel)
98+
99+
By("Create test Namespace")
100+
testNamespace = &corev1.Namespace{
101+
ObjectMeta: metav1.ObjectMeta{
102+
GenerateName: testID + "-",
103+
},
104+
}
105+
Expect(testClient.Create(ctx, testNamespace)).To(Succeed())
106+
log.Info("Created Namespace for test", "namespaceName", testNamespace.Name)
107+
testRunID = testNamespace.Name
108+
log = log.WithValues("testRunID", testRunID)
109+
testRunLabels = map[string]string{testID: testRunID}
110+
111+
DeferCleanup(func(ctx SpecContext) {
112+
By("Delete test Namespace")
113+
Expect(testClient.Delete(ctx, testNamespace)).To(Or(Succeed(), BeNotFoundError()))
114+
}, NodeTimeout(time.Minute))
115+
116+
By("Setup manager")
117+
mgr, err := manager.New(restConfig, manager.Options{
118+
Scheme: utilclient.SharderScheme,
119+
Metrics: metricsserver.Options{BindAddress: "0"},
120+
Cache: cache.Options{
121+
DefaultNamespaces: map[string]cache.Config{testNamespace.Name: {}},
122+
ByObject: map[client.Object]cache.ByObject{
123+
&shardingv1alpha1.ControllerRing{}: {
124+
Label: labels.SelectorFromSet(testRunLabels),
125+
},
126+
&coordinationv1.Lease{}: {
127+
Label: labels.SelectorFromSet(testRunLabels),
128+
},
129+
},
130+
},
131+
})
132+
Expect(err).NotTo(HaveOccurred())
133+
134+
By("Register controller")
135+
config := &configv1alpha1.SharderConfig{}
136+
mgr.GetScheme().Default(config)
137+
138+
clock = testclock.NewFakePassiveClock(time.Now())
139+
140+
Expect((&controllerring.Reconciler{
141+
Clock: clock,
142+
Config: config,
143+
}).AddToManager(mgr)).To(Succeed())
144+
145+
By("Start manager")
146+
mgrContext, mgrCancel := context.WithCancel(context.Background())
147+
148+
go func() {
149+
defer GinkgoRecover()
150+
Expect(mgr.Start(mgrContext)).To(Succeed())
151+
}()
152+
153+
DeferCleanup(func() {
154+
By("Stop manager")
155+
mgrCancel()
156+
})
157+
}, NodeTimeout(time.Minute))

‎test/integration/sharder/controller/controllerring/controllerring_test.go‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
/*
2+
Copyright 2025 Tim Ebert.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controllerring_test
18+
19+
import (
20+
"maps"
21+
"time"
22+
23+
. "github.com/onsi/ginkgo/v2"
24+
. "github.com/onsi/gomega"
25+
gomegatypes "github.com/onsi/gomega/types"
26+
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
27+
coordinationv1 "k8s.io/api/coordination/v1"
28+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
29+
"k8s.io/utils/ptr"
30+
. "sigs.k8s.io/controller-runtime/pkg/envtest/komega"
31+
32+
shardingv1alpha1 "github.com/timebertt/kubernetes-controller-sharding/pkg/apis/sharding/v1alpha1"
33+
"github.com/timebertt/kubernetes-controller-sharding/pkg/utils/test"
34+
. "github.com/timebertt/kubernetes-controller-sharding/pkg/utils/test/matchers"
35+
)
36+
37+
var _ = Describe("ControllerRing controller", func() {
38+
var (
39+
controllerRing *shardingv1alpha1.ControllerRing
40+
)
41+
42+
BeforeEach(func(ctx SpecContext) {
43+
controllerRing = &shardingv1alpha1.ControllerRing{
44+
ObjectMeta: metav1.ObjectMeta{
45+
GenerateName: testRunID + "-",
46+
Labels: maps.Clone(testRunLabels),
47+
},
48+
Spec: shardingv1alpha1.ControllerRingSpec{
49+
Resources: []shardingv1alpha1.RingResource{
50+
{
51+
GroupResource: metav1.GroupResource{Group: "apps", Resource: "deployments"},
52+
},
53+
},
54+
NamespaceSelector: nil,
55+
},
56+
}
57+
58+
Expect(testClient.Create(ctx, controllerRing)).To(Succeed())
59+
log.Info("Created ControllerRing for test", "controllerRingName", controllerRing.Name)
60+
61+
DeferCleanup(func(ctx SpecContext) {
62+
Expect(testClient.Delete(ctx, controllerRing)).To(Or(Succeed(), BeNotFoundError()))
63+
}, NodeTimeout(time.Minute))
64+
}, NodeTimeout(time.Minute), OncePerOrdered)
65+
66+
It("should observe the generation", func(ctx SpecContext) {
67+
Eventually(ctx, Object(controllerRing)).Should(
68+
HaveField("Status.ObservedGeneration", Equal(controllerRing.Generation)),
69+
)
70+
}, SpecTimeout(time.Minute))
71+
72+
It("should report readiness", func(ctx SpecContext) {
73+
Eventually(ctx, Object(controllerRing)).Should(
74+
HaveField("Status.Conditions", ConsistOf(
75+
MatchCondition(
76+
OfType(shardingv1alpha1.ControllerRingReady),
77+
WithStatus(metav1.ConditionTrue),
78+
),
79+
)),
80+
)
81+
}, SpecTimeout(time.Minute))
82+
83+
It("should apply the sharder webhook", func(ctx SpecContext) {
84+
webhookConfig := &admissionregistrationv1.MutatingWebhookConfiguration{
85+
ObjectMeta: metav1.ObjectMeta{Name: "controllerring-" + controllerRing.Name},
86+
}
87+
Eventually(ctx, Object(webhookConfig)).Should(And(
88+
HaveField("ObjectMeta.OwnerReferences", ConsistOf(And(
89+
HaveField("Kind", Equal("ControllerRing")),
90+
HaveField("Name", Equal(controllerRing.Name)),
91+
))),
92+
HaveField("Webhooks", ConsistOf(And(
93+
HaveField("ClientConfig.Service.Path", HaveValue(Equal("/webhooks/sharder/controllerring/"+controllerRing.Name))),
94+
HaveField("Rules", ConsistOf(And(
95+
HaveField("APIGroups", ConsistOf("apps")),
96+
HaveField("Resources", ConsistOf("deployments")),
97+
))),
98+
))),
99+
))
100+
}, SpecTimeout(time.Minute))
101+
102+
Describe("should reflect the shard leases in the status", Ordered, func() {
103+
var lease *coordinationv1.Lease
104+
105+
It("Create available shard lease", func(ctx SpecContext) {
106+
lease = newLease(controllerRing.Name)
107+
Expect(testClient.Create(ctx, lease)).To(Succeed())
108+
109+
Eventually(ctx, Object(controllerRing)).Should(haveStatusShards(1, 1))
110+
}, SpecTimeout(time.Minute))
111+
112+
It("Create orphaned shard lease", func(ctx SpecContext) {
113+
lease = newLease(controllerRing.Name)
114+
lease.Spec.HolderIdentity = nil
115+
Expect(testClient.Create(ctx, lease)).To(Succeed())
116+
117+
Eventually(ctx, Object(controllerRing)).Should(haveStatusShards(1, 2))
118+
}, SpecTimeout(time.Minute))
119+
120+
It("Make lease healthy", func(ctx SpecContext) {
121+
Eventually(ctx, Update(lease, func() {
122+
lease.Spec.HolderIdentity = ptr.To(lease.Name)
123+
})).Should(Succeed())
124+
125+
Eventually(ctx, Object(controllerRing)).Should(haveStatusShards(2, 2))
126+
}, SpecTimeout(time.Minute))
127+
128+
It("Make lease unhealthy", func(ctx SpecContext) {
129+
Eventually(ctx, Update(lease, func() {
130+
lease.Spec.HolderIdentity = nil
131+
})).Should(Succeed())
132+
133+
Eventually(ctx, Object(controllerRing)).Should(haveStatusShards(1, 2))
134+
}, SpecTimeout(time.Minute))
135+
136+
It("Delete unhealthy lease", func(ctx SpecContext) {
137+
Expect(testClient.Delete(ctx, lease)).To(Succeed())
138+
139+
Eventually(ctx, Object(controllerRing)).Should(haveStatusShards(1, 1))
140+
}, SpecTimeout(time.Minute))
141+
})
142+
})
143+
144+
func newLease(controllerRingName string) *coordinationv1.Lease {
145+
name := testRunID + "-" + test.RandomSuffix()
146+
147+
lease := &coordinationv1.Lease{
148+
ObjectMeta: metav1.ObjectMeta{
149+
Name: name,
150+
Namespace: testRunID,
151+
Labels: maps.Clone(testRunLabels),
152+
},
153+
Spec: coordinationv1.LeaseSpec{
154+
HolderIdentity: ptr.To(name),
155+
LeaseDurationSeconds: ptr.To[int32](10),
156+
AcquireTime: ptr.To(metav1.NewMicroTime(clock.Now().Add(-5 * time.Minute))),
157+
RenewTime: ptr.To(metav1.NewMicroTime(clock.Now().Add(-2 * time.Second))),
158+
},
159+
}
160+
metav1.SetMetaDataLabel(&lease.ObjectMeta, shardingv1alpha1.LabelControllerRing, controllerRingName)
161+
162+
return lease
163+
}
164+
165+
func haveStatusShards(availableShards, shards int32) gomegatypes.GomegaMatcher {
166+
return And(
167+
HaveField("Status.AvailableShards", Equal(availableShards)),
168+
HaveField("Status.Shards", Equal(shards)),
169+
)
170+
}

0 commit comments

Comments
 (0)