Skip to content

Commit 6f77724

Browse files
timebertttimebertt
authored
Add integration test for shard lease (#497)
* Simplify existing integration tests * Add integration test for shard lease
1 parent c6c02ed commit 6f77724

File tree

3 files changed

+273
-7
lines changed

3 files changed

+273
-7
lines changed

‎test/integration/shard/controller/controller_suite_test.go‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ import (
3535
"sigs.k8s.io/controller-runtime/pkg/manager"
3636
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
3737

38-
"github.com/timebertt/kubernetes-controller-sharding/pkg/utils/test"
3938
. "github.com/timebertt/kubernetes-controller-sharding/pkg/utils/test/matchers"
4039
. "github.com/timebertt/kubernetes-controller-sharding/test/integration/shard/controller"
4140
)
@@ -63,12 +62,7 @@ var _ = BeforeSuite(func(ctx SpecContext) {
6362
log = logf.Log.WithName(testID)
6463

6564
By("Start test environment")
66-
testEnv := &envtest.Environment{
67-
CRDInstallOptions: envtest.CRDInstallOptions{
68-
Paths: []string{test.PathShardingCRDs()},
69-
},
70-
ErrorIfCRDPathMissing: true,
71-
}
65+
testEnv := &envtest.Environment{}
7266

7367
restConfig, err := testEnv.Start()
7468
Expect(err).NotTo(HaveOccurred())

‎test/integration/shard/lease/lease_suite_test.go‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/*
2+
Copyright 2025 Tim Ebert.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controller_test
18+
19+
import (
20+
"context"
21+
"testing"
22+
"time"
23+
24+
"github.com/go-logr/logr"
25+
. "github.com/onsi/ginkgo/v2"
26+
. "github.com/onsi/gomega"
27+
corev1 "k8s.io/api/core/v1"
28+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
29+
"k8s.io/client-go/rest"
30+
"sigs.k8s.io/controller-runtime/pkg/client"
31+
"sigs.k8s.io/controller-runtime/pkg/envtest"
32+
"sigs.k8s.io/controller-runtime/pkg/envtest/komega"
33+
logf "sigs.k8s.io/controller-runtime/pkg/log"
34+
"sigs.k8s.io/controller-runtime/pkg/log/zap"
35+
36+
. "github.com/timebertt/kubernetes-controller-sharding/pkg/utils/test/matchers"
37+
)
38+
39+
func TestController(t *testing.T) {
40+
RegisterFailHandler(Fail)
41+
RunSpecs(t, "Shard Lease Integration Test Suite")
42+
}
43+
44+
const testID = "shard-lease-test"
45+
46+
var (
47+
log logr.Logger
48+
49+
restConfig *rest.Config
50+
51+
testClient client.Client
52+
53+
testRunID string
54+
)
55+
56+
var _ = BeforeSuite(func(ctx SpecContext) {
57+
logf.SetLogger(zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter)))
58+
log = logf.Log.WithName(testID)
59+
60+
By("Start test environment")
61+
testEnv := &envtest.Environment{}
62+
63+
var err error
64+
restConfig, err = testEnv.Start()
65+
Expect(err).NotTo(HaveOccurred())
66+
Expect(restConfig).NotTo(BeNil())
67+
68+
DeferCleanup(func() {
69+
By("Stop test environment")
70+
Expect(testEnv.Stop()).To(Succeed())
71+
})
72+
73+
By("Create test clients")
74+
testClient, err = client.New(restConfig, client.Options{})
75+
Expect(err).NotTo(HaveOccurred())
76+
Expect(testClient).NotTo(BeNil())
77+
78+
clientContext, clientCancel := context.WithCancel(context.Background())
79+
komega.SetClient(testClient)
80+
komega.SetContext(clientContext)
81+
DeferCleanup(clientCancel)
82+
83+
By("Create test Namespace")
84+
testNamespace := &corev1.Namespace{
85+
ObjectMeta: metav1.ObjectMeta{
86+
GenerateName: testID + "-",
87+
},
88+
}
89+
Expect(testClient.Create(ctx, testNamespace)).To(Succeed())
90+
log.Info("Created Namespace for test", "namespaceName", testNamespace.Name)
91+
testRunID = testNamespace.Name
92+
log = log.WithValues("testRunID", testRunID)
93+
94+
DeferCleanup(func(ctx SpecContext) {
95+
By("Delete test Namespace")
96+
Expect(testClient.Delete(ctx, testNamespace)).To(Or(Succeed(), BeNotFoundError()))
97+
}, NodeTimeout(time.Minute))
98+
}, NodeTimeout(time.Minute))

‎test/integration/shard/lease/lease_test.go‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
/*
2+
Copyright 2025 Tim Ebert.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controller_test
18+
19+
import (
20+
"context"
21+
"time"
22+
23+
. "github.com/onsi/ginkgo/v2"
24+
. "github.com/onsi/gomega"
25+
coordinationv1 "k8s.io/api/coordination/v1"
26+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27+
"k8s.io/utils/ptr"
28+
. "sigs.k8s.io/controller-runtime/pkg/envtest/komega"
29+
"sigs.k8s.io/controller-runtime/pkg/manager"
30+
31+
shardlease "github.com/timebertt/kubernetes-controller-sharding/pkg/shard/lease"
32+
"github.com/timebertt/kubernetes-controller-sharding/pkg/utils/test"
33+
)
34+
35+
var _ = Describe("Shard lease", func() {
36+
var (
37+
mgrOptions manager.Options
38+
mgrCancel context.CancelFunc
39+
40+
leaseOptions shardlease.Options
41+
lease *coordinationv1.Lease
42+
)
43+
44+
BeforeEach(func() {
45+
mgrOptions = manager.Options{
46+
LeaderElection: true,
47+
LeaderElectionResourceLock: "should-be-ignored", // -> should use provided lock instead
48+
LeaderElectionNamespace: "should-be-ignored", // -> should use provided namespace in lock instead
49+
LeaderElectionID: "should-be-ignored", // -> should be shard name instead
50+
51+
LeaseDuration: ptr.To(time.Second),
52+
RenewDeadline: ptr.To(100 * time.Millisecond),
53+
RetryPeriod: ptr.To(50 * time.Millisecond),
54+
}
55+
56+
leaseOptions = shardlease.Options{
57+
ControllerRingName: "test-" + test.RandomSuffix(),
58+
LeaseNamespace: testRunID,
59+
ShardName: "test-" + test.RandomSuffix(),
60+
}
61+
62+
lease = &coordinationv1.Lease{
63+
ObjectMeta: metav1.ObjectMeta{
64+
Namespace: leaseOptions.LeaseNamespace,
65+
Name: leaseOptions.ShardName,
66+
},
67+
}
68+
}, OncePerOrdered)
69+
70+
JustBeforeEach(func() {
71+
shardLease, err := shardlease.NewResourceLock(restConfig, nil, leaseOptions)
72+
Expect(err).NotTo(HaveOccurred())
73+
mgrOptions.LeaderElectionResourceLockInterface = shardLease
74+
75+
By("Setup manager")
76+
mgrOptions.Metrics.BindAddress = "0"
77+
mgr, err := manager.New(restConfig, mgrOptions)
78+
Expect(err).NotTo(HaveOccurred())
79+
80+
By("Start manager")
81+
var mgrContext context.Context
82+
mgrContext, mgrCancel = context.WithCancel(context.Background())
83+
84+
mgrDone := make(chan struct{})
85+
go func() {
86+
defer GinkgoRecover()
87+
Expect(mgr.Start(mgrContext)).To(Succeed())
88+
close(mgrDone)
89+
}()
90+
91+
DeferCleanup(func(ctx SpecContext) {
92+
By("Stop manager")
93+
mgrCancel()
94+
95+
By("Wait for manager to stop")
96+
Eventually(ctx, mgrDone).Should(BeClosed())
97+
}, NodeTimeout(time.Minute))
98+
}, OncePerOrdered)
99+
100+
Describe("should maintain the shard lease", Ordered, func() {
101+
It("should create the shard lease according to the config", func(ctx SpecContext) {
102+
Eventually(ctx, Object(lease)).Should(And(
103+
HaveField("ObjectMeta.Labels", HaveKeyWithValue("alpha.sharding.timebertt.dev/controllerring", leaseOptions.ControllerRingName)),
104+
HaveField("Spec.HolderIdentity", HaveValue(Equal(leaseOptions.ShardName))),
105+
HaveField("Spec.LeaseDurationSeconds", HaveValue(BeEquivalentTo(1))),
106+
HaveField("Spec.AcquireTime", Not(BeNil())),
107+
HaveField("Spec.RenewTime", Not(BeNil())),
108+
))
109+
}, SpecTimeout(time.Minute))
110+
111+
It("should renew the shard lease", func(ctx SpecContext) {
112+
Eventually(ctx, Object(lease)).Should(And(
113+
HaveField("Spec.HolderIdentity", HaveValue(Equal(leaseOptions.ShardName))),
114+
HaveField("Spec.LeaseDurationSeconds", HaveValue(BeEquivalentTo(1))),
115+
HaveField("Spec.AcquireTime.Time", BeTemporally("~", lease.Spec.AcquireTime.Time)),
116+
HaveField("Spec.RenewTime.Time", BeTemporally(">", lease.Spec.RenewTime.Time)),
117+
))
118+
}, SpecTimeout(time.Minute))
119+
})
120+
121+
Describe("should renew shard lease that was acquired by sharder", Ordered, func() {
122+
var sharderAcquireTime time.Time
123+
124+
BeforeAll(func(ctx SpecContext) {
125+
sharderAcquireTime = time.Now()
126+
127+
lease.Spec.HolderIdentity = ptr.To("sharder")
128+
lease.Spec.AcquireTime = ptr.To(metav1.NewMicroTime(sharderAcquireTime))
129+
lease.Spec.RenewTime = ptr.To(metav1.NewMicroTime(sharderAcquireTime))
130+
lease.Spec.LeaseDurationSeconds = ptr.To[int32](3)
131+
lease.Spec.LeaseTransitions = ptr.To[int32](1)
132+
133+
Eventually(ctx, func() error {
134+
return testClient.Create(ctx, lease)
135+
}).Should(Succeed())
136+
}, NodeTimeout(time.Minute))
137+
138+
It("should wait for lease to expire", func(ctx SpecContext) {
139+
Consistently(ctx, Object(lease)).WithTimeout(2 * time.Second).ShouldNot(
140+
HaveField("Spec.HolderIdentity", HaveValue(Equal(leaseOptions.ShardName))),
141+
)
142+
}, SpecTimeout(time.Minute))
143+
144+
It("should renew the shard lease", func(ctx SpecContext) {
145+
Eventually(ctx, Object(lease)).Should(And(
146+
HaveField("Spec.HolderIdentity", HaveValue(Equal(leaseOptions.ShardName))),
147+
HaveField("Spec.LeaseDurationSeconds", HaveValue(BeEquivalentTo(1))),
148+
HaveField("Spec.AcquireTime.Time", BeTemporally(">", sharderAcquireTime)),
149+
HaveField("Spec.RenewTime.Time", BeTemporally(">", sharderAcquireTime)),
150+
HaveField("Spec.LeaseTransitions", HaveValue(BeEquivalentTo(2))),
151+
))
152+
}, SpecTimeout(time.Minute))
153+
})
154+
155+
When("LeaderElectionReleaseOnCancel is true", Ordered, func() {
156+
BeforeAll(func() {
157+
mgrOptions.LeaderElectionReleaseOnCancel = true
158+
})
159+
160+
It("should create the shard lease", func(ctx SpecContext) {
161+
Eventually(ctx, Object(lease)).Should(
162+
HaveField("Spec.HolderIdentity", HaveValue(Equal(leaseOptions.ShardName))),
163+
)
164+
}, SpecTimeout(time.Minute))
165+
166+
It("should release the shard lease when canceling the manager", func(ctx SpecContext) {
167+
mgrCancel()
168+
169+
Eventually(ctx, Object(lease)).Should(
170+
HaveField("Spec.HolderIdentity", HaveValue(BeEmpty())),
171+
)
172+
}, SpecTimeout(time.Minute))
173+
})
174+
})

0 commit comments

Comments
 (0)