Skip to content

Commit ca9734a

Browse files
Henrywu573Henrywu573
committed
Add third party strategy to the coordinate leader election integration test
1 parent a5dda5d commit ca9734a

File tree

2 files changed

+149
-26
lines changed

2 files changed

+149
-26
lines changed

‎pkg/controlplane/controller/leaderelection/leaderelection_controller.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -374,7 +374,7 @@ func (c *Controller) reconcileElectionStep(ctx context.Context, leaseNN types.Na
374374
orig := existing.DeepCopy()
375375

376376
isExpired := isLeaseExpired(c.clock, existing)
377-
noHolderIdentity := leaderLease.Spec.HolderIdentity != nil && existing.Spec.HolderIdentity == nil || *existing.Spec.HolderIdentity == ""
377+
noHolderIdentity := leaderLease.Spec.HolderIdentity != nil && (existing.Spec.HolderIdentity == nil || *existing.Spec.HolderIdentity == "")
378378
expiredAndNewHolder := isExpired && leaderLease.Spec.HolderIdentity != nil && *existing.Spec.HolderIdentity != *leaderLease.Spec.HolderIdentity
379379
strategyChanged := existing.Spec.Strategy == nil || *existing.Spec.Strategy != strategy
380380
differentHolder := leaderLease.Spec.HolderIdentity != nil && *leaderLease.Spec.HolderIdentity != *existing.Spec.HolderIdentity
@@ -402,7 +402,11 @@ func (c *Controller) reconcileElectionStep(ctx context.Context, leaseNN types.Na
402402
}
403403

404404
if reflect.DeepEqual(existing, orig) {
405-
klog.V(5).Infof("Lease %s already has the most optimal leader %q", leaseNN, *existing.Spec.HolderIdentity)
405+
if existing.Spec.HolderIdentity != nil {
406+
klog.V(5).Infof("Lease %s is managed by a third party strategy", *existing.Spec.HolderIdentity)
407+
} else {
408+
klog.V(5).Infof("Lease %s already has the most optimal leader %q", leaseNN, "")
409+
}
406410
// We need to requeue to ensure that we are aware of an expired lease
407411
return defaultRequeueInterval, nil
408412
}

‎test/integration/apiserver/coordinated_leader_election_test.go‎

Lines changed: 143 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,21 @@ import (
3939
apiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
4040
"k8s.io/kubernetes/pkg/controlplane/apiserver"
4141
"k8s.io/kubernetes/test/integration/framework"
42+
"k8s.io/utils/ptr"
4243
)
4344

4445
func TestSingleLeaseCandidate(t *testing.T) {
46+
tests := []struct {
47+
name string
48+
preferredStrategy v1.CoordinatedLeaseStrategy
49+
expectedHolderIdentity *string
50+
}{
51+
{
52+
name: "default strategy, has lease identity",
53+
preferredStrategy: v1.OldestEmulationVersion,
54+
expectedHolderIdentity: ptr.To("foo1"),
55+
},
56+
}
4557
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.CoordinatedLeaderElection, true)
4658

4759
flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha2.SchemeGroupVersion)}
@@ -51,16 +63,72 @@ func TestSingleLeaseCandidate(t *testing.T) {
5163
}
5264
defer server.TearDownFn()
5365
config := server.ClientConfig
66+
ctx, cancel := context.WithCancel(context.Background())
67+
defer cancel()
68+
69+
for _, tc := range tests {
70+
t.Run(tc.name, func(t *testing.T) {
71+
cletest := setupCLE(config, ctx, cancel, t)
72+
defer cletest.cleanup()
73+
go cletest.createAndRunFakeController("foo1", "default", "foo", "1.20.0", "1.20.0", tc.preferredStrategy)
74+
cletest.pollForLease(ctx, "foo", "default", tc.expectedHolderIdentity)
75+
})
76+
}
77+
}
5478

79+
func TestSingleLeaseCandidateUsingThirdPartyStrategy(t *testing.T) {
80+
tests := []struct {
81+
name string
82+
preferredStrategy v1.CoordinatedLeaseStrategy
83+
expectedHolderIdentity *string
84+
}{
85+
{
86+
name: "third party strategy, no holder identity",
87+
preferredStrategy: v1.CoordinatedLeaseStrategy("foo.com/bar"),
88+
// Because a third-party CoordinatedLeaseStrategy is in use,
89+
// the coordinated leader election controller doesn't manage
90+
// the lease's leader election and does not set the holder identity,
91+
// and leave it to some other controller. The lease will be created
92+
// with the strategy set but holderIdentity set to nil.
93+
expectedHolderIdentity: nil,
94+
},
95+
}
96+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.CoordinatedLeaderElection, true)
97+
flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha2.SchemeGroupVersion)}
98+
server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), flags, framework.SharedEtcd())
99+
if err != nil {
100+
t.Fatal(err)
101+
}
102+
defer server.TearDownFn()
103+
config := server.ClientConfig
55104
ctx, cancel := context.WithCancel(context.Background())
56105
defer cancel()
57-
cletest := setupCLE(config, ctx, cancel, t)
58-
defer cletest.cleanup()
59-
go cletest.createAndRunFakeController("foo1", "default", "foo", "1.20.0", "1.20.0")
60-
cletest.pollForLease(ctx, "foo", "default", "foo1")
106+
107+
for _, tc := range tests {
108+
t.Run(tc.name, func(t *testing.T) {
109+
cletest := setupCLE(config, ctx, cancel, t)
110+
defer cletest.cleanup()
111+
go cletest.createAndRunFakeController("foo1", "default", "foo", "1.20.0", "1.20.0", tc.preferredStrategy)
112+
cletest.pollForLease(ctx, "foo", "default", tc.expectedHolderIdentity)
113+
})
114+
}
61115
}
62116

63117
func TestMultipleLeaseCandidate(t *testing.T) {
118+
tests := []struct {
119+
name string
120+
preferredStrategy v1.CoordinatedLeaseStrategy
121+
expectedHolderIdentity *string
122+
}{
123+
{
124+
name: "default strategy, has lease identity",
125+
preferredStrategy: v1.OldestEmulationVersion,
126+
// Because the OldestEmulationVersion strategy is used here, the
127+
// the coordinated leader election controller will pick the candidate
128+
// with OldestEmulationVersion and OldestBinaryVersion.
129+
expectedHolderIdentity: ptr.To("baz3"),
130+
},
131+
}
64132
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.CoordinatedLeaderElection, true)
65133

66134
flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha2.SchemeGroupVersion)}
@@ -73,14 +141,62 @@ func TestMultipleLeaseCandidate(t *testing.T) {
73141

74142
ctx, cancel := context.WithCancel(context.Background())
75143
defer cancel()
76-
cletest := setupCLE(config, ctx, cancel, t)
77-
defer cletest.cleanup()
78-
go cletest.createAndRunFakeController("baz1", "default", "baz", "1.20.0", "1.20.0")
79-
go cletest.createAndRunFakeController("baz2", "default", "baz", "1.20.0", "1.19.0")
80-
go cletest.createAndRunFakeController("baz3", "default", "baz", "1.19.0", "1.19.0")
81-
go cletest.createAndRunFakeController("baz4", "default", "baz", "1.2.0", "1.19.0")
82-
go cletest.createAndRunFakeController("baz5", "default", "baz", "1.20.0", "1.19.0")
83-
cletest.pollForLease(ctx, "baz", "default", "baz3")
144+
for _, tc := range tests {
145+
t.Run(tc.name, func(t *testing.T) {
146+
147+
cletest := setupCLE(config, ctx, cancel, t)
148+
defer cletest.cleanup()
149+
go cletest.createAndRunFakeController("baz1", "default", "baz", "1.20.0", "1.20.0", tc.preferredStrategy)
150+
go cletest.createAndRunFakeController("baz2", "default", "baz", "1.20.0", "1.19.0", tc.preferredStrategy)
151+
go cletest.createAndRunFakeController("baz3", "default", "baz", "1.19.0", "1.19.0", tc.preferredStrategy)
152+
go cletest.createAndRunFakeController("baz4", "default", "baz", "1.2.0", "1.19.0", tc.preferredStrategy)
153+
go cletest.createAndRunFakeController("baz5", "default", "baz", "1.20.0", "1.19.0", tc.preferredStrategy)
154+
cletest.pollForLease(ctx, "baz", "default", tc.expectedHolderIdentity)
155+
})
156+
}
157+
}
158+
159+
func TestMultipleLeaseCandidateUsingThirdPartyStrategy(t *testing.T) {
160+
tests := []struct {
161+
name string
162+
preferredStrategy v1.CoordinatedLeaseStrategy
163+
expectedHolderIdentity *string
164+
}{
165+
{
166+
name: "third party strategy, no holder identity",
167+
preferredStrategy: v1.CoordinatedLeaseStrategy("foo.com/bar"),
168+
// Because a third-party CoordinatedLeaseStrategy is in use,
169+
// the coordinated leader election controller doesn't manage
170+
// the lease's leader election and does not set the holder identity,
171+
// and leave it to some other controller. The lease will be created
172+
// with the strategy set but holderIdentity set to nil.
173+
expectedHolderIdentity: nil,
174+
},
175+
}
176+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.CoordinatedLeaderElection, true)
177+
flags := []string{fmt.Sprintf("--runtime-config=%s=true", v1alpha2.SchemeGroupVersion)}
178+
179+
server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), flags, framework.SharedEtcd())
180+
if err != nil {
181+
t.Fatal(err)
182+
}
183+
defer server.TearDownFn()
184+
config := server.ClientConfig
185+
ctx, cancel := context.WithCancel(context.Background())
186+
defer cancel()
187+
188+
for _, tc := range tests {
189+
t.Run(tc.name, func(t *testing.T) {
190+
cletest := setupCLE(config, ctx, cancel, t)
191+
defer cletest.cleanup()
192+
go cletest.createAndRunFakeController("baz1", "default", "baz", "1.20.0", "1.20.0", tc.preferredStrategy)
193+
go cletest.createAndRunFakeController("baz2", "default", "baz", "1.20.0", "1.19.0", tc.preferredStrategy)
194+
go cletest.createAndRunFakeController("baz3", "default", "baz", "1.19.0", "1.19.0", tc.preferredStrategy)
195+
go cletest.createAndRunFakeController("baz4", "default", "baz", "1.2.0", "1.19.0", tc.preferredStrategy)
196+
go cletest.createAndRunFakeController("baz5", "default", "baz", "1.20.0", "1.19.0", tc.preferredStrategy)
197+
cletest.pollForLease(ctx, "baz", "default", tc.expectedHolderIdentity)
198+
})
199+
}
84200
}
85201

86202
func TestLeaseSwapIfBetterAvailable(t *testing.T) {
@@ -98,10 +214,10 @@ func TestLeaseSwapIfBetterAvailable(t *testing.T) {
98214
cletest := setupCLE(config, ctx, cancel, t)
99215
defer cletest.cleanup()
100216

101-
go cletest.createAndRunFakeController("bar1", "default", "bar", "1.20.0", "1.20.0")
102-
cletest.pollForLease(ctx, "bar", "default", "bar1")
103-
go cletest.createAndRunFakeController("bar2", "default", "bar", "1.19.0", "1.19.0")
104-
cletest.pollForLease(ctx, "bar", "default", "bar2")
217+
go cletest.createAndRunFakeController("bar1", "default", "bar", "1.20.0", "1.20.0", v1.OldestEmulationVersion)
218+
cletest.pollForLease(ctx, "bar", "default", ptr.To("bar1"))
219+
go cletest.createAndRunFakeController("bar2", "default", "bar", "1.19.0", "1.19.0", v1.OldestEmulationVersion)
220+
cletest.pollForLease(ctx, "bar", "default", ptr.To("bar2"))
105221
}
106222

107223
// TestUpgradeSkew tests that a legacy client and a CLE aware client operating on the same lease do not cause errors
@@ -122,12 +238,12 @@ func TestUpgradeSkew(t *testing.T) {
122238
defer cletest.cleanup()
123239

124240
go cletest.createAndRunFakeLegacyController("foo1-130", "default", "foobar")
125-
cletest.pollForLease(ctx, "foobar", "default", "foo1-130")
126-
go cletest.createAndRunFakeController("foo1-131", "default", "foobar", "1.31.0", "1.31.0")
241+
cletest.pollForLease(ctx, "foobar", "default", ptr.To("foo1-130"))
242+
go cletest.createAndRunFakeController("foo1-131", "default", "foobar", "1.31.0", "1.31.0", v1.OldestEmulationVersion)
127243
// running a new controller should not kick off old leader
128-
cletest.pollForLease(ctx, "foobar", "default", "foo1-130")
244+
cletest.pollForLease(ctx, "foobar", "default", ptr.To("foo1-130"))
129245
cletest.cancelController("foo1-130", "default")
130-
cletest.pollForLease(ctx, "foobar", "default", "foo1-131")
246+
cletest.pollForLease(ctx, "foobar", "default", ptr.To("foo1-131"))
131247
}
132248

133249
func TestLeaseCandidateCleanup(t *testing.T) {
@@ -215,15 +331,15 @@ func (t *cleTest) createAndRunFakeLegacyController(name string, namespace string
215331
})
216332

217333
}
218-
func (t *cleTest) createAndRunFakeController(name string, namespace string, targetLease string, binaryVersion string, compatibilityVersion string) {
334+
func (t *cleTest) createAndRunFakeController(name string, namespace string, targetLease string, binaryVersion string, compatibilityVersion string, preferredStrategy v1.CoordinatedLeaseStrategy) {
219335
identityLease, _, err := leaderelection.NewCandidate(
220336
t.clientset,
221337
namespace,
222338
name,
223339
targetLease,
224340
binaryVersion,
225341
compatibilityVersion,
226-
v1.OldestEmulationVersion,
342+
preferredStrategy,
227343
)
228344
if err != nil {
229345
t.t.Error(err)
@@ -286,14 +402,17 @@ func leaderElectAndRun(ctx context.Context, kubeconfig *rest.Config, lockIdentit
286402
})
287403
}
288404

289-
func (t *cleTest) pollForLease(ctx context.Context, name, namespace, holder string) {
405+
func (t *cleTest) pollForLease(ctx context.Context, name, namespace string, holder *string) {
290406
err := wait.PollUntilContextTimeout(ctx, 1000*time.Millisecond, 25*time.Second, true, func(ctx context.Context) (done bool, err error) {
291407
lease, err := t.clientset.CoordinationV1().Leases(namespace).Get(ctx, name, metav1.GetOptions{})
292408
if err != nil {
293409
fmt.Println(err)
294410
return false, nil
295411
}
296-
return lease.Spec.HolderIdentity != nil && *lease.Spec.HolderIdentity == holder, nil
412+
if holder == nil {
413+
return lease.Spec.HolderIdentity == nil, nil
414+
}
415+
return lease.Spec.HolderIdentity != nil && *lease.Spec.HolderIdentity == *holder, nil
297416
})
298417
if err != nil {
299418
t.t.Fatalf("timeout awiting for Lease %s %s err: %v", name, namespace, err)

0 commit comments

Comments
 (0)