Skip to content

Commit 48c65d1

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#128576 from bart0sh/PR166-refactor-kubelet-stop-and-restart
e2e_node: refactor Kubelet stopping and restarting
2 parents 0994499 + 3aa95da commit 48c65d1

14 files changed

+104
-221
lines changed

‎test/e2e_node/container_lifecycle_test.go‎

Lines changed: 14 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1234,17 +1234,14 @@ var _ = SIGDescribe(framework.WithSerial(), "Containers Lifecycle", func() {
12341234
podSandboxID := sandboxes[0].Id
12351235

12361236
ginkgo.By("Stopping the kubelet")
1237-
restartKubelet := stopKubelet()
1238-
gomega.Eventually(ctx, func() bool {
1239-
return kubeletHealthCheck(kubeletHealthCheckURL)
1240-
}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeFalseBecause("kubelet was expected to be stopped but it is still running"))
1237+
restartKubelet := mustStopKubelet(ctx, f)
12411238

12421239
ginkgo.By("Stopping the pod sandbox to simulate the node reboot")
12431240
err = rs.StopPodSandbox(ctx, podSandboxID)
12441241
framework.ExpectNoError(err)
12451242

12461243
ginkgo.By("Restarting the kubelet")
1247-
restartKubelet()
1244+
restartKubelet(ctx)
12481245
gomega.Eventually(ctx, func() bool {
12491246
return kubeletHealthCheck(kubeletHealthCheckURL)
12501247
}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeTrueBecause("kubelet was expected to be healthy"))
@@ -1361,14 +1358,10 @@ var _ = SIGDescribe(framework.WithSerial(), "Containers Lifecycle", func() {
13611358

13621359
ginkgo.It("should not restart any completed init container after the kubelet restart", func(ctx context.Context) {
13631360
ginkgo.By("stopping the kubelet")
1364-
startKubelet := stopKubelet()
1365-
// wait until the kubelet health check will fail
1366-
gomega.Eventually(ctx, func() bool {
1367-
return kubeletHealthCheck(kubeletHealthCheckURL)
1368-
}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeFalseBecause("kubelet should be stopped"))
1361+
restartKubelet := mustStopKubelet(ctx, f)
13691362

13701363
ginkgo.By("restarting the kubelet")
1371-
startKubelet()
1364+
restartKubelet(ctx)
13721365
// wait until the kubelet health check will succeed
13731366
gomega.Eventually(ctx, func() bool {
13741367
return kubeletHealthCheck(kubeletHealthCheckURL)
@@ -1409,11 +1402,7 @@ var _ = SIGDescribe(framework.WithSerial(), "Containers Lifecycle", func() {
14091402

14101403
ginkgo.It("should not restart any completed init container, even after the completed init container statuses have been removed and the kubelet restarted", func(ctx context.Context) {
14111404
ginkgo.By("stopping the kubelet")
1412-
startKubelet := stopKubelet()
1413-
// wait until the kubelet health check will fail
1414-
gomega.Eventually(ctx, func() bool {
1415-
return kubeletHealthCheck(kubeletHealthCheckURL)
1416-
}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeFalseBecause("kubelet should be stopped"))
1405+
restartKubelet := mustStopKubelet(ctx, f)
14171406

14181407
ginkgo.By("removing the completed init container statuses from the container runtime")
14191408
rs, _, err := getCRIClient()
@@ -1437,7 +1426,7 @@ var _ = SIGDescribe(framework.WithSerial(), "Containers Lifecycle", func() {
14371426
}
14381427

14391428
ginkgo.By("restarting the kubelet")
1440-
startKubelet()
1429+
restartKubelet(ctx)
14411430
// wait until the kubelet health check will succeed
14421431
gomega.Eventually(ctx, func() bool {
14431432
return kubeletHealthCheck(kubeletHealthCheckURL)
@@ -1544,18 +1533,10 @@ var _ = SIGDescribe(framework.WithSerial(), "Containers Lifecycle", func() {
15441533

15451534
ginkgo.It("should not restart any completed init container after the kubelet restart", func(ctx context.Context) {
15461535
ginkgo.By("stopping the kubelet")
1547-
startKubelet := stopKubelet()
1548-
// wait until the kubelet health check will fail
1549-
gomega.Eventually(ctx, func() bool {
1550-
return kubeletHealthCheck(kubeletHealthCheckURL)
1551-
}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeFalseBecause("kubelet should be stopped"))
1536+
restartKubelet := mustStopKubelet(ctx, f)
15521537

15531538
ginkgo.By("restarting the kubelet")
1554-
startKubelet()
1555-
// wait until the kubelet health check will succeed
1556-
gomega.Eventually(ctx, func() bool {
1557-
return kubeletHealthCheck(kubeletHealthCheckURL)
1558-
}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeTrueBecause("kubelet should be restarted"))
1539+
restartKubelet(ctx)
15591540

15601541
ginkgo.By("ensuring that no completed init container is restarted")
15611542
gomega.Consistently(ctx, func() bool {
@@ -1588,11 +1569,7 @@ var _ = SIGDescribe(framework.WithSerial(), "Containers Lifecycle", func() {
15881569

15891570
ginkgo.It("should not restart any completed init container, even after the completed init container statuses have been removed and the kubelet restarted", func(ctx context.Context) {
15901571
ginkgo.By("stopping the kubelet")
1591-
startKubelet := stopKubelet()
1592-
// wait until the kubelet health check will fail
1593-
gomega.Eventually(ctx, func() bool {
1594-
return kubeletHealthCheck(kubeletHealthCheckURL)
1595-
}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeFalseBecause("kubelet should be stopped"))
1572+
restartKubelet := mustStopKubelet(ctx, f)
15961573

15971574
ginkgo.By("removing the completed init container statuses from the container runtime")
15981575
rs, _, err := getCRIClient()
@@ -1616,11 +1593,7 @@ var _ = SIGDescribe(framework.WithSerial(), "Containers Lifecycle", func() {
16161593
}
16171594

16181595
ginkgo.By("restarting the kubelet")
1619-
startKubelet()
1620-
// wait until the kubelet health check will succeed
1621-
gomega.Eventually(ctx, func() bool {
1622-
return kubeletHealthCheck(kubeletHealthCheckURL)
1623-
}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeTrueBecause("kubelet should be restarted"))
1596+
restartKubelet(ctx)
16241597

16251598
ginkgo.By("ensuring that no completed init container is restarted")
16261599
gomega.Consistently(ctx, func() bool {
@@ -5517,17 +5490,14 @@ var _ = SIGDescribe(nodefeature.SidecarContainers, framework.WithSerial(), "Cont
55175490
podSandboxID := sandboxes[0].Id
55185491

55195492
ginkgo.By("Stopping the kubelet")
5520-
restartKubelet := stopKubelet()
5521-
gomega.Eventually(ctx, func() bool {
5522-
return kubeletHealthCheck(kubeletHealthCheckURL)
5523-
}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeFalseBecause("expected kubelet would have been stopped but it is still running"))
5493+
restartKubelet := mustStopKubelet(ctx, f)
55245494

55255495
ginkgo.By("Stopping the pod sandbox to simulate the node reboot")
55265496
err = rs.StopPodSandbox(ctx, podSandboxID)
55275497
framework.ExpectNoError(err)
55285498

55295499
ginkgo.By("Restarting the kubelet")
5530-
restartKubelet()
5500+
restartKubelet(ctx)
55315501
gomega.Eventually(ctx, func() bool {
55325502
return kubeletHealthCheck(kubeletHealthCheckURL)
55335503
}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeTrueBecause("kubelet was expected to be healthy"))
@@ -5662,10 +5632,7 @@ var _ = SIGDescribe(nodefeature.SidecarContainers, framework.WithSerial(), "Cont
56625632
podSandboxID := sandboxes[0].Id
56635633

56645634
ginkgo.By("Stopping the kubelet")
5665-
restartKubelet := stopKubelet()
5666-
gomega.Eventually(ctx, func() bool {
5667-
return kubeletHealthCheck(kubeletHealthCheckURL)
5668-
}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeFalseBecause("expected kubelet would have been stopped but it is still running"))
5635+
restartKubelet := mustStopKubelet(ctx, f)
56695636

56705637
if nodeReboot {
56715638
ginkgo.By("Stopping the pod sandbox to simulate the node reboot")
@@ -5674,7 +5641,7 @@ var _ = SIGDescribe(nodefeature.SidecarContainers, framework.WithSerial(), "Cont
56745641
}
56755642

56765643
ginkgo.By("Restarting the kubelet")
5677-
restartKubelet()
5644+
restartKubelet(ctx)
56785645
gomega.Eventually(ctx, func() bool {
56795646
return kubeletHealthCheck(kubeletHealthCheckURL)
56805647
}, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeTrueBecause("kubelet was expected to be healthy"))

‎test/e2e_node/device_manager_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ var _ = SIGDescribe("Device Manager", framework.WithSerial(), nodefeature.Device
194194
framework.Logf("pod %s/%s running", testPod.Namespace, testPod.Name)
195195

196196
ginkgo.By("stopping the kubelet")
197-
startKubelet := stopKubelet()
197+
restartKubelet := mustStopKubelet(ctx, f)
198198

199199
ginkgo.By("stopping all the local containers - using CRI")
200200
rs, _, err := getCRIClient()
@@ -210,7 +210,7 @@ var _ = SIGDescribe("Device Manager", framework.WithSerial(), nodefeature.Device
210210
}
211211

212212
ginkgo.By("restarting the kubelet")
213-
startKubelet()
213+
restartKubelet(ctx)
214214

215215
ginkgo.By("waiting for the kubelet to be ready again")
216216
// Wait for the Kubelet to be ready.

‎test/e2e_node/device_plugin_test.go‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
211211
e2epod.NewPodClient(f).DeleteSync(ctx, p.Name, metav1.DeleteOptions{}, 2*time.Minute)
212212
}
213213

214-
restartKubelet(true)
214+
restartKubelet(ctx, true)
215215

216216
ginkgo.By("Waiting for devices to become unavailable on the local node")
217217
gomega.Eventually(ctx, func(ctx context.Context) bool {
@@ -365,7 +365,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
365365
framework.Logf("testing pod: pre-restart UID=%s namespace=%s name=%s ready=%v", pod1.UID, pod1.Namespace, pod1.Name, podutils.IsPodReady(pod1))
366366

367367
ginkgo.By("Restarting Kubelet")
368-
restartKubelet(true)
368+
restartKubelet(ctx, true)
369369

370370
ginkgo.By("Wait for node to be ready again")
371371
e2enode.WaitForAllNodesSchedulable(ctx, f.ClientSet, 5*time.Minute)
@@ -430,7 +430,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
430430
gomega.Expect(devIDRestart1).To(gomega.Equal(devID1))
431431

432432
ginkgo.By("Restarting Kubelet")
433-
restartKubelet(true)
433+
restartKubelet(ctx, true)
434434

435435
ginkgo.By("Wait for node to be ready again")
436436
e2enode.WaitForAllNodesSchedulable(ctx, f.ClientSet, 5*time.Minute)
@@ -499,7 +499,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
499499
gomega.Eventually(getNodeResourceValues, devicePluginGracefulTimeout, f.Timeouts.Poll).WithContext(ctx).WithArguments(SampleDeviceResourceName).Should(gomega.Equal(ResourceValue{Allocatable: 0, Capacity: int(expectedSampleDevsAmount)}))
500500

501501
ginkgo.By("Restarting Kubelet")
502-
restartKubelet(true)
502+
restartKubelet(ctx, true)
503503

504504
ginkgo.By("Wait for node to be ready again")
505505
gomega.Expect(e2enode.WaitForAllNodesSchedulable(ctx, f.ClientSet, 5*time.Minute)).To(gomega.Succeed())
@@ -587,7 +587,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
587587
framework.ExpectNoError(err)
588588

589589
ginkgo.By("Restarting Kubelet")
590-
restartKubelet(true)
590+
restartKubelet(ctx, true)
591591

592592
ginkgo.By("Wait for node to be ready again")
593593
e2enode.WaitForAllNodesSchedulable(ctx, f.ClientSet, 5*time.Minute)
@@ -652,7 +652,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
652652
framework.ExpectNoError(err)
653653

654654
ginkgo.By("stopping the kubelet")
655-
startKubelet := stopKubelet()
655+
restartKubelet := mustStopKubelet(ctx, f)
656656

657657
// wait until the kubelet health check will fail
658658
gomega.Eventually(ctx, func() bool {
@@ -666,7 +666,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
666666
deletePodSyncByName(ctx, f, pod.Name)
667667

668668
framework.Logf("Starting the kubelet")
669-
startKubelet()
669+
restartKubelet(ctx)
670670

671671
// wait until the kubelet health check will succeed
672672
gomega.Eventually(ctx, func() bool {
@@ -947,7 +947,7 @@ func testDevicePluginNodeReboot(f *framework.Framework, pluginSockDir string) {
947947
framework.ExpectNoError(err)
948948

949949
ginkgo.By("stopping the kubelet")
950-
startKubelet := stopKubelet()
950+
restartKubelet := mustStopKubelet(ctx, f)
951951

952952
ginkgo.By("stopping all the local containers - using CRI")
953953
rs, _, err := getCRIClient()
@@ -963,7 +963,7 @@ func testDevicePluginNodeReboot(f *framework.Framework, pluginSockDir string) {
963963
}
964964

965965
ginkgo.By("restarting the kubelet")
966-
startKubelet()
966+
restartKubelet(ctx)
967967

968968
ginkgo.By("Wait for node to be ready again")
969969
e2enode.WaitForAllNodesSchedulable(ctx, f.ClientSet, 5*time.Minute)

‎test/e2e_node/dra_test.go‎

Lines changed: 21 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
107107
}
108108

109109
ginkgo.By("restarting Kubelet")
110-
restartKubelet(true)
110+
restartKubelet(ctx, true)
111111

112112
ginkgo.By("wait for Kubelet plugin re-registration")
113113
gomega.Eventually(getNewCalls).WithTimeout(pluginRegistrationTimeout).Should(testdriver.BeRegistered)
@@ -129,16 +129,15 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
129129

130130
// Stop Kubelet
131131
ginkgo.By("stop kubelet")
132-
startKubelet := stopKubelet()
132+
restartKubelet := mustStopKubelet(ctx, f)
133133
pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{driverName})
134134
// Pod must be in pending state
135135
err := e2epod.WaitForPodCondition(ctx, f.ClientSet, f.Namespace.Name, pod.Name, "Pending", framework.PodStartShortTimeout, func(pod *v1.Pod) (bool, error) {
136136
return pod.Status.Phase == v1.PodPending, nil
137137
})
138138
framework.ExpectNoError(err)
139-
// Start Kubelet
140139
ginkgo.By("restart kubelet")
141-
startKubelet()
140+
restartKubelet(ctx)
142141
// Pod should succeed
143142
err = e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, pod.Name, f.Namespace.Name, framework.PodStartShortTimeout)
144143
framework.ExpectNoError(err)
@@ -227,12 +226,12 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
227226
gomega.Eventually(kubeletPlugin.GetGRPCCalls).WithTimeout(retryTestTimeout).Should(testdriver.NodePrepareResourcesFailed)
228227

229228
ginkgo.By("stop Kubelet")
230-
startKubelet := stopKubelet()
229+
restartKubelet := mustStopKubelet(ctx, f)
231230

232231
unsetNodePrepareResourcesFailureMode()
233232

234-
ginkgo.By("start Kubelet")
235-
startKubelet()
233+
ginkgo.By("restart Kubelet")
234+
restartKubelet(ctx)
236235

237236
ginkgo.By("wait for NodePrepareResources call to succeed")
238237
gomega.Eventually(kubeletPlugin.GetGRPCCalls).WithTimeout(retryTestTimeout).Should(testdriver.NodePrepareResourcesSucceeded)
@@ -254,12 +253,12 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
254253
gomega.Eventually(kubeletPlugin.GetGRPCCalls).WithTimeout(retryTestTimeout).Should(testdriver.NodeUnprepareResourcesFailed)
255254

256255
ginkgo.By("stop Kubelet")
257-
startKubelet := stopKubelet()
256+
restartKubelet := mustStopKubelet(ctx, f)
258257

259258
unsetNodeUnprepareResourcesFailureMode()
260259

261-
ginkgo.By("start Kubelet")
262-
startKubelet()
260+
ginkgo.By("restart Kubelet")
261+
restartKubelet(ctx)
263262

264263
ginkgo.By("wait for NodeUnprepareResources call to succeed")
265264
gomega.Eventually(kubeletPlugin.GetGRPCCalls).WithTimeout(retryTestTimeout).Should(testdriver.NodeUnprepareResourcesSucceeded)
@@ -313,7 +312,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
313312
gomega.Eventually(kubeletPlugin.GetGRPCCalls).WithTimeout(retryTestTimeout).Should(testdriver.NodeUnprepareResourcesFailed)
314313

315314
ginkgo.By("restart Kubelet")
316-
stopKubelet()()
315+
restartKubelet(ctx, true)
317316

318317
ginkgo.By("wait for NodeUnprepareResources call to fail")
319318
gomega.Eventually(kubeletPlugin.GetGRPCCalls).WithTimeout(retryTestTimeout).Should(testdriver.NodeUnprepareResourcesFailed)
@@ -337,15 +336,15 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
337336
framework.ExpectNoError(err)
338337

339338
ginkgo.By("stop Kubelet")
340-
startKubelet := stopKubelet()
339+
restartKubelet := mustStopKubelet(ctx, f)
341340

342341
ginkgo.By("delete pod")
343342
e2epod.DeletePodOrFail(ctx, f.ClientSet, f.Namespace.Name, pod.Name)
344343

345344
unblockNodePrepareResources()
346345

347-
ginkgo.By("start Kubelet")
348-
startKubelet()
346+
ginkgo.By("restart Kubelet")
347+
restartKubelet(ctx)
349348

350349
calls := kubeletPlugin.CountCalls("/NodePrepareResources")
351350
ginkgo.By("make sure NodePrepareResources is not called again")
@@ -447,7 +446,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
447446
framework.ExpectNoError(err)
448447

449448
ginkgo.By("restart Kubelet")
450-
restartKubelet(true)
449+
restartKubelet(ctx, true)
451450

452451
unblockNodePrepareResources()
453452

@@ -472,7 +471,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
472471
gomega.Eventually(kubeletPlugin2.GetGRPCCalls).WithTimeout(retryTestTimeout).Should(testdriver.NodePrepareResourcesSucceeded)
473472

474473
ginkgo.By("restart Kubelet")
475-
restartKubelet(true)
474+
restartKubelet(ctx, true)
476475

477476
unblockNodeUnprepareResources()
478477

@@ -500,10 +499,10 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
500499

501500
f.It("must be removed on kubelet startup", f.WithDisruptive(), func(ctx context.Context) {
502501
ginkgo.By("stop kubelet")
503-
startKubelet := stopKubelet()
502+
restartKubelet := mustStopKubelet(ctx, f)
504503
ginkgo.DeferCleanup(func() {
505-
if startKubelet != nil {
506-
startKubelet()
504+
if restartKubelet != nil {
505+
restartKubelet(ctx)
507506
}
508507
})
509508

@@ -518,9 +517,9 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
518517

519518
gomega.Consistently(ctx, listResources).WithTimeout(5*time.Second).Should(matchAll, "ResourceSlices without kubelet")
520519

521-
ginkgo.By("start kubelet")
522-
startKubelet()
523-
startKubelet = nil
520+
ginkgo.By("restart kubelet")
521+
restartKubelet(ctx)
522+
restartKubelet = nil
524523

525524
ginkgo.By("wait for exactly the node's ResourceSlice to get deleted")
526525
gomega.Eventually(ctx, listResources).Should(matchOtherNode, "ResourceSlices with kubelet")

0 commit comments

Comments
 (0)