Skip to content

Commit 597eb78

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#94160 from matthyx/startupprobe-ga
Remove StartupProbe featuregate and related logic
2 parents 3ed839c + ea14585 commit 597eb78

File tree

12 files changed

+164
-268
lines changed

12 files changed

+164
-268
lines changed

‎api/openapi-spec/swagger.json‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/api/pod/util.go‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -406,14 +406,6 @@ func dropDisabledFields(
406406
})
407407
}
408408

409-
if !utilfeature.DefaultFeatureGate.Enabled(features.StartupProbe) && !startupProbeInUse(oldPodSpec) {
410-
// drop startupProbe from all containers if the feature is disabled
411-
VisitContainers(podSpec, AllContainers, func(c *api.Container, containerType ContainerType) bool {
412-
c.StartupProbe = nil
413-
return true
414-
})
415-
}
416-
417409
dropDisabledRunAsGroupField(podSpec, oldPodSpec)
418410

419411
dropDisabledFSGroupFields(podSpec, oldPodSpec)
@@ -696,24 +688,6 @@ func subpathExprInUse(podSpec *api.PodSpec) bool {
696688
return inUse
697689
}
698690

699-
// startupProbeInUse returns true if the pod spec is non-nil and has a container that has a startupProbe defined
700-
func startupProbeInUse(podSpec *api.PodSpec) bool {
701-
if podSpec == nil {
702-
return false
703-
}
704-
705-
var inUse bool
706-
VisitContainers(podSpec, AllContainers, func(c *api.Container, containerType ContainerType) bool {
707-
if c.StartupProbe != nil {
708-
inUse = true
709-
return false
710-
}
711-
return true
712-
})
713-
714-
return inUse
715-
}
716-
717691
// csiInUse returns true if any pod's spec include inline CSI volumes.
718692
func csiInUse(podSpec *api.PodSpec) bool {
719693
if podSpec == nil {

‎pkg/features/kube_features.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -555,6 +555,7 @@ const (
555555
// owner: @matthyx
556556
// alpha: v1.16
557557
// beta: v1.18
558+
// GA: v1.20
558559
//
559560
// Enables the startupProbe in kubelet worker.
560561
StartupProbe featuregate.Feature = "StartupProbe"
@@ -729,7 +730,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
729730
EndpointSliceProxying: {Default: true, PreRelease: featuregate.Beta},
730731
WindowsEndpointSliceProxying: {Default: false, PreRelease: featuregate.Alpha},
731732
EvenPodsSpread: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.21
732-
StartupProbe: {Default: true, PreRelease: featuregate.Beta},
733+
StartupProbe: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.21
733734
AllowInsecureBackendProxy: {Default: true, PreRelease: featuregate.Beta},
734735
PodDisruptionBudget: {Default: true, PreRelease: featuregate.Beta},
735736
ServiceTopology: {Default: false, PreRelease: featuregate.Alpha},

‎pkg/kubelet/prober/BUILD‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ go_library(
1616
importpath = "k8s.io/kubernetes/pkg/kubelet/prober",
1717
deps = [
1818
"//pkg/api/v1/pod:go_default_library",
19-
"//pkg/features:go_default_library",
2019
"//pkg/kubelet/container:go_default_library",
2120
"//pkg/kubelet/events:go_default_library",
2221
"//pkg/kubelet/prober/results:go_default_library",
@@ -32,7 +31,6 @@ go_library(
3231
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
3332
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
3433
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
35-
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
3634
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
3735
"//staging/src/k8s.io/component-base/metrics:go_default_library",
3836
"//vendor/k8s.io/klog/v2:go_default_library",
@@ -50,7 +48,6 @@ go_test(
5048
],
5149
embed = [":go_default_library"],
5250
deps = [
53-
"//pkg/features:go_default_library",
5451
"//pkg/kubelet/container:go_default_library",
5552
"//pkg/kubelet/container/testing:go_default_library",
5653
"//pkg/kubelet/pod:go_default_library",
@@ -67,10 +64,8 @@ go_test(
6764
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
6865
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
6966
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
70-
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
7167
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
7268
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
73-
"//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
7469
"//vendor/k8s.io/klog/v2:go_default_library",
7570
"//vendor/k8s.io/utils/exec:go_default_library",
7671
],

‎pkg/kubelet/prober/prober_manager.go‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,9 @@ import (
2323
"k8s.io/apimachinery/pkg/types"
2424
"k8s.io/apimachinery/pkg/util/sets"
2525
"k8s.io/apimachinery/pkg/util/wait"
26-
utilfeature "k8s.io/apiserver/pkg/util/feature"
2726
"k8s.io/client-go/tools/record"
2827
"k8s.io/component-base/metrics"
2928
"k8s.io/klog/v2"
30-
"k8s.io/kubernetes/pkg/features"
3129
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
3230
"k8s.io/kubernetes/pkg/kubelet/prober/results"
3331
"k8s.io/kubernetes/pkg/kubelet/status"
@@ -168,7 +166,7 @@ func (m *manager) AddPod(pod *v1.Pod) {
168166
for _, c := range pod.Spec.Containers {
169167
key.containerName = c.Name
170168

171-
if c.StartupProbe != nil && utilfeature.DefaultFeatureGate.Enabled(features.StartupProbe) {
169+
if c.StartupProbe != nil {
172170
key.probeType = startup
173171
if _, ok := m.workers[key]; ok {
174172
klog.Errorf("Startup probe already exists! %v - %v",
@@ -238,9 +236,6 @@ func (m *manager) UpdatePodStatus(podUID types.UID, podStatus *v1.PodStatus) {
238236
var started bool
239237
if c.State.Running == nil {
240238
started = false
241-
} else if !utilfeature.DefaultFeatureGate.Enabled(features.StartupProbe) {
242-
// the container is running, assume it is started if the StartupProbe feature is disabled
243-
started = true
244239
} else if result, ok := m.startupManager.Get(kubecontainer.ParseContainerID(c.ContainerID)); ok {
245240
started = result == results.Success
246241
} else {

‎pkg/kubelet/prober/prober_manager_test.go‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,7 @@ import (
2828
"k8s.io/apimachinery/pkg/util/runtime"
2929
"k8s.io/apimachinery/pkg/util/sets"
3030
"k8s.io/apimachinery/pkg/util/wait"
31-
utilfeature "k8s.io/apiserver/pkg/util/feature"
32-
featuregatetesting "k8s.io/component-base/featuregate/testing"
3331
"k8s.io/klog/v2"
34-
"k8s.io/kubernetes/pkg/features"
3532
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
3633
"k8s.io/kubernetes/pkg/kubelet/prober/results"
3734
"k8s.io/kubernetes/pkg/probe"
@@ -92,7 +89,6 @@ func TestAddRemovePods(t *testing.T) {
9289
}
9390

9491
m := newTestManager()
95-
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StartupProbe, true)()
9692
defer cleanup(t, m)
9793
if err := expectProbes(m, nil); err != nil {
9894
t.Error(err)
@@ -139,7 +135,6 @@ func TestAddRemovePods(t *testing.T) {
139135

140136
func TestCleanupPods(t *testing.T) {
141137
m := newTestManager()
142-
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StartupProbe, true)()
143138
defer cleanup(t, m)
144139
podToCleanup := v1.Pod{
145140
ObjectMeta: metav1.ObjectMeta{
@@ -202,7 +197,6 @@ func TestCleanupPods(t *testing.T) {
202197

203198
func TestCleanupRepeated(t *testing.T) {
204199
m := newTestManager()
205-
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StartupProbe, true)()
206200
defer cleanup(t, m)
207201
podTemplate := v1.Pod{
208202
Spec: v1.PodSpec{

‎staging/src/k8s.io/api/core/v1/generated.proto‎

Lines changed: 0 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎staging/src/k8s.io/api/core/v1/types.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2287,7 +2287,6 @@ type Container struct {
22872287
// This can be used to provide different probe parameters at the beginning of a Pod's lifecycle,
22882288
// when it might take a long time to load data or warm a cache, than during steady-state operation.
22892289
// This cannot be updated.
2290-
// This is a beta feature enabled by the StartupProbe feature flag.
22912290
// More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
22922291
// +optional
22932292
StartupProbe *Probe `json:"startupProbe,omitempty" protobuf:"bytes,22,opt,name=startupProbe"`

‎staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎test/e2e/common/container_probe.go‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,144 @@ var _ = framework.KubeDescribe("Probing container", func() {
264264
framework.ExpectNoError(e2eevents.WaitTimeoutForEvent(
265265
f.ClientSet, f.Namespace.Name, expectedEvent, "0.0.0.0", framework.PodEventTimeout))
266266
})
267+
268+
/*
269+
Release: v1.16
270+
Testname: Pod startup probe restart
271+
Description: A Pod is created with a failing startup probe. The Pod MUST be killed and restarted incrementing restart count to 1, even if liveness would succeed.
272+
*/
273+
ginkgo.It("should be restarted startup probe fails", func() {
274+
cmd := []string{"/bin/sh", "-c", "sleep 600"}
275+
livenessProbe := &v1.Probe{
276+
Handler: v1.Handler{
277+
Exec: &v1.ExecAction{
278+
Command: []string{"/bin/true"},
279+
},
280+
},
281+
InitialDelaySeconds: 15,
282+
FailureThreshold: 1,
283+
}
284+
startupProbe := &v1.Probe{
285+
Handler: v1.Handler{
286+
Exec: &v1.ExecAction{
287+
Command: []string{"/bin/false"},
288+
},
289+
},
290+
InitialDelaySeconds: 15,
291+
FailureThreshold: 3,
292+
}
293+
pod := startupPodSpec(startupProbe, nil, livenessProbe, cmd)
294+
RunLivenessTest(f, pod, 1, defaultObservationTimeout)
295+
})
296+
297+
/*
298+
Release: v1.16
299+
Testname: Pod liveness probe delayed (long) by startup probe
300+
Description: A Pod is created with failing liveness and startup probes. Liveness probe MUST NOT fail until startup probe expires.
301+
*/
302+
ginkgo.It("should *not* be restarted by liveness probe because startup probe delays it", func() {
303+
cmd := []string{"/bin/sh", "-c", "sleep 600"}
304+
livenessProbe := &v1.Probe{
305+
Handler: v1.Handler{
306+
Exec: &v1.ExecAction{
307+
Command: []string{"/bin/false"},
308+
},
309+
},
310+
InitialDelaySeconds: 15,
311+
FailureThreshold: 1,
312+
}
313+
startupProbe := &v1.Probe{
314+
Handler: v1.Handler{
315+
Exec: &v1.ExecAction{
316+
Command: []string{"/bin/false"},
317+
},
318+
},
319+
InitialDelaySeconds: 15,
320+
FailureThreshold: 60,
321+
}
322+
pod := startupPodSpec(startupProbe, nil, livenessProbe, cmd)
323+
RunLivenessTest(f, pod, 0, defaultObservationTimeout)
324+
})
325+
326+
/*
327+
Release: v1.16
328+
Testname: Pod liveness probe fails after startup success
329+
Description: A Pod is created with failing liveness probe and delayed startup probe that uses 'exec' command to cat /temp/health file. The Container is started by creating /tmp/startup after 10 seconds, triggering liveness probe to fail. The Pod MUST now be killed and restarted incrementing restart count to 1.
330+
*/
331+
ginkgo.It("should be restarted by liveness probe after startup probe enables it", func() {
332+
cmd := []string{"/bin/sh", "-c", "sleep 10; echo ok >/tmp/startup; sleep 600"}
333+
livenessProbe := &v1.Probe{
334+
Handler: v1.Handler{
335+
Exec: &v1.ExecAction{
336+
Command: []string{"/bin/false"},
337+
},
338+
},
339+
InitialDelaySeconds: 15,
340+
FailureThreshold: 1,
341+
}
342+
startupProbe := &v1.Probe{
343+
Handler: v1.Handler{
344+
Exec: &v1.ExecAction{
345+
Command: []string{"cat", "/tmp/startup"},
346+
},
347+
},
348+
InitialDelaySeconds: 15,
349+
FailureThreshold: 60,
350+
}
351+
pod := startupPodSpec(startupProbe, nil, livenessProbe, cmd)
352+
RunLivenessTest(f, pod, 1, defaultObservationTimeout)
353+
})
354+
355+
/*
356+
Release: v1.16
357+
Testname: Pod readiness probe, delayed by startup probe
358+
Description: A Pod is created with startup and readiness probes. The Container is started by creating /tmp/startup after 45 seconds, delaying the ready state by this amount of time. This is similar to the "Pod readiness probe, with initial delay" test.
359+
*/
360+
ginkgo.It("should not be ready until startupProbe succeeds", func() {
361+
cmd := []string{"/bin/sh", "-c", "echo ok >/tmp/health; sleep 45; echo ok >/tmp/startup; sleep 600"}
362+
readinessProbe := &v1.Probe{
363+
Handler: v1.Handler{
364+
Exec: &v1.ExecAction{
365+
Command: []string{"cat", "/tmp/health"},
366+
},
367+
},
368+
InitialDelaySeconds: 0,
369+
}
370+
startupProbe := &v1.Probe{
371+
Handler: v1.Handler{
372+
Exec: &v1.ExecAction{
373+
Command: []string{"cat", "/tmp/startup"},
374+
},
375+
},
376+
InitialDelaySeconds: 0,
377+
FailureThreshold: 60,
378+
}
379+
p := podClient.Create(startupPodSpec(startupProbe, readinessProbe, nil, cmd))
380+
381+
p, err := podClient.Get(context.TODO(), p.Name, metav1.GetOptions{})
382+
framework.ExpectNoError(err)
383+
384+
e2epod.WaitTimeoutForPodReadyInNamespace(f.ClientSet, p.Name, f.Namespace.Name, framework.PodStartTimeout)
385+
386+
p, err = podClient.Get(context.TODO(), p.Name, metav1.GetOptions{})
387+
framework.ExpectNoError(err)
388+
389+
isReady, err := testutils.PodRunningReady(p)
390+
framework.ExpectNoError(err)
391+
framework.ExpectEqual(isReady, true, "pod should be ready")
392+
393+
// We assume the pod became ready when the container became ready. This
394+
// is true for a single container pod.
395+
readyTime, err := GetTransitionTimeForReadyCondition(p)
396+
framework.ExpectNoError(err)
397+
startedTime, err := GetContainerStartedTime(p, "busybox")
398+
framework.ExpectNoError(err)
399+
400+
framework.Logf("Container started at %v, pod became ready at %v", startedTime, readyTime)
401+
if readyTime.Sub(startedTime) < 40*time.Second {
402+
framework.Failf("Pod became ready before startupProbe succeeded")
403+
}
404+
})
267405
})
268406

269407
func GetContainerStartedTime(p *v1.Pod, containerName string) (time.Time, error) {
@@ -354,6 +492,27 @@ func livenessPodSpec(readinessProbe, livenessProbe *v1.Probe) *v1.Pod {
354492
}
355493
}
356494

495+
func startupPodSpec(startupProbe, readinessProbe, livenessProbe *v1.Probe, cmd []string) *v1.Pod {
496+
return &v1.Pod{
497+
ObjectMeta: metav1.ObjectMeta{
498+
Name: "startup-" + string(uuid.NewUUID()),
499+
Labels: map[string]string{"test": "startup"},
500+
},
501+
Spec: v1.PodSpec{
502+
Containers: []v1.Container{
503+
{
504+
Name: "busybox",
505+
Image: imageutils.GetE2EImage(imageutils.BusyBox),
506+
Command: cmd,
507+
LivenessProbe: livenessProbe,
508+
ReadinessProbe: readinessProbe,
509+
StartupProbe: startupProbe,
510+
},
511+
},
512+
},
513+
}
514+
}
515+
357516
func execHandler(cmd []string) v1.Handler {
358517
return v1.Handler{
359518
Exec: &v1.ExecAction{

0 commit comments

Comments
 (0)