Skip to content

Commit aa1b2d6

Browse files
author
Han Kang
committed
add /livez as a liveness endpoint for kube-apiserver
go fmt make func private refactor config_test Two primary refactorings: 1. config test checkPath method is now each a distinct test run (which makes it easier to see what is actually failing) 2. TestNewWithDelegate's root path check now parses the json output and does a comparison against a list of expected paths (no more whitespace and ordering issues when updating this test, yay). go fmt modify and simplify existing integration test for readyz/livez simplify integration test set default rbac policy rules for livez rename a few functions and the entrypoint command line argument (and etcetera) simplify interface for installing readyz and livez and make auto-register completion a bootstrapped check untangle some of the nested functions, restructure the code
1 parent 2af52db commit aa1b2d6

File tree

15 files changed

+210
-178
lines changed

15 files changed

+210
-178
lines changed

cmd/kube-apiserver/app/aggregator.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ func createAggregatorServer(aggregatorConfig *aggregatorapiserver.Config, delega
153153
return nil, err
154154
}
155155

156-
err = aggregatorServer.GenericAPIServer.AddHealthChecks(
156+
err = aggregatorServer.GenericAPIServer.AddBootSequenceHealthChecks(
157157
makeAPIServiceAvailableHealthCheck(
158158
"autoregister-completion",
159159
apiServices,

cmd/kube-apiserver/app/testing/BUILD

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ go_library(
1717
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
1818
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
1919
"//staging/src/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library",
20-
"//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library",
2120
"//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
2221
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
2322
"//staging/src/k8s.io/client-go/rest:go_default_library",

cmd/kube-apiserver/app/testing/testserver.go

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ import (
3131
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3232
"k8s.io/apimachinery/pkg/util/wait"
3333
"k8s.io/apiserver/pkg/registry/generic/registry"
34-
"k8s.io/apiserver/pkg/server/healthz"
3534
"k8s.io/apiserver/pkg/storage/storagebackend"
3635
"k8s.io/client-go/kubernetes"
3736
restclient "k8s.io/client-go/rest"
@@ -46,8 +45,6 @@ type TearDownFunc func()
4645
type TestServerInstanceOptions struct {
4746
// DisableStorageCleanup Disable the automatic storage cleanup
4847
DisableStorageCleanup bool
49-
// Injected health
50-
InjectedHealthChecker healthz.HealthChecker
5148
}
5249

5350
// TestServer return values supplied by kube-test-ApiServer
@@ -151,13 +148,6 @@ func StartTestServer(t Logger, instanceOptions *TestServerInstanceOptions, custo
151148
return result, fmt.Errorf("failed to create server chain: %v", err)
152149
}
153150

154-
if instanceOptions.InjectedHealthChecker != nil {
155-
t.Logf("Adding health check with delay %v %v", s.GenericServerRunOptions.MaxStartupSequenceDuration, instanceOptions.InjectedHealthChecker.Name())
156-
if err := server.GenericAPIServer.AddDelayedHealthzChecks(s.GenericServerRunOptions.MaxStartupSequenceDuration, instanceOptions.InjectedHealthChecker); err != nil {
157-
return result, err
158-
}
159-
}
160-
161151
errCh := make(chan error)
162152
go func(stopCh <-chan struct{}) {
163153
prepared, err := server.PrepareRun()

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,8 @@ func ClusterRoles() []rbacv1.ClusterRole {
197197
ObjectMeta: metav1.ObjectMeta{Name: "system:discovery"},
198198
Rules: []rbacv1.PolicyRule{
199199
rbacv1helpers.NewRule("get").URLs(
200-
"/readyz", "/healthz", "/version", "/version/",
200+
"/livez", "/readyz", "/healthz",
201+
"/version", "/version/",
201202
"/openapi", "/openapi/*",
202203
"/api", "/api/*",
203204
"/apis", "/apis/*",
@@ -217,7 +218,7 @@ func ClusterRoles() []rbacv1.ClusterRole {
217218
ObjectMeta: metav1.ObjectMeta{Name: "system:public-info-viewer"},
218219
Rules: []rbacv1.PolicyRule{
219220
rbacv1helpers.NewRule("get").URLs(
220-
"/readyz", "/healthz", "/version", "/version/",
221+
"/livez", "/readyz", "/healthz", "/version", "/version/",
221222
).RuleOrDie(),
222223
},
223224
},

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -548,6 +548,7 @@ items:
548548
- /apis
549549
- /apis/*
550550
- /healthz
551+
- /livez
551552
- /openapi
552553
- /openapi/*
553554
- /readyz
@@ -1185,6 +1186,7 @@ items:
11851186
rules:
11861187
- nonResourceURLs:
11871188
- /healthz
1189+
- /livez
11881190
- /readyz
11891191
- /version
11901192
- /version/

staging/src/k8s.io/apiserver/pkg/server/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ go_test(
2222
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
2323
"//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
2424
"//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library",
25+
"//staging/src/k8s.io/apimachinery/pkg/util/json:go_default_library",
2526
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
2627
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
2728
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",

staging/src/k8s.io/apiserver/pkg/server/config.go

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,8 @@ type Config struct {
139139
DiscoveryAddresses discovery.Addresses
140140
// The default set of healthz checks. There might be more added via AddHealthChecks dynamically.
141141
HealthzChecks []healthz.HealthChecker
142+
// The default set of livez checks. There might be more added via AddHealthChecks dynamically.
143+
LivezChecks []healthz.HealthChecker
142144
// The default set of readyz-only checks. There might be more added via AddReadyzChecks dynamically.
143145
ReadyzChecks []healthz.HealthChecker
144146
// LegacyAPIGroupPrefixes is used to set up URL parsing for authorization and for validating requests
@@ -165,9 +167,9 @@ type Config struct {
165167

166168
// This represents the maximum amount of time it should take for apiserver to complete its startup
167169
// sequence and become healthy. From apiserver's start time to when this amount of time has
168-
// elapsed, /healthz will assume that unfinished post-start hooks will complete successfully and
170+
// elapsed, /livez will assume that unfinished post-start hooks will complete successfully and
169171
// therefore return true.
170-
MaxStartupSequenceDuration time.Duration
172+
LivezGracePeriod time.Duration
171173
// ShutdownDelayDuration allows to block shutdown for some time, e.g. until endpoints pointing to this API server
172174
// have converged on all node. During this time, the API server keeps serving, /healthz will return 200,
173175
// but /readyz will return failure.
@@ -278,6 +280,7 @@ func NewConfig(codecs serializer.CodecFactory) *Config {
278280
DisabledPostStartHooks: sets.NewString(),
279281
HealthzChecks: append([]healthz.HealthChecker{}, defaultHealthChecks...),
280282
ReadyzChecks: append([]healthz.HealthChecker{}, defaultHealthChecks...),
283+
LivezChecks: append([]healthz.HealthChecker{}, defaultHealthChecks...),
281284
EnableIndex: true,
282285
EnableDiscovery: true,
283286
EnableProfiling: true,
@@ -286,7 +289,7 @@ func NewConfig(codecs serializer.CodecFactory) *Config {
286289
MaxMutatingRequestsInFlight: 200,
287290
RequestTimeout: time.Duration(60) * time.Second,
288291
MinRequestTimeout: 1800,
289-
MaxStartupSequenceDuration: time.Duration(0),
292+
LivezGracePeriod: time.Duration(0),
290293
ShutdownDelayDuration: time.Duration(0),
291294
// 10MB is the recommended maximum client request size in bytes
292295
// the etcd server should accept. See
@@ -509,15 +512,16 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
509512
preShutdownHooks: map[string]preShutdownHookEntry{},
510513
disabledPostStartHooks: c.DisabledPostStartHooks,
511514

512-
healthzChecks: c.HealthzChecks,
513-
readyzChecks: c.ReadyzChecks,
514-
readinessStopCh: make(chan struct{}),
515-
maxStartupSequenceDuration: c.MaxStartupSequenceDuration,
515+
healthzChecks: c.HealthzChecks,
516+
livezChecks: c.LivezChecks,
517+
readyzChecks: c.ReadyzChecks,
518+
readinessStopCh: make(chan struct{}),
519+
livezGracePeriod: c.LivezGracePeriod,
516520

517521
DiscoveryGroupManager: discovery.NewRootAPIsHandler(c.DiscoveryAddresses, c.Serializer),
518522

519523
maxRequestBodyBytes: c.MaxRequestBodyBytes,
520-
healthzClock: clock.RealClock{},
524+
livezClock: clock.RealClock{},
521525
}
522526

523527
for {
@@ -563,9 +567,7 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
563567
if skip {
564568
continue
565569
}
566-
567-
s.healthzChecks = append(s.healthzChecks, delegateCheck)
568-
s.readyzChecks = append(s.readyzChecks, delegateCheck)
570+
s.AddHealthChecks(delegateCheck)
569571
}
570572

571573
s.listedPathProvider = routes.ListedPathProviders{s.listedPathProvider, delegationTarget}

staging/src/k8s.io/apiserver/pkg/server/config_test.go

Lines changed: 82 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"reflect"
2727
"testing"
2828

29+
"k8s.io/apimachinery/pkg/util/json"
2930
"k8s.io/apimachinery/pkg/util/sets"
3031
"k8s.io/apiserver/pkg/server/healthz"
3132
"k8s.io/client-go/informers"
@@ -135,31 +136,36 @@ func TestNewWithDelegate(t *testing.T) {
135136
// Wait for the hooks to finish before checking the response
136137
<-delegatePostStartHookChan
137138
<-wrappingPostStartHookChan
138-
139-
checkPath(server.URL, http.StatusOK, `{
140-
"paths": [
141-
"/apis",
142-
"/bar",
143-
"/foo",
144-
"/healthz",
145-
"/healthz/delegate-health",
146-
"/healthz/log",
147-
"/healthz/ping",
148-
"/healthz/poststarthook/delegate-post-start-hook",
149-
"/healthz/poststarthook/generic-apiserver-start-informers",
150-
"/healthz/poststarthook/wrapping-post-start-hook",
151-
"/healthz/wrapping-health",
152-
"/metrics",
153-
"/readyz",
154-
"/readyz/delegate-health",
155-
"/readyz/log",
156-
"/readyz/ping",
157-
"/readyz/poststarthook/delegate-post-start-hook",
158-
"/readyz/poststarthook/generic-apiserver-start-informers",
159-
"/readyz/poststarthook/wrapping-post-start-hook",
160-
"/readyz/shutdown"
161-
]
162-
}`, t)
139+
expectedPaths := []string{
140+
"/apis",
141+
"/bar",
142+
"/foo",
143+
"/healthz",
144+
"/healthz/delegate-health",
145+
"/healthz/log",
146+
"/healthz/ping",
147+
"/healthz/poststarthook/delegate-post-start-hook",
148+
"/healthz/poststarthook/generic-apiserver-start-informers",
149+
"/healthz/poststarthook/wrapping-post-start-hook",
150+
"/healthz/wrapping-health",
151+
"/livez",
152+
"/livez/delegate-health",
153+
"/livez/log",
154+
"/livez/ping",
155+
"/livez/poststarthook/delegate-post-start-hook",
156+
"/livez/poststarthook/generic-apiserver-start-informers",
157+
"/livez/poststarthook/wrapping-post-start-hook",
158+
"/metrics",
159+
"/readyz",
160+
"/readyz/delegate-health",
161+
"/readyz/log",
162+
"/readyz/ping",
163+
"/readyz/poststarthook/delegate-post-start-hook",
164+
"/readyz/poststarthook/generic-apiserver-start-informers",
165+
"/readyz/poststarthook/wrapping-post-start-hook",
166+
"/readyz/shutdown",
167+
}
168+
checkExpectedPathsAtRoot(server.URL, expectedPaths, t)
163169
checkPath(server.URL+"/healthz", http.StatusInternalServerError, `[+]ping ok
164170
[+]log ok
165171
[-]wrapping-health failed: reason withheld
@@ -181,22 +187,57 @@ healthz check failed
181187
}
182188

183189
func checkPath(url string, expectedStatusCode int, expectedBody string, t *testing.T) {
184-
resp, err := http.Get(url)
185-
if err != nil {
186-
t.Fatal(err)
187-
}
188-
dump, _ := httputil.DumpResponse(resp, true)
189-
t.Log(string(dump))
190+
t.Run(url, func(t *testing.T) {
191+
resp, err := http.Get(url)
192+
if err != nil {
193+
t.Fatal(err)
194+
}
195+
dump, _ := httputil.DumpResponse(resp, true)
196+
t.Log(string(dump))
190197

191-
body, err := ioutil.ReadAll(resp.Body)
192-
if err != nil {
193-
t.Fatal(err)
194-
}
198+
body, err := ioutil.ReadAll(resp.Body)
199+
if err != nil {
200+
t.Fatal(err)
201+
}
195202

196-
if e, a := expectedBody, string(body); e != a {
197-
t.Errorf("%q expected %v, got %v", url, e, a)
198-
}
199-
if e, a := expectedStatusCode, resp.StatusCode; e != a {
200-
t.Errorf("%q expected %v, got %v", url, e, a)
201-
}
203+
if e, a := expectedBody, string(body); e != a {
204+
t.Errorf("%q expected %v, got %v", url, e, a)
205+
}
206+
if e, a := expectedStatusCode, resp.StatusCode; e != a {
207+
t.Errorf("%q expected %v, got %v", url, e, a)
208+
}
209+
})
210+
}
211+
212+
func checkExpectedPathsAtRoot(url string, expectedPaths []string, t *testing.T) {
213+
t.Run(url, func(t *testing.T) {
214+
resp, err := http.Get(url)
215+
if err != nil {
216+
t.Fatal(err)
217+
}
218+
dump, _ := httputil.DumpResponse(resp, true)
219+
t.Log(string(dump))
220+
221+
body, err := ioutil.ReadAll(resp.Body)
222+
if err != nil {
223+
t.Fatal(err)
224+
}
225+
var result map[string]interface{}
226+
json.Unmarshal(body, &result)
227+
paths, ok := result["paths"].([]interface{})
228+
if !ok {
229+
t.Errorf("paths not found")
230+
}
231+
pathset := sets.NewString()
232+
for _, p := range paths {
233+
pathset.Insert(p.(string))
234+
}
235+
expectedset := sets.NewString(expectedPaths...)
236+
for _, p := range pathset.Difference(expectedset) {
237+
t.Errorf("Got %v path, which we did not expect", p)
238+
}
239+
for _, p := range expectedset.Difference(pathset) {
240+
t.Errorf(" Expected %v path which we did not get", p)
241+
}
242+
})
202243
}

staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -146,14 +146,19 @@ type GenericAPIServer struct {
146146
preShutdownHooksCalled bool
147147

148148
// healthz checks
149-
healthzLock sync.Mutex
150-
healthzChecks []healthz.HealthChecker
151-
healthzChecksInstalled bool
152-
readyzLock sync.Mutex
153-
readyzChecks []healthz.HealthChecker
154-
readyzChecksInstalled bool
155-
maxStartupSequenceDuration time.Duration
156-
healthzClock clock.Clock
149+
healthzLock sync.Mutex
150+
healthzChecks []healthz.HealthChecker
151+
healthzChecksInstalled bool
152+
// livez checks
153+
livezLock sync.Mutex
154+
livezChecks []healthz.HealthChecker
155+
livezChecksInstalled bool
156+
// readyz checks
157+
readyzLock sync.Mutex
158+
readyzChecks []healthz.HealthChecker
159+
readyzChecksInstalled bool
160+
livezGracePeriod time.Duration
161+
livezClock clock.Clock
157162
// the readiness stop channel is used to signal that the apiserver has initiated a shutdown sequence, this
158163
// will cause readyz to return unhealthy.
159164
readinessStopCh chan struct{}
@@ -281,7 +286,12 @@ func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
281286
}
282287

283288
s.installHealthz()
284-
s.installReadyz(s.readinessStopCh)
289+
s.installLivez()
290+
err := s.addReadyzShutdownCheck(s.readinessStopCh)
291+
if err != nil {
292+
klog.Errorf("Failed to install readyz shutdown check %s", err)
293+
}
294+
s.installReadyz()
285295

286296
// Register audit backend preShutdownHook.
287297
if s.AuditBackend != nil {

0 commit comments

Comments
 (0)