Skip to content

Commit 502cb1e

Browse files
zolkinevgenymikerinjpmcjohanbrandhorst
authored
Health check implementation (feature request #890) (#1056)
* #890 health check implementation (fully working, but lacks tests) * #890 added tests for client health check implementation * #890 defer Unlock instead of calling manually * go.sum reverted to the original state * #890 switches in main.go reworked to ServeMux, backoff implementation replaced with existing module, health check function simplified * Update mod files * #890 client health check simplified and refactored * #890 test error handling, necessary comments * #890 health service initialization reqorked; minor fixes in tests * #890 test error handling fixed * #890 test timeout handling reworked * Update go/grpcweb/health_test.go Co-authored-by: Johan Brandhorst-Satzkorn <[email protected]> * Update go/grpcweb/health_test.go Co-authored-by: Johan Brandhorst-Satzkorn <[email protected]> * #890 docs regenerated * #890 proper repo name Co-authored-by: Evgeny Mikerin <[email protected]> Co-authored-by: Johan Brandhorst-Satzkorn <[email protected]>
1 parent ccbe285 commit 502cb1e

File tree

6 files changed

+312
-44
lines changed

6 files changed

+312
-44
lines changed

go.mod

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ module github.com/improbable-eng/grpc-web
33
go 1.16
44

55
require (
6+
github.com/cenkalti/backoff/v4 v4.1.1
67
github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f
78
github.com/golang/protobuf v1.4.3
89
github.com/grpc-ecosystem/go-grpc-middleware v1.2.2
@@ -17,9 +18,8 @@ require (
1718
github.com/sirupsen/logrus v1.7.0
1819
github.com/spf13/pflag v1.0.5
1920
github.com/stretchr/testify v1.7.0
20-
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
21-
golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
22-
golang.org/x/text v0.3.5 // indirect
21+
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d
22+
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect
2323
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
2424
google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506 // indirect
2525
google.golang.org/grpc v1.32.0

go.sum

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,10 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
2828
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
2929
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
3030
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
31+
github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
3132
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
33+
github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ=
34+
github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
3235
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
3336
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
3437
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -373,8 +376,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
373376
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
374377
golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
375378
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
376-
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
377-
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
379+
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI=
380+
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
378381
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
379382
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
380383
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -408,15 +411,14 @@ golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7w
408411
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
409412
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
410413
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
411-
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
412-
golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
413-
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
414+
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
415+
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA=
416+
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
414417
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
415418
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
416419
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
417-
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
418-
golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
419-
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
420+
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
421+
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
420422
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
421423
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
422424
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

go/grpcweb/DOC.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,15 @@ If you'd like to have a standalone binary, please take a look at `grpcwebproxy`.
3030

3131
## Usage
3232

33+
#### func ClientHealthCheck
34+
35+
```go
36+
func ClientHealthCheck(ctx context.Context, backendConn *grpc.ClientConn, service string, setServingStatus func(serving bool)) error
37+
```
38+
Client health check function is also part of the grpc/internal package The
39+
following code is a simplified version of client.go For more details see:
40+
https://pkg.go.dev/google.golang.org/grpc/health
41+
3342
#### func ListGRPCResources
3443

3544
```go

go/grpcweb/health.go

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package grpcweb
2+
3+
import (
4+
"context"
5+
"time"
6+
7+
backoff "github.com/cenkalti/backoff/v4"
8+
"google.golang.org/grpc"
9+
"google.golang.org/grpc/codes"
10+
healthpb "google.golang.org/grpc/health/grpc_health_v1"
11+
"google.golang.org/grpc/status"
12+
)
13+
14+
const healthCheckMethod = "/grpc.health.v1.Health/Watch"
15+
16+
// Client health check function is also part of the grpc/internal package
17+
// The following code is a simplified version of client.go
18+
// For more details see: https://pkg.go.dev/google.golang.org/grpc/health
19+
func ClientHealthCheck(ctx context.Context, backendConn *grpc.ClientConn, service string, setServingStatus func(serving bool)) error {
20+
shouldBackoff := false // No need for backoff on the first connection attempt
21+
backoffSrc := backoff.NewExponentialBackOff()
22+
healthClient := healthpb.NewHealthClient(backendConn)
23+
24+
for {
25+
// Backs off if the connection has failed in some way without receiving a message in the previous retry.
26+
if shouldBackoff {
27+
select {
28+
case <-time.After(backoffSrc.NextBackOff()):
29+
case <-ctx.Done():
30+
return nil
31+
}
32+
}
33+
shouldBackoff = true // we should backoff next time, since we attempt connecting below
34+
35+
req := healthpb.HealthCheckRequest{Service: service}
36+
s, err := healthClient.Watch(ctx, &req)
37+
if err != nil {
38+
continue
39+
}
40+
41+
resp := new(healthpb.HealthCheckResponse)
42+
for {
43+
err = s.RecvMsg(resp)
44+
if err != nil {
45+
setServingStatus(false)
46+
// The health check functionality should be disabled if health check service is not implemented on the backend
47+
if status.Code(err) == codes.Unimplemented {
48+
return err
49+
}
50+
// breaking out of the loop, since we got an error from Recv, triggering reconnect
51+
break
52+
}
53+
54+
// As a message has been received, removes the need for backoff for the next retry.
55+
shouldBackoff = false
56+
backoffSrc.Reset()
57+
58+
if resp.Status == healthpb.HealthCheckResponse_SERVING {
59+
setServingStatus(true)
60+
} else {
61+
setServingStatus(false)
62+
}
63+
}
64+
}
65+
}

go/grpcweb/health_test.go

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
package grpcweb_test
2+
3+
import (
4+
"context"
5+
"net"
6+
"testing"
7+
"time"
8+
9+
"github.com/improbable-eng/grpc-web/go/grpcweb"
10+
testproto "github.com/improbable-eng/grpc-web/integration_test/go/_proto/improbable/grpcweb/test"
11+
"github.com/stretchr/testify/assert"
12+
"github.com/stretchr/testify/require"
13+
"google.golang.org/grpc"
14+
"google.golang.org/grpc/health"
15+
healthpb "google.golang.org/grpc/health/grpc_health_v1"
16+
)
17+
18+
func TestClientWithNoHealthServiceOnServer(t *testing.T) {
19+
// Set up and run a server with no health check handler registered
20+
grpcServer := grpc.NewServer()
21+
testproto.RegisterTestServiceServer(grpcServer, &testServiceImpl{})
22+
listener, err := net.Listen("tcp", "127.0.0.1:0")
23+
require.NoError(t, err)
24+
25+
go func() {
26+
_ = grpcServer.Serve(listener)
27+
}()
28+
t.Cleanup(grpcServer.Stop)
29+
30+
grpcClientConn, err := grpc.Dial(listener.Addr().String(),
31+
grpc.WithBlock(),
32+
grpc.WithTimeout(100*time.Millisecond),
33+
grpc.WithInsecure(),
34+
)
35+
require.NoError(t, err)
36+
37+
ctx := context.Background()
38+
39+
servingStatus := true
40+
err = grpcweb.ClientHealthCheck(ctx, grpcClientConn, "", func(serving bool) {
41+
servingStatus = serving
42+
})
43+
assert.Error(t, err)
44+
assert.False(t, servingStatus)
45+
}
46+
47+
type clientHealthTestData struct {
48+
listener net.Listener
49+
serving bool
50+
healthServer *health.Server
51+
}
52+
53+
func setupTestData(t *testing.T) clientHealthTestData {
54+
s := clientHealthTestData{}
55+
56+
grpcServer := grpc.NewServer()
57+
s.healthServer = health.NewServer()
58+
healthpb.RegisterHealthServer(grpcServer, s.healthServer)
59+
60+
var err error
61+
s.listener, err = net.Listen("tcp", "127.0.0.1:0")
62+
require.NoError(t, err)
63+
64+
go func() {
65+
grpcServer.Serve(s.listener)
66+
}()
67+
t.Cleanup(grpcServer.Stop)
68+
69+
return s
70+
}
71+
72+
func (s *clientHealthTestData) dialBackend(t *testing.T) *grpc.ClientConn {
73+
grpcClientConn, err := grpc.Dial(s.listener.Addr().String(),
74+
grpc.WithBlock(),
75+
grpc.WithTimeout(100*time.Millisecond),
76+
grpc.WithInsecure(),
77+
)
78+
require.NoError(t, err)
79+
return grpcClientConn
80+
}
81+
82+
func (s *clientHealthTestData) checkServingStatus(t *testing.T, expStatus bool) {
83+
for start := time.Now(); time.Since(start) < 100*time.Millisecond; {
84+
if s.serving == expStatus {
85+
break
86+
}
87+
}
88+
assert.Equal(t, expStatus, s.serving)
89+
}
90+
91+
func (s *clientHealthTestData) startClientHealthCheck(ctx context.Context, conn *grpc.ClientConn) {
92+
go func() {
93+
_ = grpcweb.ClientHealthCheck(ctx, conn, "", func(status bool) {
94+
s.serving = status
95+
})
96+
}()
97+
}
98+
99+
func TestClientHealthCheck_FailsIfNotServing(t *testing.T) {
100+
s := setupTestData(t)
101+
102+
s.healthServer.SetServingStatus("", healthpb.HealthCheckResponse_NOT_SERVING)
103+
104+
backendConn := s.dialBackend(t)
105+
ctx, cancel := context.WithCancel(context.Background())
106+
defer cancel()
107+
108+
s.startClientHealthCheck(ctx, backendConn)
109+
s.checkServingStatus(t, false)
110+
}
111+
112+
func TestClientHealthCheck_SucceedsIfServing(t *testing.T) {
113+
s := setupTestData(t)
114+
115+
s.healthServer.SetServingStatus("", healthpb.HealthCheckResponse_SERVING)
116+
117+
backendConn := s.dialBackend(t)
118+
ctx, cancel := context.WithCancel(context.Background())
119+
defer cancel()
120+
121+
s.startClientHealthCheck(ctx, backendConn)
122+
s.checkServingStatus(t, true)
123+
}
124+
125+
func TestClientHealthCheck_ReactsToStatusChange(t *testing.T) {
126+
s := setupTestData(t)
127+
128+
s.healthServer.SetServingStatus("", healthpb.HealthCheckResponse_NOT_SERVING)
129+
130+
backendConn := s.dialBackend(t)
131+
ctx, cancel := context.WithCancel(context.Background())
132+
defer cancel()
133+
134+
s.startClientHealthCheck(ctx, backendConn)
135+
s.checkServingStatus(t, false)
136+
137+
s.healthServer.SetServingStatus("", healthpb.HealthCheckResponse_SERVING)
138+
s.checkServingStatus(t, true)
139+
140+
s.healthServer.SetServingStatus("", healthpb.HealthCheckResponse_NOT_SERVING)
141+
s.checkServingStatus(t, false)
142+
}

0 commit comments

Comments
 (0)