observer: fix TiProxy reports unmarshal body in healthy check failed" when TiDB is graceful shutdown (#569)

Author: djshow832

pkg/balance/observer/health_check.go

@@ -100,28 +100,29 @@ func (dhc *DefaultHealthCheck) checkSqlPort(ctx context.Context, addr string, bh
 func (dhc *DefaultHealthCheck) backendStatusCheck(httpCli *http.Client, url string, bh *BackendHealth) error {
 	resp, err := httpCli.Get(url)
 	if err == nil {
+		defer func() {
+			if ignoredErr := resp.Body.Close(); ignoredErr != nil {
+				dhc.logger.Warn("close http response in health check failed", zap.Error(ignoredErr))
+			}
+		}()
 
 		if resp.StatusCode != http.StatusOK {
-			err = backoff.Permanent(errors.Errorf("http status %d", resp.StatusCode))
+			return backoff.Permanent(errors.Errorf("http status %d", resp.StatusCode))
 		}
 
 		body, err := io.ReadAll(resp.Body)
 		if err != nil {
-			dhc.logger.Error("read response body in healthy check failed ", zap.Error(err))
+			dhc.logger.Error("read response body in healthy check failed", zap.String("url", url), zap.Error(err))
 			return err
 		}
 
 		var respBody backendHttpStatusRespBody
 		err = json.Unmarshal(body, &respBody)
 		if err != nil {
-			dhc.logger.Error("unmarshal body in healthy check failed", zap.String("resp body", string(body)), zap.Error(err))
+			dhc.logger.Error("unmarshal body in healthy check failed", zap.String("url", url), zap.String("resp body", string(body)), zap.Error(err))
 			return err
 		}
 
-		if ignoredErr := resp.Body.Close(); ignoredErr != nil {
-			dhc.logger.Warn("close http response in health check failed", zap.Error(ignoredErr))
-		}
-
 		bh.ServerVersion = respBody.Version
 	}
 	return err

pkg/balance/observer/health_check_test.go

@@ -23,11 +23,9 @@ func TestReadServerVersion(t *testing.T) {
 	hc := NewDefaultHealthCheck(nil, newHealthCheckConfigForTest(), lg)
 	backend, info := newBackendServer(t)
 	backend.setServerVersion("1.0")
-	//backend.serverVersion.Store("1.0")
 	health := hc.Check(context.Background(), backend.sqlAddr, info)
 	require.Equal(t, "1.0", health.ServerVersion)
 	backend.stopSQLServer()
-	//backend.serverVersion.Store("2.0")
 	backend.setServerVersion("2.0")
 	backend.startSQLServer()
 	health = hc.Check(context.Background(), backend.sqlAddr, info)
@@ -47,7 +45,7 @@ func TestReadServerVersion(t *testing.T) {
 
 // Test that the backend status is correct when the backend starts or shuts down.
 func TestHealthCheck(t *testing.T) {
-	lg, _ := logger.CreateLoggerForTest(t)
+	lg, text := logger.CreateLoggerForTest(t)
 	cfg := newHealthCheckConfigForTest()
 	hc := NewDefaultHealthCheck(nil, cfg, lg)
 	backend, info := newBackendServer(t)
@@ -65,6 +63,7 @@ func TestHealthCheck(t *testing.T) {
 	backend.setHTTPResp(false)
 	health = hc.Check(context.Background(), backend.sqlAddr, info)
 	require.False(t, health.Healthy)
+	require.NotContains(t, text.String(), "unmarshal body")
 	backend.setHTTPResp(true)
 	health = hc.Check(context.Background(), backend.sqlAddr, info)
 	require.True(t, health.Healthy)
@@ -113,6 +112,7 @@ func (srv *backendServer) setServerVersion(version string) {
 	body, _ := json.Marshal(resp)
 	srv.mockHttpHandler.setHTTPRespBody(string(body))
 }
+
 func (srv *backendServer) startHTTPServer() {
 	if srv.mockHttpHandler == nil {
 		srv.mockHttpHandler = &mockHttpHandler{