metricsreader: only report error once if Prometheus is not deployed (#579)

Author: djshow832

pkg/balance/metricsreader/metrics_reader.go

@@ -25,9 +25,9 @@ import (
 )
 
 const (
-	getPromNone = iota
-	getPromOK
-	getPromFail
+	readResultNone = iota
+	readResultOK
+	readResultFail
 )
 
 type PromInfoFetcher interface {
@@ -54,7 +54,7 @@ type DefaultMetricsReader struct {
 	lg           *zap.Logger
 	cfg          *config.HealthCheck
 	lastID       uint64
-	getPromRes   int
+	readResult   int
 }
 
 func NewDefaultMetricsReader(lg *zap.Logger, promFetcher PromInfoFetcher, cfg *config.HealthCheck) *DefaultMetricsReader {
@@ -79,11 +79,21 @@ func (dmr *DefaultMetricsReader) Start(ctx context.Context) {
 		defer ticker.Stop()
 		for childCtx.Err() == nil {
 			if results, err := dmr.readMetrics(childCtx); err != nil {
-				dmr.lg.Warn("read metrics failed", zap.Error(err))
-			} else if len(results) > 0 {
-				dmr.Lock()
-				dmr.queryResults = results
-				dmr.Unlock()
+				// If there are successive errors, only log it once.
+				if dmr.readResult != readResultFail {
+					dmr.readResult = readResultFail
+					dmr.lg.Warn("read metrics failed", zap.Error(err))
+				}
+			} else {
+				if dmr.readResult != readResultOK {
+					dmr.readResult = readResultOK
+					dmr.lg.Debug("read metrics succeed")
+				}
+				if len(results) > 0 {
+					dmr.Lock()
+					dmr.queryResults = results
+					dmr.Unlock()
+				}
 			}
 			select {
 			case <-ticker.C:
@@ -98,24 +108,16 @@ func (dmr *DefaultMetricsReader) Start(ctx context.Context) {
 func (dmr *DefaultMetricsReader) getPromAPI(ctx context.Context) (promv1.API, error) {
 	promInfo, err := dmr.promFetcher.GetPromInfo(ctx)
 	if promInfo == nil {
-		if dmr.getPromRes != getPromFail {
-			dmr.getPromRes = getPromFail
-		}
 		if err == nil {
 			err = errors.New("no prometheus info found")
 		}
-		dmr.lg.Warn("get prometheus address fails", zap.Error(err))
 		return nil, err
 	}
 	if err != nil {
 		return nil, err
 	}
 	// TODO: support TLS and authentication.
 	promAddr := fmt.Sprintf("http://%s", net.JoinHostPort(promInfo.IP, strconv.Itoa(promInfo.Port)))
-	if dmr.getPromRes != getPromOK {
-		dmr.getPromRes = getPromOK
-		dmr.lg.Info("get prometheus address succeeds", zap.String("addr", promAddr))
-	}
 	promClient, err := api.NewClient(api.Config{
 		Address: promAddr,
 	})

pkg/balance/metricsreader/metrics_reader_test.go

@@ -208,7 +208,7 @@ func TestNoPromAddr(t *testing.T) {
 			return nil, nil
 		},
 	}
-	lg, _ := logger.CreateLoggerForTest(t)
+	lg, text := logger.CreateLoggerForTest(t)
 	cfg := newHealthCheckConfigForTest()
 	mr := NewDefaultMetricsReader(lg, mpf, cfg)
 	promInfo, err := mr.getPromAPI(context.Background())
@@ -225,6 +225,7 @@ func TestNoPromAddr(t *testing.T) {
 	time.Sleep(10 * cfg.MetricsInterval)
 	qr := mr.GetQueryResult(id)
 	require.True(t, qr.Empty())
+	require.Equal(t, 1, strings.Count(text.String(), "no prometheus info found"))
 }
 
 func setupTypicalMetricsReader(t *testing.T) (*mockHttpHandler, MetricsReader) {