metricreader: fix that the metrics of CPU and health are not collected (#617)

Author: djshow832

lib/util/logger/logger.go

@@ -38,6 +38,6 @@ func CreateLoggerForTest(t *testing.T) (*zap.Logger, fmt.Stringer) {
 	return zap.New(zapcore.NewCore(
 		zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()),
 		zapcore.AddSync(log),
-		zap.InfoLevel,
+		zap.DebugLevel,
 	)).Named(t.Name()), log
 }

pkg/balance/metricsreader/backend_reader.go

@@ -283,13 +283,13 @@ func (br *BackendReader) readFromBackends(ctx context.Context, excludeZones []st
 				}
 				resp, err := br.readBackendMetric(ctx, addr)
 				if err != nil {
-					br.lg.Error("read metrics from backend failed", zap.String("addr", addr), zap.Error(err))
+					br.lg.Debug("read metrics from backend failed", zap.String("addr", addr), zap.Error(err))
 					return
 				}
 				text := filterMetrics(hack.String(resp), allNames)
 				mf, err := parseMetrics(text)
 				if err != nil {
-					br.lg.Error("parse metrics failed", zap.String("addr", addr), zap.Error(err))
+					br.lg.Warn("parse metrics failed", zap.String("addr", addr), zap.Error(err))
 					return
 				}
 				br.metric2History(mf, addr)
@@ -362,10 +362,9 @@ func (br *BackendReader) metric2History(mfs map[string]*dto.MetricFamily, backen
 		// step 2: get the latest pair by the history and add it to step2History
 		// E.g. calculate irate(process_cpu_seconds_total/tidb_server_maxprocs[30s])
 		sampleValue = rule.Range2Value(beHistory.Step1History)
-		if math.IsNaN(float64(sampleValue)) {
-			continue
+		if !math.IsNaN(float64(sampleValue)) {
+			beHistory.Step2History = append(beHistory.Step2History, model.SamplePair{Timestamp: now, Value: sampleValue})
 		}
-		beHistory.Step2History = append(beHistory.Step2History, model.SamplePair{Timestamp: now, Value: sampleValue})
 		ruleHistory[backend] = beHistory
 	}
 }

pkg/balance/metricsreader/backend_reader_test.go

@@ -363,7 +363,7 @@ func TestOneRuleMultiHistory(t *testing.T) {
 	lg, _ := logger.CreateLoggerForTest(t)
 	br := NewBackendReader(lg, nil, nil, nil, nil, nil)
 	var lastTs model.Time
-	var length int
+	var step1Len, step2Len int
 	for i, test := range tests {
 		br.queryRules = map[string]QueryRule{
 			"key": {
@@ -379,17 +379,30 @@ func TestOneRuleMultiHistory(t *testing.T) {
 		}
 
 		br.metric2History(mfs, "backend")
+		history := br.history["key"]
+		if !math.IsNaN(float64(test.step1Value)) {
+			step1Len++
+		}
+		if step1Len > 0 {
+			require.Equal(t, step1Len, len(history["backend"].Step1History), "case %d", i)
+		}
+		if !math.IsNaN(float64(test.step2Value)) {
+			step2Len++
+		}
+		if step2Len > 0 {
+			require.Equal(t, step2Len, len(history["backend"].Step2History), "case %d", i)
+		}
+
 		res := br.history2Value("backend")
 		value := res["key"]
 		if math.IsNaN(float64(test.step2Value)) {
 			continue
 		}
-		length++
 		require.Equal(t, model.ValMatrix, value.Type(), "case %d", i)
 		matrix := value.(model.Matrix)
 		require.Len(t, matrix, 1, "case %d", i)
 		require.Equal(t, model.LabelValue("backend"), matrix[0].Metric[LabelNameInstance], "case %d", i)
-		require.Len(t, matrix[0].Values, length, "case %d", i)
+		require.Len(t, matrix[0].Values, step2Len, "case %d", i)
 		pair := matrix[0].Values[len(matrix[0].Values)-1]
 		require.Equal(t, test.step2Value, pair.Value, "case %d", i)
 		require.GreaterOrEqual(t, pair.Timestamp, lastTs, "case %d", i)

pkg/manager/vip/manager.go

@@ -87,7 +87,7 @@ func (vm *vipManager) OnElected() {
 		return
 	}
 	if hasIP {
-		vm.lg.Info("already has VIP, do nothing")
+		vm.lg.Debug("already has VIP, do nothing")
 		return
 	}
 	if err := vm.operation.AddIP(); err != nil {
@@ -108,7 +108,7 @@ func (vm *vipManager) OnRetired() {
 		return
 	}
 	if !hasIP {
-		vm.lg.Info("does not have VIP, do nothing")
+		vm.lg.Debug("does not have VIP, do nothing")
 		return
 	}
 	if err := vm.operation.DeleteIP(); err != nil {

pkg/manager/vip/network.go

@@ -43,12 +43,12 @@ func (no *networkOperation) initAddr(addressStr, linkStr string) error {
 	}
 	address, err := netlink.ParseAddr(addressStr)
 	if err != nil {
-		return errors.WithStack(err)
+		return errors.Wrapf(errors.WithStack(err), "failed to parse address '%s'", addressStr)
 	}
 	no.address = address
 	link, err := netlink.LinkByName(linkStr)
 	if err != nil {
-		return errors.WithStack(err)
+		return errors.Wrapf(errors.WithStack(err), "failed to find network interface '%s'", linkStr)
 	}
 	no.link = link
 	return nil