metricreader: fix backend label value doesn't match Prometheus in k8s (#635)

Author: djshow832

pkg/balance/metricsreader/backend_reader.go

@@ -272,8 +272,12 @@ func (br *BackendReader) readFromBackends(ctx context.Context, excludeZones []st
 		return nil
 	}
 
+	backendLabels := make([]string, 0, len(addrs))
 	for _, addr := range addrs {
-		func(addr string) {
+		backendLabels = append(backendLabels, getLabel4Addr(addr))
+	}
+	for i := range addrs {
+		func(addr, label string) {
 			br.wgp.RunWithRecover(func() {
 				if ctx.Err() != nil {
 					return
@@ -289,15 +293,15 @@ func (br *BackendReader) readFromBackends(ctx context.Context, excludeZones []st
 					br.lg.Warn("parse metrics failed", zap.String("addr", addr), zap.Error(err))
 					return
 				}
-				br.metric2History(mf, addr)
+				br.metric2History(mf, label)
 			}, nil, br.lg)
-		}(addr)
+		}(addrs[i], backendLabels[i])
 	}
 	br.wgp.Wait()
 
 	br.history2QueryResult()
-	if err := br.marshalHistory(addrs); err != nil {
-		br.lg.Error("marshal backend history failed", zap.Any("addrs", addrs), zap.Error(err))
+	if err := br.marshalHistory(backendLabels); err != nil {
+		br.lg.Error("marshal backend history failed", zap.Any("addrs", backendLabels), zap.Error(err))
 	}
 	return nil
 }

pkg/balance/metricsreader/query_result.go

@@ -125,3 +125,13 @@ func getLabel4Backend(backend policy.BackendCtx) string {
 	backendInfo := backend.GetBackendInfo()
 	return net.JoinHostPort(backendInfo.IP, strconv.Itoa(int(backendInfo.StatusPort)))
 }
+
+// addr is the address of the backend status port.
+func getLabel4Addr(addr string) string {
+	if strings.Contains(addr, ".svc:") {
+		// In operator deployment, the label value of `instance` is the pod name.
+		return addr[:strings.Index(addr, ".")]
+	}
+	// In tiup deployment, the label value of `instance` is hostname:statusPort.
+	return addr
+}

pkg/balance/metricsreader/query_result_test.go

@@ -174,3 +174,23 @@ func TestVectorMatchLabel(t *testing.T) {
 		require.Equal(t, test.expectedSample.Value, sample.Value, "test index %d", i)
 	}
 }
+
+func TestAddrMatchLabel(t *testing.T) {
+	tests := []struct {
+		addr  string
+		label string
+	}{
+		{
+			addr:  "10.10.11.1:3080",
+			label: "10.10.11.1:3080",
+		},
+		{
+			addr:  "tc-tidb-0.tc-tidb-peer.ns.svc:3080",
+			label: "tc-tidb-0",
+		},
+	}
+
+	for i, test := range tests {
+		require.Equal(t, test.label, getLabel4Addr(test.addr), "test index %d", i)
+	}
+}