get return codes for ipconfig at least (#1231)

* get return codes for ipconfig at least

* fix lint

* valid label name

Author: paulgmiller

cns/restserver/ipam.go

@@ -34,6 +34,7 @@ func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r
 				Message:    returnMessage,
 			},
 		}
+		w.Header().Set(cnsReturnCode, reserveResp.Response.ReturnCode.String())
 		err = service.Listener.Encode(w, &reserveResp)
 		logger.ResponseEx(service.Name+operationName, ipconfigRequest, reserveResp, reserveResp.Response.ReturnCode, err)
 		return
@@ -51,6 +52,7 @@ func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r
 			},
 			PodIpInfo: podIPInfo,
 		}
+		w.Header().Set(cnsReturnCode, reserveResp.Response.ReturnCode.String())
 		err = service.Listener.Encode(w, &reserveResp)
 		logger.ResponseEx(service.Name+operationName, ipconfigRequest, reserveResp, reserveResp.Response.ReturnCode, err)
 		return
@@ -69,7 +71,7 @@ func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r
 		},
 		PodIpInfo: podIPInfo,
 	}
-
+	w.Header().Set(cnsReturnCode, reserveResp.Response.ReturnCode.String())
 	err = service.Listener.Encode(w, &reserveResp)
 	logger.ResponseEx(service.Name+operationName, ipconfigRequest, reserveResp, reserveResp.Response.ReturnCode, err)
 }
@@ -84,6 +86,7 @@ func (service *HTTPRestService) releaseIPConfigHandler(w http.ResponseWriter, r
 			Message:    err.Error(),
 		}
 		logger.Errorf("releaseIPConfigHandler decode failed becase %v, release IP config info %s", resp.Message, req)
+		w.Header().Set(cnsReturnCode, resp.ReturnCode.String())
 		err = service.Listener.Encode(w, &resp)
 		logger.ResponseEx(service.Name, req, resp, resp.ReturnCode, err)
 		return
@@ -100,6 +103,7 @@ func (service *HTTPRestService) releaseIPConfigHandler(w http.ResponseWriter, r
 		ReturnCode: returnCode,
 		Message:    message,
 	}
+	w.Header().Set(cnsReturnCode, resp.ReturnCode.String())
 	err = service.Listener.Encode(w, &resp)
 	logger.ResponseEx(service.Name, req, resp, resp.ReturnCode, err)
 }

cns/restserver/metrics.go

@@ -17,11 +17,7 @@ var httpRequestLatency = prometheus.NewHistogramVec(
 		//nolint:gomnd // default bucket consts
 		Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), // 1 ms to ~16 seconds
 	},
-	// TODO(rbtr):
-	// there's no easy way to extract the HTTP response code from the response due to the
-	// way the restserver is designed currently - but we should fix that and include "code" as
-	// a label and value.
-	[]string{"url", "verb"},
+	[]string{"url", "verb", "cns_return_code"},
 )
 
 var ipAssignmentLatency = prometheus.NewHistogram(
@@ -51,11 +47,18 @@ func init() {
 	)
 }
 
+const cnsReturnCode = "Cns-Return-Code"
+
+// Every http response is 200 so we really want cns  response code.
+// Hard tto do with middleware unless we derserialize the responses but making it an explit header works around it.
+// if that doesn't work we could have a separate countervec just for response codes.
+
 func newHandlerFuncWithHistogram(handler http.HandlerFunc, histogram *prometheus.HistogramVec) http.HandlerFunc {
 	return func(w http.ResponseWriter, req *http.Request) {
 		start := time.Now()
 		defer func() {
-			histogram.WithLabelValues(req.URL.RequestURI(), req.Method).Observe(time.Since(start).Seconds())
+			code := w.Header().Get(cnsReturnCode)
+			histogram.WithLabelValues(req.URL.RequestURI(), req.Method, code).Observe(time.Since(start).Seconds())
 		}()
 		handler(w, req)
 	}