feat: [NPM] perf metrics for pod/ns/policy CRUD (#1220)

* add dataplane health metrics

* change counters to countervecs

* wip

* uncomment metrics.ReinitializeAll()

* add comment about ReinitializeAll

* restructure prometheus-metrics.go, address comments, and finish UTs for v1

* properly record exec times and include error labels

* add error label to add_policy_exec_time

* add v2 UTs, test NoOp, and address comment

* resolve lints

Author: huntergregory

npm/http/server/server.go

@@ -28,8 +28,8 @@ func NPMRestServerListenAndServe(config npmconfig.Config, npmEncoder json.Marsha
 
 	// prometheus handlers
 	if config.Toggles.EnablePrometheusMetrics {
-		rs.router.Handle(api.NodeMetricsPath, metrics.GetHandler(true))
-		rs.router.Handle(api.ClusterMetricsPath, metrics.GetHandler(false))
+		rs.router.Handle(api.NodeMetricsPath, metrics.GetHandler(metrics.NodeMetrics))
+		rs.router.Handle(api.ClusterMetricsPath, metrics.GetHandler(metrics.ClusterMetrics))
 	}
 
 	if config.Toggles.EnableHTTPDebugAPI {

npm/metrics/ipsets.go

@@ -5,7 +5,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-var ipsetInventoryMap = make(map[string]int)
+var ipsetInventoryMap map[string]int
 
 // IncNumIPSets increments the number of IPSets.
 func IncNumIPSets() {

npm/metrics/namespaces.go

@@ -0,0 +1,13 @@
+package metrics
+
+// RecordControllerNamespaceExecTime adds an observation of namespace exec time (unless the operation is NoOp).
+// The execution time is from the timer's start until now.
+func RecordControllerNamespaceExecTime(timer *Timer, op OperationKind, hadError bool) {
+	timer.stopAndRecordCRUDExecTime(controllerNamespaceExecTime, op, hadError)
+}
+
+// GetControllerNamespaceExecCount returns the number of observations for namespace exec time for the specified operation.
+// This function is slow.
+func GetControllerNamespaceExecCount(op OperationKind, hadError bool) (int, error) {
+	return getCountVecValue(controllerNamespaceExecTime, getCRUDExecTimeLabels(op, hadError))
+}

npm/metrics/namespaces_test.go

@@ -0,0 +1,10 @@
+package metrics
+
+import "testing"
+
+func TestRecordControllerNamespaceExecTime(t *testing.T) {
+	testStopAndRecordCRUDExecTime(t, &crudExecMetric{
+		RecordControllerNamespaceExecTime,
+		GetControllerNamespaceExecCount,
+	})
+}

npm/metrics/pods.go

@@ -0,0 +1,13 @@
+package metrics
+
+// RecordControllerPodExecTime adds an observation of pod exec time for the specified operation (unless the operation is NoOp).
+// The execution time is from the timer's start until now.
+func RecordControllerPodExecTime(timer *Timer, op OperationKind, hadError bool) {
+	timer.stopAndRecordCRUDExecTime(controllerPodExecTime, op, hadError)
+}
+
+// GetControllerPodExecCount returns the number of observations for pod exec time for the specified operation.
+// This function is slow.
+func GetControllerPodExecCount(op OperationKind, hadError bool) (int, error) {
+	return getCountVecValue(controllerPodExecTime, getCRUDExecTimeLabels(op, hadError))
+}

npm/metrics/pods_test.go

@@ -0,0 +1,10 @@
+package metrics
+
+import "testing"
+
+func TestRecordControllerPodExecTime(t *testing.T) {
+	testStopAndRecordCRUDExecTime(t, &crudExecMetric{
+		RecordControllerPodExecTime,
+		GetControllerPodExecCount,
+	})
+}

npm/metrics/policies.go

@@ -15,10 +15,14 @@ func ResetNumPolicies() {
 	numPolicies.Set(0)
 }
 
-// RecordPolicyExecTime adds an observation of execution time for adding a policy.
+// RecordControllerPolicyExecTime adds an observation of policy exec time  (unless the operation is NoOp).
 // The execution time is from the timer's start until now.
-func RecordPolicyExecTime(timer *Timer) {
-	timer.stopAndRecord(addPolicyExecTime)
+func RecordControllerPolicyExecTime(timer *Timer, op OperationKind, hadError bool) {
+	if op == CreateOp {
+		timer.stopAndRecordExecTimeWithError(addPolicyExecTime, hadError)
+	} else {
+		timer.stopAndRecordCRUDExecTime(controllerPolicyExecTime, op, hadError)
+	}
 }
 
 // GetNumPolicies returns the number of policies.
@@ -27,8 +31,11 @@ func GetNumPolicies() (int, error) {
 	return getValue(numPolicies)
 }
 
-// GetPolicyExecCount returns the number of observations for execution time of adding policies.
+// GetControllerPolicyExecCount returns the number of observations for policy exec time for the specified operation.
 // This function is slow.
-func GetPolicyExecCount() (int, error) {
-	return getCountValue(addPolicyExecTime)
+func GetControllerPolicyExecCount(op OperationKind, hadError bool) (int, error) {
+	if op == CreateOp {
+		return getCountVecValue(addPolicyExecTime, getErrorLabels(hadError))
+	}
+	return getCountVecValue(controllerPolicyExecTime, getCRUDExecTimeLabels(op, hadError))
 }

npm/metrics/policies_test.go

@@ -2,13 +2,13 @@ package metrics
 
 import "testing"
 
-var (
-	numPoliciesMetric = &basicMetric{ResetNumPolicies, IncNumPolicies, DecNumPolicies, GetNumPolicies}
-	policyExecMetric  = &recordingMetric{RecordPolicyExecTime, GetPolicyExecCount}
-)
+var numPoliciesMetric = &basicMetric{ResetNumPolicies, IncNumPolicies, DecNumPolicies, GetNumPolicies}
 
-func TestRecordPolicyExecTime(t *testing.T) {
-	testStopAndRecord(t, policyExecMetric)
+func TestRecordControllerPolicyExecTime(t *testing.T) {
+	testStopAndRecordCRUDExecTime(t, &crudExecMetric{
+		RecordControllerPolicyExecTime,
+		GetControllerPolicyExecCount,
+	})
 }
 
 func TestIncNumPolicies(t *testing.T) {