change total IPs and add secondary IP metric (#2172)

* change total IPs and add secondary IP metric

Updates the Total IPs metrics to include the NC
Primary IP in the total. Adds a Secondary IPs
metric which holds the value that the Total IPs
previously held: NC Secondary IPs known to CNS
which could be used by Pods.

Signed-off-by: GitHub <[email protected]>

* update help wording on IPAM metrics

Signed-off-by: Evan Baker <[email protected]>

* reword PrimaryIP metric help

Signed-off-by: Evan Baker <[email protected]>

---------

Signed-off-by: GitHub <[email protected]>
Signed-off-by: Evan Baker <[email protected]>

Author: rbtr

cns/ipampool/metrics.go

@@ -20,23 +20,23 @@ var (
 	ipamAllocatedIPCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_pod_allocated_ips",
-			Help:        "Count of IPs CNS has allocated to Pods.",
+			Help:        "IPs currently in use by Pods on this CNS Node.",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
 	ipamAvailableIPCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_available_ips",
-			Help:        "Available IP count.",
+			Help:        "IPs available on this CNS Node for use by a Pod.",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
 	ipamBatchSize = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_batch_size",
-			Help:        "IPAM IP pool batch size.",
+			Help:        "IPAM IP pool scaling batch size.",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
@@ -60,65 +60,73 @@ var (
 	ipamMaxIPCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_max_ips",
-			Help:        "Maximum IP count.",
+			Help:        "Maximum Secondary IPs allowed on this Node.",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
 	ipamPendingProgramIPCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_pending_programming_ips",
-			Help:        "Pending programming IP count.",
+			Help:        "IPs reserved but not yet available (Pending Programming).",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
 	ipamPendingReleaseIPCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_pending_release_ips",
-			Help:        "Pending release IP count.",
+			Help:        "IPs reserved but not available anymore (Pending Release).",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
 	ipamPrimaryIPCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_primary_ips",
-			Help:        "NC Primary IP count.",
+			Help:        "NC Primary IP count (reserved from Pod Subnet for DNS and IMDS SNAT).",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
 	ipamRequestedIPConfigCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_requested_ips",
-			Help:        "Requested IP count.",
+			Help:        "Secondary Pod Subnet IPs requested by this CNS Node (for Pods).",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
-	ipamTotalIPCount = prometheus.NewGaugeVec(
+	ipamSecondaryIPCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name:        "cx_ipam_total_ips",
-			Help:        "Count of total IP pool size allocated to CNS by DNC.",
+			Name:        "cx_ipam_secondary_ips",
+			Help:        "Node NC Secondary IP count (reserved usable by Pods).",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
+	ipamSubnetExhaustionCount = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "cx_ipam_subnet_exhaustion_state_count_total",
+			Help: "Count of the number of times the ipam pool monitor sees subnet exhaustion",
+		},
+		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel, subnetExhaustionStateLabel},
+	)
 	ipamSubnetExhaustionState = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_subnet_exhaustion_state",
-			Help:        "IPAM view of subnet exhaustion state",
+			Help:        "CNS view of subnet exhaustion state",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
-	ipamSubnetExhaustionCount = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "cx_ipam_subnet_exhaustion_state_count_total",
-			Help: "Count of the number of times the ipam pool monitor sees subnet exhaustion",
+	ipamTotalIPCount = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name:        "cx_ipam_total_ips",
+			Help:        "Total IPs reserved from the Pod Subnet by this Node.",
+			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
-		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel, subnetExhaustionStateLabel},
+		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
 )
 
@@ -134,9 +142,10 @@ func init() {
 		ipamPendingReleaseIPCount,
 		ipamPrimaryIPCount,
 		ipamRequestedIPConfigCount,
-		ipamTotalIPCount,
-		ipamSubnetExhaustionState,
+		ipamSecondaryIPCount,
 		ipamSubnetExhaustionCount,
+		ipamSubnetExhaustionState,
+		ipamTotalIPCount,
 	)
 }
 
@@ -152,7 +161,8 @@ func observeIPPoolState(state ipPoolState, meta metaState) {
 	ipamPendingReleaseIPCount.WithLabelValues(labels...).Set(float64(state.pendingRelease))
 	ipamPrimaryIPCount.WithLabelValues(labels...).Set(float64(len(meta.primaryIPAddresses)))
 	ipamRequestedIPConfigCount.WithLabelValues(labels...).Set(float64(state.requestedIPs))
-	ipamTotalIPCount.WithLabelValues(labels...).Set(float64(state.totalIPs))
+	ipamSecondaryIPCount.WithLabelValues(labels...).Set(float64(state.secondaryIPs))
+	ipamTotalIPCount.WithLabelValues(labels...).Set(float64(state.secondaryIPs + int64(len(meta.primaryIPAddresses))))
 	if meta.exhausted {
 		ipamSubnetExhaustionState.WithLabelValues(labels...).Set(float64(subnetIPExhausted))
 	} else {

cns/ipampool/monitor.go

@@ -164,13 +164,13 @@ type ipPoolState struct {
 	pendingRelease int64
 	// requestedIPs are the IPs CNS has requested that it be allocated by DNC.
 	requestedIPs int64
-	// totalIPs are all the IPs given to CNS by DNC.
-	totalIPs int64
+	// secondaryIPs are all the IPs given to CNS by DNC, not including the primary IP of the NC.
+	secondaryIPs int64
 }
 
 func buildIPPoolState(ips map[string]cns.IPConfigurationStatus, spec v1alpha.NodeNetworkConfigSpec) ipPoolState {
 	state := ipPoolState{
-		totalIPs:     int64(len(ips)),
+		secondaryIPs: int64(len(ips)),
 		requestedIPs: spec.RequestedIPCount,
 	}
 	for i := range ips {
@@ -186,7 +186,7 @@ func buildIPPoolState(ips map[string]cns.IPConfigurationStatus, spec v1alpha.Nod
 			state.pendingRelease++
 		}
 	}
-	state.currentAvailableIPs = state.totalIPs - state.allocatedToPods - state.pendingRelease
+	state.currentAvailableIPs = state.secondaryIPs - state.allocatedToPods - state.pendingRelease
 	state.expectedAvailableIPs = state.requestedIPs - state.allocatedToPods
 	return state
 }