add kube_janitor_resource_deleted_total

Signed-off-by: Markus Blaschke <[email protected]>

Author: mblaschke

README.md

@@ -67,5 +67,6 @@ Supported relative timestamps ([`time.Duration`](https://pkg.go.dev/time) and [`
 
 | Metric                                                | Description                                                                                         |
 |-------------------------------------------------------|-----------------------------------------------------------------------------------------------------|
+| `kube_janitor_resource_deleted_total`                 | Total number of deleted resources (by namespace, gvk, rule)                                         |
 | `kube_janitor_resource_ttl_expiry_timestamp_seconds`  | Expiry date (unix timestamp) for every resource which was detected matching the TTL expiry          |
 | `kube_janitor_resource_rule_expiry_timestamp_seconds` | Expiry date (unix timestamp) for every resource which was detected matching the static expiry rules |

kube_janitor/metrics.go

@@ -6,28 +6,41 @@ import (
 
 type (
 	JanitorMetrics struct {
-		ttl  *prometheus.GaugeVec
-		rule *prometheus.GaugeVec
+		deleted *prometheus.CounterVec
+		ttl     *prometheus.GaugeVec
+		rule    *prometheus.GaugeVec
 	}
 )
 
 // setupMetrics setups all Prometheus metrics with name, help and corresponding labels
 func (j *Janitor) setupMetrics() {
-	commonLabels := []string{
+	ttlLabels := []string{
 		"rule",
-		"version",
-		"kind",
+		"groupVersionKind",
 		"namespace",
 		"name",
 		"ttl",
 	}
 
+	j.prometheus.deleted = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "kube_janitor_resource_deleted_total",
+			Help: "Total count of deleted Kubernetes resources",
+		},
+		[]string{
+			"rule",
+			"groupVersionKind",
+			"namespace",
+		},
+	)
+	prometheus.MustRegister(j.prometheus.deleted)
+
 	j.prometheus.ttl = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "kube_janitor_resource_ttl_expiry_timestamp_seconds",
 			Help: "Expiry unix timestamp for Kubernetes resources by ttl",
 		},
-		commonLabels,
+		ttlLabels,
 	)
 	prometheus.MustRegister(j.prometheus.ttl)
 
@@ -36,7 +49,7 @@ func (j *Janitor) setupMetrics() {
 			Name: "kube_janitor_resource_rule_expiry_timestamp_seconds",
 			Help: "Expiry unix timestamp for Kubernetes resources by rule",
 		},
-		commonLabels,
+		ttlLabels,
 	)
 	prometheus.MustRegister(j.prometheus.rule)
 }

kube_janitor/task.common.go

@@ -78,9 +78,6 @@ func (j *Janitor) runRule(ctx context.Context, logger *slogger.Logger, rule *Con
 					rule.Id,
 					ttl,
 					metricList,
-					prometheus.Labels{
-						"rule": rule.String(),
-					},
 				)
 			})
 			if err != nil {
@@ -95,13 +92,15 @@ func (j *Janitor) runRule(ctx context.Context, logger *slogger.Logger, rule *Con
 }
 
 // checkResourceTtlAndTriggerDeleteIfExpired checks the resource against the defined TTL and deletes if the resource is expired
-func (j *Janitor) checkResourceTtlAndTriggerDeleteIfExpired(ctx context.Context, logger *slogger.Logger, resourceConfig *ConfigResource, resource unstructured.Unstructured, ruleId string, ttlValue string, metricResourceTtl *prometheusCommon.MetricList, labels prometheus.Labels) error {
+func (j *Janitor) checkResourceTtlAndTriggerDeleteIfExpired(ctx context.Context, logger *slogger.Logger, resourceConfig *ConfigResource, resource unstructured.Unstructured, ruleId string, ttlValue string, metricResourceTtl *prometheusCommon.MetricList) error {
 	resourceLogger := logger.WithGroup("resource").With(
 		slog.String("namespace", resource.GetNamespace()),
 		slog.String("name", resource.GetName()),
 		slog.String("ttl", ttlValue),
 	)
 
+	groupVersionKind := resource.GroupVersionKind()
+
 	// no ttl, no processing
 	// better safe than sorry
 	if ttlValue == "" {
@@ -143,13 +142,6 @@ func (j *Janitor) checkResourceTtlAndTriggerDeleteIfExpired(ctx context.Context,
 		return nil
 	}
 
-	labels["version"] = resource.GetAPIVersion()
-	labels["kind"] = resource.GetKind()
-	labels["namespace"] = resource.GetNamespace()
-	labels["name"] = resource.GetName()
-	labels["ttl"] = ttlValue
-	metricResourceTtl.AddTime(labels, *parsedDate)
-
 	resourceLogger.Debug("found resource with valid TTL", slog.Time("expiry", *parsedDate))
 
 	if expired {
@@ -162,6 +154,15 @@ func (j *Janitor) checkResourceTtlAndTriggerDeleteIfExpired(ctx context.Context,
 				return err
 			}
 
+			// increase deleted counter
+			j.prometheus.deleted.With(
+				prometheus.Labels{
+					"rule":             ruleId,
+					"groupVersionKind": fmt.Sprintf("%s/%s/%s", groupVersionKind.Group, groupVersionKind.Version, groupVersionKind.Kind),
+					"namespace":        resource.GetNamespace(),
+				},
+			).Inc()
+
 			reason := "TimeToLiveExpired"
 			message := fmt.Sprintf(`TTL of "%v" is expired and resource is being deleted (%s)`, ttlValue, ruleId)
 
@@ -170,6 +171,20 @@ func (j *Janitor) checkResourceTtlAndTriggerDeleteIfExpired(ctx context.Context,
 				resourceLogger.Error("unable to create Kubernetes Event", slog.Any("error", err))
 			}
 		}
+	} else {
+		// resource not yet expired, but add expiry as metric
+
+		metricResourceTtl.AddTime(
+			prometheus.Labels{
+				"rule":             ruleId,
+				"groupVersionKind": fmt.Sprintf("%s/%s/%s", groupVersionKind.Group, groupVersionKind.Version, groupVersionKind.Kind),
+				"namespace":        resource.GetNamespace(),
+				"name":             resource.GetName(),
+				"ttl":              ttlValue,
+			},
+			*parsedDate,
+		)
+
 	}
 
 	return nil