fix: recording rules missing postgres pod metrics for CNPG dashboard

CNPG dashboard was missing data because recording rule metrics only
appeared for kube-state-metrics, not for PostgreSQL database pods.

Fixed by setting honorLabels: true on kube-state-metrics and kubelet
ServiceMonitors, which preserves the correct pod labels needed for the
recording rule joins to succeed.

Now all pods including pg-{region}-* appear in recording rule metrics.

Author: ardentperf

monitoring/kube-state-metrics/deployment.yaml

@@ -85,6 +85,7 @@ spec:
   endpoints:
   - port: http-metrics
     interval: 30s
+    honorLabels: true
   - port: telemetry
     interval: 30s
   selector:

monitoring/prometheus-instance/servicemonitor-kubelet.yaml

@@ -36,6 +36,7 @@ spec:
     bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
     tlsConfig:
       insecureSkipVerify: true
+    honorLabels: true
     relabelings:
     - targetLabel: metrics_path
       replacement: /metrics/cadvisor

test-1-setup.sh

@@ -139,7 +139,7 @@ for region in eu us; do
   sleep 3
 
   # Check if recording rule metrics exist
-  RULE_METRICS=$(curl -s "http://localhost:${PORT}/api/v1/query?query=node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate" | jq -r '.data.result | length' || echo "0")
+  RULE_METRICS=$(curl -s "http://localhost:${PORT}/api/v1/query" --data-urlencode "query=node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod=~\"pg-${region}-.*\",namespace=\"default\",node!=\"\"}" | jq -r '.data.result | length' || echo "0")
   [ "${RULE_METRICS}" -ge 1 ] && \
     pass "${region}: Recording rule metrics (${RULE_METRICS})" || fail "${region}: Recording rule metrics missing"
 done