feat: Adding PodMonitor to OTEL and OBI (#662)

Author: ruckgy

charts/kvisor/reliability-stack-installation.md

@@ -172,6 +172,19 @@ agent:
       OTEL_EBPF_SKIP_GO_SPECIFIC_TRACERS: "true"           # Skip expensive Go uprobe attachment
       OTEL_EBPF_BPF_HIGH_REQUEST_VOLUME: "true"            # Ring-buffer mode for high-throughput nodes
 
+    # OBI-specific settings
+    obi:
+      # Internal metrics — exposes OBI's own health via Prometheus endpoint
+      internalMetrics:
+        enabled: false
+        port: 6061                   # HTTP port for internal metrics
+        path: "/internal/metrics"    # Scrape path
+        podMonitor:
+          enabled: false
+          labels: {}
+          interval: 30s
+          scrapeTimeout: 10s
+
     # OTel Collector sidecar (agent)
     collector:
       enabled: true
@@ -187,6 +200,12 @@ agent:
       clickhouseExporter:
         enabled: true
         address: "tcp://castai-kvisor-clickhouse.castai-agent.svc.cluster.local:9000"
+      # PodMonitor for Prometheus Operator (scrapes collector self-metrics on port 8888)
+      podMonitor:
+        enabled: false
+        labels: {}           # Extra labels for Prometheus Operator selector filtering
+        interval: 30s
+        scrapeTimeout: 10s
 
 controller:
   reliabilityMetrics:
@@ -200,6 +219,12 @@ controller:
         limits:
           memory: 512Mi
       prometheusPort: 9401
+      # PodMonitor for Prometheus Operator (scrapes collector self-metrics on port 8889)
+      podMonitor:
+        enabled: false
+        labels: {}
+        interval: 30s
+        scrapeTimeout: 10s
 
 # Subchart (reliability-metrics-ch-exporter)
 reliabilityMetrics:
@@ -252,13 +277,20 @@ reliabilityMetrics:
     grpcAddr: ""           # Defaults to reliabilityMetrics.castai.grpcAddr if empty
     image:
       repository: ghcr.io/castai/kvisor/reliability-metrics-ch-exporter
-      tag: "v0.3.6"
+      tag: "v0.3.7"
     resources:
       requests:
         cpu: 50m
         memory: 64Mi
       limits:
         memory: 128Mi
+    # PodMonitor for Prometheus Operator (scrapes exporter metrics on port 8080)
+    podMonitor:
+      enabled: false
+      labels: {}
+      selectorLabels: {}    # Override auto-detected pod selector
+      interval: 30s
+      scrapeTimeout: 10s
 
   # External ClickHouse (alternative to install.enabled)
   external:
@@ -418,6 +450,91 @@ Approximate per-component resource consumption:
 
 For clusters with 30+ nodes or high-cardinality workloads, consider increasing the agent OTel Collector memory limit above 256 MiB.
 
+## Monitoring with Prometheus Operator
+
+If your cluster runs [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator), you can create PodMonitor resources to scrape the reliability metrics components automatically.
+
+### Available PodMonitors
+
+| Component | Values path | Metrics port | Key metrics |
+|-----------|------------|-------------|-------------|
+| Agent OTel Collector | `agent.reliabilityMetrics.collector.podMonitor` | 8888 | `otelcol_receiver_accepted_metric_points`, `otelcol_exporter_sent_metric_points`, `otelcol_processor_dropped_metric_points`, queue sizes |
+| OBI (eBPF instrumenter) | `agent.reliabilityMetrics.obi.internalMetrics` | 6061 | Instrumented process count, eBPF map usage, Go runtime stats |
+| Controller OTel Collector | `controller.reliabilityMetrics.collector.podMonitor` | 8889 | Same as agent collector (k8s_cluster receiver pipeline) |
+| ch-exporter | `reliabilityMetrics.exporter.podMonitor` | 8080 | Export throughput, ClickHouse query latency, gRPC send errors |
+
+**Note:** OBI internal metrics require two enable flags: `agent.reliabilityMetrics.obi.internalMetrics.enabled` (exposes the `/internal/metrics` endpoint) and `agent.reliabilityMetrics.obi.internalMetrics.podMonitor.enabled` (creates the PodMonitor).
+
+### Enable All PodMonitors
+
+Add these to your values file to enable scraping of all components:
+
+```yaml
+agent:
+  reliabilityMetrics:
+    # OBI settings
+    obi:
+      internalMetrics:
+        enabled: true
+        podMonitor:
+          enabled: true
+          labels:
+            release: prometheus
+    # Agent OTel Collector
+    collector:
+      podMonitor:
+        enabled: true
+        labels:
+          release: prometheus   # Match your Prometheus Operator's serviceMonitorSelector
+
+controller:
+  reliabilityMetrics:
+    collector:
+      podMonitor:
+        enabled: true
+        labels:
+          release: prometheus
+
+reliabilityMetrics:
+  exporter:
+    podMonitor:
+      enabled: true
+      labels:
+        release: prometheus
+```
+
+Or via `--set` flags:
+
+```bash
+helm upgrade castai-kvisor castai-helm/castai-kvisor \
+  -n castai-agent \
+  --reset-then-reuse-values \
+  --set agent.reliabilityMetrics.obi.internalMetrics.enabled=true \
+  --set agent.reliabilityMetrics.obi.internalMetrics.podMonitor.enabled=true \
+  --set agent.reliabilityMetrics.collector.podMonitor.enabled=true \
+  --set controller.reliabilityMetrics.collector.podMonitor.enabled=true \
+  --set reliabilityMetrics.exporter.podMonitor.enabled=true
+```
+
+### Prometheus Operator Label Matching
+
+Prometheus Operator uses label selectors to decide which PodMonitors to pick up. If your Prometheus is configured with a `podMonitorSelector` (e.g., `release: prometheus`), add matching labels:
+
+```yaml
+podMonitor:
+  enabled: true
+  labels:
+    release: prometheus
+```
+
+To check what selector your Prometheus uses:
+
+```bash
+kubectl get prometheus -A -o jsonpath='{.items[*].spec.podMonitorSelector}'
+```
+
+An empty `podMonitorSelector` means Prometheus picks up all PodMonitors in its namespace.
+
 ## Troubleshooting
 
 ### OBI: "data refused due to high memory usage"

charts/kvisor/templates/agent.yaml

@@ -189,10 +189,26 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+          {{- if (dig "reliabilityMetrics" "obi" "internalMetrics" "enabled" false .Values.agent) }}
+            - name: OTEL_EBPF_INTERNAL_METRICS_EXPORTER
+              value: "prometheus"
+            - name: OTEL_EBPF_INTERNAL_METRICS_PROMETHEUS_PORT
+              value: {{ .Values.agent.reliabilityMetrics.obi.internalMetrics.port | default 6061 | quote }}
+            {{- with (dig "reliabilityMetrics" "obi" "internalMetrics" "path" "" .Values.agent) }}
+            - name: OTEL_EBPF_INTERNAL_METRICS_PROMETHEUS_PATH
+              value: {{ . | quote }}
+            {{- end }}
+          {{- end }}
           {{- range $k, $v := .Values.agent.reliabilityMetrics.env }}
             - name: {{ $k }}
               value: "{{ $v }}"
           {{- end }}
+          {{- if (dig "reliabilityMetrics" "obi" "internalMetrics" "enabled" false .Values.agent) }}
+          ports:
+            - containerPort: {{ .Values.agent.reliabilityMetrics.obi.internalMetrics.port | default 6061 }}
+              name: obi-metrics
+              protocol: TCP
+          {{- end }}
           volumeMounts:
             - name: var-run-obi
               mountPath: /var/run/beyla

charts/kvisor/templates/reliability-pod-monitors.yaml

@@ -0,0 +1,94 @@
+{{/*
+PodMonitors for the reliability metrics stack components.
+
+1. Agent OTel Collector — self-metrics (port 8888):
+     otelcol_receiver_accepted_metric_points, otelcol_exporter_sent_metric_points,
+     otelcol_processor_dropped_metric_points, queue sizes, Go runtime metrics
+2. OBI (eBPF instrumentation) — internal metrics (port 6061 default):
+     Instrumented process count, eBPF map usage, Go runtime stats
+3. Controller OTel Collector — self-metrics (port 8889):
+     Same as agent collector for the k8s_cluster receiver pipeline
+*/}}
+{{- if and (dig "reliabilityMetrics" "enabled" false .Values.agent) (dig "reliabilityMetrics" "collector" "enabled" false .Values.agent) (dig "reliabilityMetrics" "collector" "podMonitor" "enabled" false .Values.agent) }}
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: {{ include "kvisor.agent.fullname" . }}-otel-collector
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "kvisor.labels" . | nindent 4 }}
+    app.kubernetes.io/component: otel-collector
+    {{- with .Values.agent.reliabilityMetrics.collector.podMonitor.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+  podMetricsEndpoints:
+    - port: otel-metrics
+      path: /metrics
+      scheme: http
+      honorLabels: true
+      interval: {{ .Values.agent.reliabilityMetrics.collector.podMonitor.interval | default "30s" }}
+      scrapeTimeout: {{ .Values.agent.reliabilityMetrics.collector.podMonitor.scrapeTimeout | default "10s" }}
+  selector:
+    matchLabels:
+      {{- include "kvisor.agent.selectorLabels" . | nindent 6 }}
+{{- end }}
+---
+{{- if and (dig "reliabilityMetrics" "enabled" false .Values.agent) (dig "reliabilityMetrics" "obi" "internalMetrics" "enabled" false .Values.agent) (dig "reliabilityMetrics" "obi" "internalMetrics" "podMonitor" "enabled" false .Values.agent) }}
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: {{ include "kvisor.agent.fullname" . }}-obi
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "kvisor.labels" . | nindent 4 }}
+    app.kubernetes.io/component: obi
+    {{- with .Values.agent.reliabilityMetrics.obi.internalMetrics.podMonitor.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+  podMetricsEndpoints:
+    - port: obi-metrics
+      path: {{ .Values.agent.reliabilityMetrics.obi.internalMetrics.path | default "/internal/metrics" }}
+      scheme: http
+      honorLabels: true
+      interval: {{ .Values.agent.reliabilityMetrics.obi.internalMetrics.podMonitor.interval | default "30s" }}
+      scrapeTimeout: {{ .Values.agent.reliabilityMetrics.obi.internalMetrics.podMonitor.scrapeTimeout | default "10s" }}
+  selector:
+    matchLabels:
+      {{- include "kvisor.agent.selectorLabels" . | nindent 6 }}
+{{- end }}
+---
+{{- if and (dig "reliabilityMetrics" "enabled" false .Values.controller) (dig "reliabilityMetrics" "collector" "enabled" false .Values.controller) (dig "reliabilityMetrics" "collector" "podMonitor" "enabled" false .Values.controller) }}
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: {{ include "kvisor.controller.fullname" . }}-otel-collector
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "kvisor.labels" . | nindent 4 }}
+    app.kubernetes.io/component: otel-collector
+    {{- with .Values.controller.reliabilityMetrics.collector.podMonitor.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+  podMetricsEndpoints:
+    - port: k8s-metrics
+      path: /metrics
+      scheme: http
+      honorLabels: true
+      interval: {{ .Values.controller.reliabilityMetrics.collector.podMonitor.interval | default "30s" }}
+      scrapeTimeout: {{ .Values.controller.reliabilityMetrics.collector.podMonitor.scrapeTimeout | default "10s" }}
+  selector:
+    matchLabels:
+      {{- include "kvisor.controller.selectorLabels" . | nindent 6 }}
+{{- end }}

charts/kvisor/values.yaml

@@ -226,6 +226,21 @@ agent:
       OTEL_EBPF_BPF_HIGH_REQUEST_VOLUME: "true"
     # Container security context override (if empty, uses unprivileged defaults with fine-grained capabilities).
     containerSecurityContext: {}
+    # --- OBI-specific settings ---
+    obi:
+      # Internal metrics — exposes OBI's own health via a Prometheus endpoint.
+      # Metrics include: instrumented process count, eBPF map usage, Go runtime stats.
+      # Disabled by default. Set port > 0 to enable.
+      internalMetrics:
+        enabled: false
+        port: 6061
+        path: "/internal/metrics"
+        # PodMonitor for Prometheus Operator — scrapes OBI internal metrics.
+        podMonitor:
+          enabled: false
+          labels: {}
+          interval: 30s
+          scrapeTimeout: 10s
 
     # --- OTel Collector sidecar settings ---
     # Receives OTLP from OBI, applies golden signal filtering, cardinality control,
@@ -248,6 +263,15 @@ agent:
       clickhouseExporter:
         enabled: true
         address: "tcp://castai-kvisor-clickhouse.castai-agent.svc.cluster.local:9000"
+      # PodMonitor for Prometheus Operator — scrapes collector self-metrics (port 8888).
+      # Provides: otelcol_receiver_accepted_metric_points, otelcol_exporter_sent_metric_points,
+      # otelcol_processor_dropped_metric_points, queue sizes, Go runtime metrics.
+      podMonitor:
+        enabled: false
+        # Extra labels on the PodMonitor metadata (for Prometheus Operator selector filtering).
+        labels: {}
+        interval: 30s
+        scrapeTimeout: 10s
 
 controller:
   enabled: true
@@ -395,8 +419,13 @@ controller:
         address: "tcp://castai-kvisor-clickhouse.castai-agent.svc.cluster.local:9000"
       # Port for the collector's Prometheus exporter (different from agent collector's 9400).
       prometheusPort: 9401
-      # Labels to add to the PodMonitor (e.g., for Prometheus Operator selector filtering).
-      podMonitorLabels: {}
+      # PodMonitor for Prometheus Operator — scrapes collector self-metrics (port 8889).
+      podMonitor:
+        enabled: false
+        # Extra labels on the PodMonitor metadata (for Prometheus Operator selector filtering).
+        labels: {}
+        interval: 30s
+        scrapeTimeout: 10s
 
 eventGenerator:
   enabled: false
@@ -476,4 +505,4 @@ reliabilityMetrics:
   exporter:
     image:
       repository: us-docker.pkg.dev/castai-hub/library/reliability-metrics-ch-exporter
-      tag: "v0.3.6"
+      tag: "v0.3.7"