Prometheus is not picking up the pod monitor #5735

veerendra2 · 2021-10-18T14:17:54Z

veerendra2
Oct 18, 2021

Openshift v3.11.0+cbab8ee-94(K8s v1.11.0+d4cacc0)
Operators Version: 0.20.1

Kafka Cluster

---
apiVersion: kafka.strimzi.io/v1beta1
kind: Kafka
metadata:
  name: sirius-dev
  namespace: kafka
spec:
  kafka:
    version: 2.6.0
    replicas: 3
    template:
      pod:
        affinity:
          podAntiAffinity:
            requiredDuringSchedulingIgnoredDuringExecution:
              - labelSelector:
                  matchExpressions:
                    - key: strimzi.io/name
                      operator: In
                      values:
                        - sirius-dev-kafka
                topologyKey: "kubernetes.io/hostname"
    listeners:
      - name: tlsinternal
        port: 9093
        type: internal
        tls: true
        authentication:
          type: tls
      - name: tlsroute
        port: 9094
        type: route
        tls: true
        authentication:
          type: tls
    authorization:
      type: simple
    readinessProbe:
      initialDelaySeconds: 15
      timeoutSeconds: 5
    livenessProbe:
      initialDelaySeconds: 15
      timeoutSeconds: 5
    config:
      offsets.topic.replication.factor: 2
      transaction.state.log.replication.factor: 1
      transaction.state.log.min.isr: 1
      log.message.format.version: "2.6"
    storage:
      type: jbod
      volumes:
        - id: 0
          type: persistent-claim
          size: 256Gi
          deleteClaim: false
    metrics:
      # Inspired by config from Kafka 2.0.0 example rules:
      # https://github.com/prometheus/jmx_exporter/blob/master/example_configs/kafka-2_0_0.yml
      lowercaseOutputName: true
      rules:
        # Special cases and very specific rules
        - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value
          name: kafka_server_$1_$2
          type: GAUGE
          labels:
            clientId: "$3"
            topic: "$4"
            partition: "$5"
        - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value
          name: kafka_server_$1_$2
          type: GAUGE
          labels:
            clientId: "$3"
            broker: "$4:$5"
        - pattern: kafka.server<type=(.+), cipher=(.+), protocol=(.+), listener=(.+), networkProcessor=(.+)><>connections
          name: kafka_server_$1_connections_tls_info
          type: GAUGE
          labels:
            listener: "$2"
            networkProcessor: "$3"
            protocol: "$4"
            cipher: "$5"
        - pattern: kafka.server<type=(.+), clientSoftwareName=(.+), clientSoftwareVersion=(.+), listener=(.+), networkProcessor=(.+)><>connections
          name: kafka_server_$1_connections_software
          type: GAUGE
          labels:
            clientSoftwareName: "$2"
            clientSoftwareVersion: "$3"
            listener: "$4"
            networkProcessor: "$5"
        - pattern: "kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+):"
          name: kafka_server_$1_$4
          type: GAUGE
          labels:
            listener: "$2"
            networkProcessor: "$3"
        - pattern: kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+)
          name: kafka_server_$1_$4
          type: GAUGE
          labels:
            listener: "$2"
            networkProcessor: "$3"
        # Some percent metrics use MeanRate attribute
        # Ex) kafka.server<type=(KafkaRequestHandlerPool), name=(RequestHandlerAvgIdlePercent)><>MeanRate
        - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>MeanRate
          name: kafka_$1_$2_$3_percent
          type: GAUGE
        # Generic gauges for percents
        - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>Value
          name: kafka_$1_$2_$3_percent
          type: GAUGE
        - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*, (.+)=(.+)><>Value
          name: kafka_$1_$2_$3_percent
          type: GAUGE
          labels:
            "$4": "$5"
        # Generic per-second counters with 0-2 key/value pairs
        - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+), (.+)=(.+)><>Count
          name: kafka_$1_$2_$3_total
          type: COUNTER
          labels:
            "$4": "$5"
            "$6": "$7"
        - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+)><>Count
          name: kafka_$1_$2_$3_total
          type: COUNTER
          labels:
            "$4": "$5"
        - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*><>Count
          name: kafka_$1_$2_$3_total
          type: COUNTER
        # Generic gauges with 0-2 key/value pairs
        - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Value
          name: kafka_$1_$2_$3
          type: GAUGE
          labels:
            "$4": "$5"
            "$6": "$7"
        - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Value
          name: kafka_$1_$2_$3
          type: GAUGE
          labels:
            "$4": "$5"
        - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Value
          name: kafka_$1_$2_$3
          type: GAUGE
        # Emulate Prometheus 'Summary' metrics for the exported 'Histogram's.
        # Note that these are missing the '_sum' metric!
        - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Count
          name: kafka_$1_$2_$3_count
          type: COUNTER
          labels:
            "$4": "$5"
            "$6": "$7"
        - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\d+)thPercentile
          name: kafka_$1_$2_$3
          type: GAUGE
          labels:
            "$4": "$5"
            "$6": "$7"
            quantile: "0.$8"
        - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Count
          name: kafka_$1_$2_$3_count
          type: COUNTER
          labels:
            "$4": "$5"
        - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\d+)thPercentile
          name: kafka_$1_$2_$3
          type: GAUGE
          labels:
            "$4": "$5"
            quantile: "0.$6"
        - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Count
          name: kafka_$1_$2_$3_count
          type: COUNTER
        - pattern: kafka.(\w+)<type=(.+), name=(.+)><>(\d+)thPercentile
          name: kafka_$1_$2_$3
          type: GAUGE
          labels:
            quantile: "0.$4"
  zookeeper:
    replicas: 3
    template:
      pod:
        affinity:
          podAntiAffinity:
            requiredDuringSchedulingIgnoredDuringExecution:
              - labelSelector:
                  matchExpressions:
                    - key: strimzi.io/name
                      operator: In
                      values:
                        - sirius-dev-zookeeper
                topologyKey: "kubernetes.io/hostname"
    readinessProbe:
      initialDelaySeconds: 15
      timeoutSeconds: 5
    livenessProbe:
      initialDelaySeconds: 15
      timeoutSeconds: 5
    storage:
      type: persistent-claim
      size: 64Gi
      deleteClaim: false
    metrics:
      # Inspired by Zookeeper rules
      # https://github.com/prometheus/jmx_exporter/blob/master/example_configs/zookeeper.yaml
      lowercaseOutputName: true
      rules:
        # replicated Zookeeper
        - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+)><>(\\w+)"
          name: "zookeeper_$2"
          type: GAUGE
        - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+)><>(\\w+)"
          name: "zookeeper_$3"
          type: GAUGE
          labels:
            replicaId: "$2"
        - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+), name2=(\\w+)><>(Packets\\w+)"
          name: "zookeeper_$4"
          type: COUNTER
          labels:
            replicaId: "$2"
            memberType: "$3"
        - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+), name2=(\\w+)><>(\\w+)"
          name: "zookeeper_$4"
          type: GAUGE
          labels:
            replicaId: "$2"
            memberType: "$3"
        - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+), name2=(\\w+), name3=(\\w+)><>(\\w+)"
          name: "zookeeper_$4_$5"
          type: GAUGE
          labels:
            replicaId: "$2"
            memberType: "$3"
  entityOperator:
    topicOperator: {}
    userOperator: {}
  kafkaExporter:
    topicRegex: ".*"
    groupRegex: ".*"

All pods are up and running

$ oc get pods
NAME                                          READY     STATUS    RESTARTS   AGE
sirius-dev-entity-operator-6bfbd5bb6f-8vn6l   3/3       Running   0          4d
sirius-dev-kafka-0                            1/1       Running   0          4d
sirius-dev-kafka-1                            1/1       Running   0          4d
sirius-dev-kafka-2                            1/1       Running   0          4d
sirius-dev-kafka-exporter-c67976b6b-4k8s8     1/1       Running   0          4d
sirius-dev-zookeeper-0                        1/1       Running   0          4d
sirius-dev-zookeeper-1                        1/1       Running   0          4d
sirius-dev-zookeeper-2                        1/1       Running   0          4d
strimzi-cluster-operator-6dcbc4bcc-qjl5b      1/1       Running   2          4d

PodMonitor CRD

My cluster doesn't have PodMonitor CRD installed. So install from here

$ oc get crd | grep podmonitor
podmonitors.monitoring.coreos.com                     2021-10-18T13:00:24Z

Pod Monitor Deploy

---
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
  name: cluster-operator-metrics-dev
  labels:
    monitoring-instance: prometheus-alm
spec:
  selector:
    matchLabels:
      strimzi.io/kind: cluster-operator
  namespaceSelector:
    matchNames:
      - kafka
  podMetricsEndpoints:
    - path: /metrics
      port: http
---
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
  name: entity-operator-metrics-dev
  labels:
    monitoring-instance: prometheus-alm
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: entity-operator
  namespaceSelector:
    matchNames:
      - kafka
  podMetricsEndpoints:
    - path: /metrics
      port: healthcheck
---
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
  name: kafka-resources-metrics-dev
  labels:
    monitoring-instance: prometheus-alm
spec:
  selector:
    matchExpressions:
      - key: "strimzi.io/kind"
        operator: In
        values:
          [
            "Kafka",
            "KafkaConnect"
#            "KafkaConnectS2I",
#            "KafkaMirrorMaker",
#            "KafkaMirrorMaker2",
          ]
  namespaceSelector:
    matchNames:
      - kafka
  podMetricsEndpoints:
    - path: /metrics
      port: tcp-prometheus
      relabelings:
        - separator: ;
          regex: __meta_kubernetes_pod_label_(.+)
          replacement: $1
          action: labelmap
        - sourceLabels: [__meta_kubernetes_namespace]
          separator: ;
          regex: (.*)
          targetLabel: namespace
          replacement: $1
          action: replace
        - sourceLabels: [__meta_kubernetes_pod_name]
          separator: ;
          regex: (.*)
          targetLabel: kubernetes_pod_name
          replacement: $1
          action: replace
        - sourceLabels: [__meta_kubernetes_pod_node_name]
          separator: ;
          regex: (.*)
          targetLabel: node_name
          replacement: $1
          action: replace
        - sourceLabels: [__meta_kubernetes_pod_host_ip]
          separator: ;
          regex: (.*)
          targetLabel: node_ip
          replacement: $1
          action: replace
#---
#apiVersion: monitoring.coreos.com/v1
#kind: PodMonitor
#metadata:
#  name: bridge-metrics
#  labels:
#    app: strimzi
#spec:
#  selector:
#    matchLabels:
#      strimzi.io/kind: KafkaBridge
#  namespaceSelector:
#    matchNames:
#      - kafka
#  podMetricsEndpoints:
#    - path: /metrics
#      port: rest-api

Added labels to Prometheus

apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
...
  podMonitorSelector:
    matchLabels:
      monitoring-instance: prometheus-alm
...

Not sure what did I missed? Ednpoints are not showingup in my service discovery

Answered by veerendra2

Nov 3, 2021

Below is the scrap config I directly added to promethes config(Collected from podmonitor)

- job_name: kafka-resources-metrics
  scrape_interval: 30s
  scrape_timeout: 10s
  metrics_path: /metrics
  scheme: http
  relabel_configs:
    - source_labels: [job]
      separator: ;
      regex: (.*)
      target_label: __tmp_prometheus_job_name
      replacement: $1
      action: replace
    - source_labels: [__meta_kubernetes_pod_label_strimzi_io_kind]
      separator: ;
      regex: Kafka|KafkaConnect|KafkaConnectS2I|KafkaMirrorMaker|KafkaMirrorMaker2
      replacement: $1
      action: keep
    - source_labels: [__meta_kubernetes_pod_container_port_name]
      separator: ;
      regex: tcp-pr…

View full answer

scholzj · 2021-10-18T15:00:30Z

scholzj
Oct 18, 2021
Maintainer

These are real old versions. We do not support Kubernetes 1.11 for a long time. Not sure anyone remembers how it used to work there. @sknot-rh or @ppatierno probably used it back then, but not sure they will know the answer.

3 replies

veerendra2 Oct 22, 2021
Author

Any help?

sknot-rh Oct 25, 2021
Maintainer

What version of the Prometheus operator do you use? Are other PodMonitors picked?

veerendra2 Oct 25, 2021
Author

No, we are not using podmonitor at all, this is first PodMonitor. I have checked again our cluster RBAC configs, all has mentioned only ServiceMonitor but not Podmonitor.

veerendra2 · 2021-11-03T09:58:30Z

veerendra2
Nov 3, 2021
Author

Below is the scrap config I directly added to promethes config(Collected from podmonitor)

- job_name: kafka-resources-metrics
  scrape_interval: 30s
  scrape_timeout: 10s
  metrics_path: /metrics
  scheme: http
  relabel_configs:
    - source_labels: [job]
      separator: ;
      regex: (.*)
      target_label: __tmp_prometheus_job_name
      replacement: $1
      action: replace
    - source_labels: [__meta_kubernetes_pod_label_strimzi_io_kind]
      separator: ;
      regex: Kafka|KafkaConnect|KafkaConnectS2I|KafkaMirrorMaker|KafkaMirrorMaker2
      replacement: $1
      action: keep
    - source_labels: [__meta_kubernetes_pod_container_port_name]
      separator: ;
      regex: tcp-prometheus
      replacement: $1
      action: keep
    - source_labels: [__meta_kubernetes_namespace]
      separator: ;
      regex: (.*)
      target_label: namespace
      replacement: $1
      action: replace
    - source_labels: [__meta_kubernetes_pod_container_name]
      separator: ;
      regex: (.*)
      target_label: container
      replacement: $1
      action: replace
    - source_labels: [__meta_kubernetes_pod_name]
      separator: ;
      regex: (.*)
      target_label: pod
      replacement: $1
      action: replace
    - separator: ;
      regex: (.*)
      target_label: job
      replacement: default/kafka-resources-metrics
      action: replace
    - separator: ;
      regex: (.*)
      target_label: endpoint
      replacement: tcp-prometheus
      action: replace
    - separator: ;
      regex: __meta_kubernetes_pod_label_(.+)
      replacement: $1
      action: labelmap
    - source_labels: [__meta_kubernetes_namespace]
      separator: ;
      regex: (.*)
      target_label: namespace
      replacement: $1
      action: replace
    - source_labels: [__meta_kubernetes_pod_name]
      separator: ;
      regex: (.*)
      target_label: kubernetes_pod_name
      replacement: $1
      action: replace
    - source_labels: [__meta_kubernetes_pod_node_name]
      separator: ;
      regex: (.*)
      target_label: node_name
      replacement: $1
      action: replace
    - source_labels: [__meta_kubernetes_pod_host_ip]
      separator: ;
      regex: (.*)
      target_label: node_ip
      replacement: $1
      action: replace
    - source_labels: [__address__]
      separator: ;
      regex: (.*)
      modulus: 1
      target_label: __tmp_hash
      replacement: $1
      action: hashmod
    - source_labels: [__tmp_hash]
      separator: ;
      regex: "0"
      replacement: $1
      action: keep
  kubernetes_sd_configs:
    - api_server: null
      role: pod
      namespaces:
        names:
          - kafka

- job_name: entity-operator-metrics
  scrape_interval: 30s
  scrape_timeout: 10s
  metrics_path: /metrics
  scheme: http
  relabel_configs:
    - source_labels: [job]
      separator: ;
      regex: (.*)
      target_label: __tmp_prometheus_job_name
      replacement: $1
      action: replace
    - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
      separator: ;
      regex: entity-operator
      replacement: $1
      action: keep
    - source_labels: [__meta_kubernetes_pod_container_port_name]
      separator: ;
      regex: healthcheck
      replacement: $1
      action: keep
    - source_labels: [__meta_kubernetes_namespace]
      separator: ;
      regex: (.*)
      target_label: namespace
      replacement: $1
      action: replace
    - source_labels: [__meta_kubernetes_pod_container_name]
      separator: ;
      regex: (.*)
      target_label: container
      replacement: $1
      action: replace
    - source_labels: [__meta_kubernetes_pod_name]
      separator: ;
      regex: (.*)
      target_label: pod
      replacement: $1
      action: replace
    - separator: ;
      regex: (.*)
      target_label: job
      replacement: default/entity-operator-metrics
      action: replace
    - separator: ;
      regex: (.*)
      target_label: endpoint
      replacement: healthcheck
      action: replace
    - source_labels: [__address__]
      separator: ;
      regex: (.*)
      modulus: 1
      target_label: __tmp_hash
      replacement: $1
      action: hashmod
    - source_labels: [__tmp_hash]
      separator: ;
      regex: "0"
      replacement: $1
      action: keep
  kubernetes_sd_configs:
    - api_server: null
      role: pod
      namespaces:
        names:
          - kafka

- job_name: cluster-operator-metrics
  scrape_interval: 30s
  scrape_timeout: 10s
  metrics_path: /metrics
  scheme: http
  relabel_configs:
    - source_labels: [job]
      separator: ;
      regex: (.*)
      target_label: __tmp_prometheus_job_name
      replacement: $1
      action: replace
    - source_labels: [__meta_kubernetes_pod_label_strimzi_io_kind]
      separator: ;
      regex: cluster-operator
      replacement: $1
      action: keep
    - source_labels: [__meta_kubernetes_pod_container_port_name]
      separator: ;
      regex: http
      replacement: $1
      action: keep
    - source_labels: [__meta_kubernetes_namespace]
      separator: ;
      regex: (.*)
      target_label: namespace
      replacement: $1
      action: replace
    - source_labels: [__meta_kubernetes_pod_container_name]
      separator: ;
      regex: (.*)
      target_label: container
      replacement: $1
      action: replace
    - source_labels: [__meta_kubernetes_pod_name]
      separator: ;
      regex: (.*)
      target_label: pod
      replacement: $1
      action: replace
    - separator: ;
      regex: (.*)
      target_label: job
      replacement: default/cluster-operator-metrics
      action: replace
    - separator: ;
      regex: (.*)
      target_label: endpoint
      replacement: http
      action: replace
    - source_labels: [__address__]
      separator: ;
      regex: (.*)
      modulus: 1
      target_label: __tmp_hash
      replacement: $1
      action: hashmod
    - source_labels: [__tmp_hash]
      separator: ;
      regex: "0"
      replacement: $1
      action: keep
  kubernetes_sd_configs:
    - api_server: null
      role: pod
      namespaces:
        names:
          - kafka

0 replies

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Strimzi

Prometheus is not picking up the pod monitor #5735

Uh oh!

{{title}}

Uh oh!

Replies: 2 comments 3 replies

Uh oh!

{{title}}

Uh oh!

Uh oh!

{{title}}

Uh oh!

Uh oh!

{{title}}

Uh oh!

Uh oh!

{{title}}

Uh oh!

Uh oh!

{{title}}

Uh oh!

Select a reply

Uh oh!

Strimzi

Prometheus is not picking up the pod monitor #5735

Uh oh!

veerendra2 Oct 18, 2021

Kafka Cluster

PodMonitor CRD

Pod Monitor Deploy

Replies: 2 comments · 3 replies

Uh oh!

scholzj Oct 18, 2021 Maintainer

Uh oh!

veerendra2 Oct 22, 2021 Author

Uh oh!

sknot-rh Oct 25, 2021 Maintainer

Uh oh!

veerendra2 Oct 25, 2021 Author

Uh oh!

veerendra2 Nov 3, 2021 Author

veerendra2
Oct 18, 2021

Replies: 2 comments 3 replies

scholzj
Oct 18, 2021
Maintainer

veerendra2 Oct 22, 2021
Author

sknot-rh Oct 25, 2021
Maintainer

veerendra2 Oct 25, 2021
Author

veerendra2
Nov 3, 2021
Author