Kafka Exporter Metrics are missing

[veerendra2]

• openshift v3.11.0+cbab8ee-94(K8s v1.11.0+d4cacc0)
• Kafka Version: 2.6.0
• Operators : 0.20.1

Kafka Deployment

---
apiVersion: kafka.strimzi.io/v1beta1
kind: Kafka
metadata:
  name: sirius-dev
  namespace: kafka
spec:
  kafka:
    version: 2.6.0
    replicas: 3
    template:
      pod:
        affinity:
          podAntiAffinity:
            requiredDuringSchedulingIgnoredDuringExecution:
              - labelSelector:
                  matchExpressions:
                    - key: strimzi.io/name
                      operator: In
                      values:
                        - sirius-dev-kafka
                topologyKey: "kubernetes.io/hostname"
    listeners:
      - name: tlsroute
        port: 9094
        type: route
        tls: true
        authentication:
          type: tls
    authorization:
      type: simple
    readinessProbe:
      initialDelaySeconds: 15
      timeoutSeconds: 5
    livenessProbe:
      initialDelaySeconds: 15
      timeoutSeconds: 5
    config:
      offsets.topic.replication.factor: 2
      transaction.state.log.replication.factor: 1
      transaction.state.log.min.isr: 1
      log.message.format.version: "2.6"
    storage:
      type: jbod
      volumes:
        - id: 0
          type: persistent-claim
          size: 256Gi
          deleteClaim: false
    metrics:
      # Inspired by config from Kafka 2.0.0 example rules:
      # https://github.com/prometheus/jmx_exporter/blob/master/example_configs/kafka-2_0_0.yml
      lowercaseOutputName: true
      rules:
        # Special cases and very specific rules
        - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value
          name: kafka_server_$1_$2
          type: GAUGE
          labels:
            clientId: "$3"
            topic: "$4"
            partition: "$5"
        - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value
          name: kafka_server_$1_$2
          type: GAUGE
          labels:
            clientId: "$3"
            broker: "$4:$5"
        - pattern: kafka.server<type=(.+), cipher=(.+), protocol=(.+), listener=(.+), networkProcessor=(.+)><>connections
          name: kafka_server_$1_connections_tls_info
          type: GAUGE
          labels:
            listener: "$2"
            networkProcessor: "$3"
            protocol: "$4"
            cipher: "$5"
        - pattern: kafka.server<type=(.+), clientSoftwareName=(.+), clientSoftwareVersion=(.+), listener=(.+), networkProcessor=(.+)><>connections
          name: kafka_server_$1_connections_software
          type: GAUGE
          labels:
            clientSoftwareName: "$2"
            clientSoftwareVersion: "$3"
            listener: "$4"
            networkProcessor: "$5"
        - pattern: "kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+):"
          name: kafka_server_$1_$4
          type: GAUGE
          labels:
            listener: "$2"
            networkProcessor: "$3"
        - pattern: kafka.server<type=(.+), listener=(.+), networkProcessor=(.+)><>(.+)
          name: kafka_server_$1_$4
          type: GAUGE
          labels:
            listener: "$2"
            networkProcessor: "$3"
        # Some percent metrics use MeanRate attribute
        # Ex) kafka.server<type=(KafkaRequestHandlerPool), name=(RequestHandlerAvgIdlePercent)><>MeanRate
        - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>MeanRate
          name: kafka_$1_$2_$3_percent
          type: GAUGE
        # Generic gauges for percents
        - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*><>Value
          name: kafka_$1_$2_$3_percent
          type: GAUGE
        - pattern: kafka.(\w+)<type=(.+), name=(.+)Percent\w*, (.+)=(.+)><>Value
          name: kafka_$1_$2_$3_percent
          type: GAUGE
          labels:
            "$4": "$5"
        # Generic per-second counters with 0-2 key/value pairs
        - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+), (.+)=(.+)><>Count
          name: kafka_$1_$2_$3_total
          type: COUNTER
          labels:
            "$4": "$5"
            "$6": "$7"
        - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+)><>Count
          name: kafka_$1_$2_$3_total
          type: COUNTER
          labels:
            "$4": "$5"
        - pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*><>Count
          name: kafka_$1_$2_$3_total
          type: COUNTER
        # Generic gauges with 0-2 key/value pairs
        - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Value
          name: kafka_$1_$2_$3
          type: GAUGE
          labels:
            "$4": "$5"
            "$6": "$7"
        - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Value
          name: kafka_$1_$2_$3
          type: GAUGE
          labels:
            "$4": "$5"
        - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Value
          name: kafka_$1_$2_$3
          type: GAUGE
        # Emulate Prometheus 'Summary' metrics for the exported 'Histogram's.
        # Note that these are missing the '_sum' metric!
        - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Count
          name: kafka_$1_$2_$3_count
          type: COUNTER
          labels:
            "$4": "$5"
            "$6": "$7"
        - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\d+)thPercentile
          name: kafka_$1_$2_$3
          type: GAUGE
          labels:
            "$4": "$5"
            "$6": "$7"
            quantile: "0.$8"
        - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Count
          name: kafka_$1_$2_$3_count
          type: COUNTER
          labels:
            "$4": "$5"
        - pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\d+)thPercentile
          name: kafka_$1_$2_$3
          type: GAUGE
          labels:
            "$4": "$5"
            quantile: "0.$6"
        - pattern: kafka.(\w+)<type=(.+), name=(.+)><>Count
          name: kafka_$1_$2_$3_count
          type: COUNTER
        - pattern: kafka.(\w+)<type=(.+), name=(.+)><>(\d+)thPercentile
          name: kafka_$1_$2_$3
          type: GAUGE
          labels:
            quantile: "0.$4"
  zookeeper:
    replicas: 3
    template:
      pod:
        affinity:
          podAntiAffinity:
            requiredDuringSchedulingIgnoredDuringExecution:
              - labelSelector:
                  matchExpressions:
                    - key: strimzi.io/name
                      operator: In
                      values:
                        - sirius-dev-zookeeper
                topologyKey: "kubernetes.io/hostname"
    readinessProbe:
      initialDelaySeconds: 15
      timeoutSeconds: 5
    livenessProbe:
      initialDelaySeconds: 15
      timeoutSeconds: 5
    storage:
      type: persistent-claim
      size: 64Gi
      deleteClaim: false
    metrics:
      # Inspired by Zookeeper rules
      # https://github.com/prometheus/jmx_exporter/blob/master/example_configs/zookeeper.yaml
      lowercaseOutputName: true
      rules:
        # replicated Zookeeper
        - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+)><>(\\w+)"
          name: "zookeeper_$2"
          type: GAUGE
        - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+)><>(\\w+)"
          name: "zookeeper_$3"
          type: GAUGE
          labels:
            replicaId: "$2"
        - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+), name2=(\\w+)><>(Packets\\w+)"
          name: "zookeeper_$4"
          type: COUNTER
          labels:
            replicaId: "$2"
            memberType: "$3"
        - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+), name2=(\\w+)><>(\\w+)"
          name: "zookeeper_$4"
          type: GAUGE
          labels:
            replicaId: "$2"
            memberType: "$3"
        - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+), name2=(\\w+), name3=(\\w+)><>(\\w+)"
          name: "zookeeper_$4_$5"
          type: GAUGE
          labels:
            replicaId: "$2"
            memberType: "$3"
  entityOperator:
    topicOperator: {}
    userOperator: {}
  kafkaExporter:
    groupRegex: ".*"
    topicRegex: ".*"
    template:
      pod:
        metadata:
          annotations:
            prometheus.io/scrape: true
            prometheus.io/port: 9404
            prometheus.io/path: /metrics

Up and running

$ oc get pods
NAME                                          READY     STATUS    RESTARTS   AGE
java-kafka-consumer-9b646844b-qzmm7           1/1       Running   0          19h
java-kafka-producer-768c788d74-z6m5d          1/1       Running   0          19h
sirius-dev-entity-operator-6bfbd5bb6f-l7s4f   3/3       Running   0          1d
sirius-dev-kafka-0                            1/1       Running   0          1d
sirius-dev-kafka-1                            1/1       Running   0          1d
sirius-dev-kafka-2                            1/1       Running   0          1d
sirius-dev-kafka-exporter-69bdf874bb-bjpz8    1/1       Running   0          19h
sirius-dev-zookeeper-0                        1/1       Running   0          1d
sirius-dev-zookeeper-1                        1/1       Running   0          1d
sirius-dev-zookeeper-2                        1/1       Running   0          1d
strimzi-cluster-operator-6dcbc4bcc-qjl5b      1/1       Running   7          13d

Exporter Metrics

$ oc exec -it sirius-dev-kafka-exporter-69bdf874bb-bjpz8 curl localhost:9404/metrics | grep kafka
# HELP kafka_brokers Number of Brokers in the Kafka Cluster.
# TYPE kafka_brokers gauge
kafka_brokers 3
# HELP kafka_exporter_build_info A metric with a constant '1' value labeled by version, revision, branch, and goversion from which kafka_exporter was built.
# TYPE kafka_exporter_build_info gauge
kafka_exporter_build_info{branch="HEAD",goversion="go1.10.3",revision="830660212e6c109e69dcb1cb58f5159fe3b38903",version="1.2.0"} 1

------------------------------------------------------------------------------------------------------------------------------

$ oc exec -it sirius-dev-kafka-exporter-69bdf874bb-bjpz8 curl localhost:9404/metrics
# HELP go_gc_duration_seconds A summary of the GC invocation durations.
# TYPE go_gc_duration_seconds summary
go_gc_duration_seconds{quantile="0"} 9.23e-05
go_gc_duration_seconds{quantile="0.25"} 0.000145501
go_gc_duration_seconds{quantile="0.5"} 0.000174401
go_gc_duration_seconds{quantile="0.75"} 0.000210901
go_gc_duration_seconds{quantile="1"} 0.005048532
go_gc_duration_seconds_sum 3.739591952
go_gc_duration_seconds_count 14039
# HELP go_goroutines Number of goroutines that currently exist.
# TYPE go_goroutines gauge
go_goroutines 12
# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
# TYPE go_memstats_alloc_bytes gauge
go_memstats_alloc_bytes 1.916624e+06
# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
# TYPE go_memstats_alloc_bytes_total counter
go_memstats_alloc_bytes_total 2.2564163008e+10
# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table.
# TYPE go_memstats_buck_hash_sys_bytes gauge
go_memstats_buck_hash_sys_bytes 1.701926e+06
# HELP go_memstats_frees_total Total number of frees.
# TYPE go_memstats_frees_total counter
go_memstats_frees_total 9.4412171e+07
# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata.
# TYPE go_memstats_gc_sys_bytes gauge
go_memstats_gc_sys_bytes 438272
# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use.
# TYPE go_memstats_heap_alloc_bytes gauge
go_memstats_heap_alloc_bytes 1.916624e+06
# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used.
# TYPE go_memstats_heap_idle_bytes gauge
go_memstats_heap_idle_bytes 2.777088e+06
# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use.
# TYPE go_memstats_heap_inuse_bytes gauge
go_memstats_heap_inuse_bytes 3.219456e+06
# HELP go_memstats_heap_objects Number of allocated objects.
# TYPE go_memstats_heap_objects gauge
go_memstats_heap_objects 7671
# HELP go_memstats_heap_released_bytes_total Total number of heap bytes released to OS.
# TYPE go_memstats_heap_released_bytes_total counter
go_memstats_heap_released_bytes_total 458752
# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system.
# TYPE go_memstats_heap_sys_bytes gauge
go_memstats_heap_sys_bytes 5.996544e+06
# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection.
# TYPE go_memstats_last_gc_time_seconds gauge
go_memstats_last_gc_time_seconds 1.6353255786407757e+09
# HELP go_memstats_lookups_total Total number of pointer lookups.
# TYPE go_memstats_lookups_total counter
go_memstats_lookups_total 795812
# HELP go_memstats_mallocs_total Total number of mallocs.
# TYPE go_memstats_mallocs_total counter
go_memstats_mallocs_total 9.4419842e+07
# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures.
# TYPE go_memstats_mcache_inuse_bytes gauge
go_memstats_mcache_inuse_bytes 13888
# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system.
# TYPE go_memstats_mcache_sys_bytes gauge
go_memstats_mcache_sys_bytes 16384
# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures.
# TYPE go_memstats_mspan_inuse_bytes gauge
go_memstats_mspan_inuse_bytes 50160
# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system.
# TYPE go_memstats_mspan_sys_bytes gauge
go_memstats_mspan_sys_bytes 81920
# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place.
# TYPE go_memstats_next_gc_bytes gauge
go_memstats_next_gc_bytes 4.194304e+06
# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations.
# TYPE go_memstats_other_sys_bytes gauge
go_memstats_other_sys_bytes 2.060498e+06
# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator.
# TYPE go_memstats_stack_inuse_bytes gauge
go_memstats_stack_inuse_bytes 1.343488e+06
# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator.
# TYPE go_memstats_stack_sys_bytes gauge
go_memstats_stack_sys_bytes 1.343488e+06
# HELP go_memstats_sys_bytes Number of bytes obtained by system. Sum of all system allocations.
# TYPE go_memstats_sys_bytes gauge
go_memstats_sys_bytes 1.1639032e+07
# HELP kafka_brokers Number of Brokers in the Kafka Cluster.
# TYPE kafka_brokers gauge
kafka_brokers 3
# HELP kafka_exporter_build_info A metric with a constant '1' value labeled by version, revision, branch, and goversion from which kafka_exporter was built.
# TYPE kafka_exporter_build_info gauge
kafka_exporter_build_info{branch="HEAD",goversion="go1.10.3",revision="830660212e6c109e69dcb1cb58f5159fe3b38903",version="1.2.0"} 1
# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
# TYPE process_cpu_seconds_total counter
process_cpu_seconds_total 541.05
# HELP process_max_fds Maximum number of open file descriptors.
# TYPE process_max_fds gauge
process_max_fds 1.048576e+06
# HELP process_open_fds Number of open file descriptors.
# TYPE process_open_fds gauge
process_open_fds 10
# HELP process_resident_memory_bytes Resident memory size in bytes.
# TYPE process_resident_memory_bytes gauge
process_resident_memory_bytes 1.0883072e+07
# HELP process_start_time_seconds Start time of the process since unix epoch in seconds.
# TYPE process_start_time_seconds gauge
process_start_time_seconds 1.63525536622e+09
# HELP process_virtual_memory_bytes Virtual memory size in bytes.
# TYPE process_virtual_memory_bytes gauge
process_virtual_memory_bytes 1.998848e+07

Is there any config missing?

[scholzj]

I'm not sure I follow the question => the curl commands show the metrics from Kafka Exporter. So if you don't have them in your Prometheus, you need to configure your Prometheus to scrape them.

[veerendra2]

Sorry for partial info. Yes I configured kubernetes_sd_configs like below(Because podmonitor CRD is not installed in our cluster and servicemonitor relabel is not working properly)

    - job_name: strimzi-kafka
      scrape_interval: 15s
      scrape_timeout: 10s
      metrics_path: /metrics
      scheme: http
      kubernetes_sd_configs:
      - api_server: null
        role: pod
        namespaces:
          names: 
          - kafka

(Still testing above config with relabel. But I able to fetch metrics from Kafka brokers, zookeepers, operators). Then I checked exporter pod with curl to see what metrics are getting, it turns out only 2 metrics are related to kafka. For example below are missing

kafka_topic_partitions
kafka_topic_partition_replicas
kafka_topic_partition_in_sync_replica

Basically kafka-exporter-dashabord is empty for me

[scholzj]

You would need to check the Kafka Exporter logs then I guess. Kafka Exporter was quite buggy in older versions. So maybe it is related.

[veerendra2]

I have enabled debug logs in oc edit deployment sirius-dev-kafka-exporter

...
 spec:
      containers:
      - command:
        - /opt/kafka-exporter/kafka_exporter_run.sh
        env:
        - name: KAFKA_EXPORTER_LOGGING
          value: debug
        - name: KAFKA_EXPORTER_KAFKA_VERSION
          value: 2.6.0
...

The logs are still showing info and desnt have much info

$ oc logs sirius-dev-kafka-exporter-6b44c5f4c9-wzkp2
time="2021-10-27T12:11:23Z" level=info msg="Starting kafka_exporter (version=1.2.0, branch=HEAD, revision=830660212e6c109e69dcb1cb58f5159fe3b38903)" source="kafka_exporter.go:474"
time="2021-10-27T12:11:23Z" level=info msg="Build context (go=go1.10.3, user=root@981cde178ac4, date=20180707-14:34:48)" source="kafka_exporter.go:475"
time="2021-10-27T12:11:23Z" level=info msg="Done Init Clients" source="kafka_exporter.go:213"
time="2021-10-27T12:11:23Z" level=info msg="Listening on :9404" source="kafka_exporter.go:499"

[scholzj]

You cannot edit the deployment, you have to change the log levels through the Kafka custom resource.

[veerendra2]

Ok, I see the issue. The producer and consumer are not pushing/pulling logs from/into Kafaka due to gave wrong bootstrap endpoint config(I was testing other things, forgot to remove). Now fixed, I understood kafka-exporter only generates metrics when there are topics in Kafka. If Kafka has no Topics/partition, it wont generate metrics

Thanks for the quick responses.

[veerendra2]

The kafka-exporter only generates metrics if producer/consumer are push/pull logs from/info kafka.
Deploy producer and consumer

bash-4.2$ curl localhost:9404/metrics | grep kafka | cut -d '{' -f 1 | sort | uniq
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 36524  100 36524    0     0   170k      0 --:--:-- --:--:-- --:--:--  170k
# HELP kafka_brokers Number of Brokers in the Kafka Cluster.
# HELP kafka_consumergroup_current_offset Current Offset of a ConsumerGroup at Topic/Partition
# HELP kafka_consumergroup_lag Current Approximate Lag of a ConsumerGroup at Topic/Partition
# HELP kafka_exporter_build_info A metric with a constant '1' value labeled by version, revision, branch, and goversion from which kafka_exporter was built.
# HELP kafka_topic_partition_current_offset Current Offset of a Broker at Topic/Partition
# HELP kafka_topic_partition_in_sync_replica Number of In-Sync Replicas for this Topic/Partition
# HELP kafka_topic_partition_leader Leader Broker ID of this Topic/Partition
# HELP kafka_topic_partition_leader_is_preferred 1 if Topic/Partition is using the Preferred Broker
# HELP kafka_topic_partition_oldest_offset Oldest Offset of a Broker at Topic/Partition
# HELP kafka_topic_partition_replicas Number of Replicas for this Topic/Partition
# HELP kafka_topic_partition_under_replicated_partition 1 if Topic/Partition is under Replicated
# HELP kafka_topic_partitions Number of partitions for this Topic
# TYPE kafka_brokers gauge
# TYPE kafka_consumergroup_current_offset gauge
# TYPE kafka_consumergroup_lag gauge
# TYPE kafka_exporter_build_info gauge
# TYPE kafka_topic_partition_current_offset gauge
# TYPE kafka_topic_partition_in_sync_replica gauge
# TYPE kafka_topic_partition_leader gauge
# TYPE kafka_topic_partition_leader_is_preferred gauge
# TYPE kafka_topic_partition_oldest_offset gauge
# TYPE kafka_topic_partition_replicas gauge
# TYPE kafka_topic_partition_under_replicated_partition gauge
# TYPE kafka_topic_partitions gauge
kafka_brokers 3
kafka_consumergroup_current_offset
kafka_consumergroup_lag
kafka_exporter_build_info
kafka_topic_partition_current_offset
kafka_topic_partition_in_sync_replica
kafka_topic_partition_leader
kafka_topic_partition_leader_is_preferred
kafka_topic_partition_oldest_offset
kafka_topic_partition_replicas
kafka_topic_partition_under_replicated_partition
kafka_topic_partitions