Kafka source missing message

[xxm404]

Data flow

script -> kafka -> vector -> console

vector version
vector 0.26.0 (x86_64-unknown-linux-gnu c6b5bc2 2022-12-05)
vector config

[sources.prometheus_style_metrics]
type = "kafka"
bootstrap_servers = "172.31.16.247:9092,172.31.15.146:9092"
group_id = "vector-script-metric-0"
topics = [ "scriptMetric" ]
commit_interval_ms = 10

[sinks.print]
type = "console"
inputs = ["prometheus_style_metrics"]
encoding.codec = "text"

when i use script pub 10 messages to kafka script log is as follows

{"metric":{"counter":{"value":0}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":1}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":2}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":3}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":4}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":5}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":6}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":7}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":8}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":9}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}

vector just print 3 messages

{"metric":{"counter":{"value":3}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":4}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":8}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}

kafka console consumer print 10 messages

{"metric":{"counter":{"value":3}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":4}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":1}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":5}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":9}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":2}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":6}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":7}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":8}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}
{"metric":{"counter":{"value":0}},"kind":"incremental","namespace":"test","name":"test_metric","tags":{"script_name":"test_prometheus.php"},"timestamp":1678943637}

describe consumer group

TOPIC           PARTITION  CURRENT-OFFSET  LOG-END-OFFSET  LAG             CONSUMER-ID                                 HOST            CLIENT-ID
scriptMetric    0          18              18              0               vector-7154169f-5e21-4469-87cd-d5ebc78ebaf3 /172.31.140.85  vector
scriptMetric    1          38              38              0               vector-7154169f-5e21-4469-87cd-d5ebc78ebaf3 /172.31.140.85  vector
scriptMetric    2          20              20              0               vector-7154169f-5e21-4469-87cd-d5ebc78ebaf3 /172.31.140.85  vector
scriptMetric    3          27              27              0               vector-9ff2a462-0394-4835-8bb8-dca95ebcfef7 /172.31.140.85  vector
scriptMetric    4          22              22              0               vector-9ff2a462-0394-4835-8bb8-dca95ebcfef7 /172.31.140.85  vector
scriptMetric    5          28              28              0               vector-9ff2a462-0394-4835-8bb8-dca95ebcfef7 /172.31.140.85  vector

looks like missing message
Is there something wrong with my configuration?

[jches]

Based on the Consumer ID column from the consumer group describe output you included, it looks like you have 2 vector processes, each consuming 3 kafka partitions. Did the missing messages get consumed by the second vector process?

[xxm404]

Thank you for your reply, this is indeed the problem, i was careless.