@@ -35,6 +35,30 @@ labels:
3535 type : nginx
3636` ` `
3737
38+ Adding a batch configuration:
39+
40+ ` ` ` yaml
41+ source : kafka
42+ brokers :
43+ - " localhost:9093"
44+ topic : " my-topic"
45+ timeout : 5
46+ tls :
47+ insecure_skip_verify : true
48+ client_cert : /path/kafkaClient.certificate.pem
49+ client_key : /path/kafkaClient.key
50+ ca_cert : /path/ca.crt
51+ labels :
52+ type : nginx
53+ batch :
54+ min_bytes : 1024 # 1KB
55+ max_bytes : 1048576 # 1MB
56+ max_wait : 5s
57+ queue_size : 1000
58+ commit_interval : 1s
59+ ` ` `
60+
61+
3862:::info
3963The reader will always start from the latest offset.
4064:::
@@ -60,7 +84,7 @@ Required.
6084
6185The consumer group id to use.
6286
63- Cannot be used with `partition`.
87+ Cannot be used with `partition`.
6488
6589:::warning
6690It is highly recommended to set this value, or crowdsec will only read logs from the 1st partition of the topic.
@@ -72,6 +96,12 @@ Read messages from the given partition. Mostly useful for debugging.
7296
7397Cannot be used with `group_id`.
7498
99+ # ## `timeout`
100+
101+ Maximum time to wait for new messages before returning an empty read.
102+
103+ Default : 5
104+
75105# ## `tls.insecure_skip_verify`
76106
77107To disable security checks on the certificate.
@@ -96,6 +126,35 @@ The CA certificate path.
96126
97127Optional, when you want to enable TLS with client certificate.
98128
129+ # ## `batch.min_bytes`
130+
131+ Minimum number of bytes to accumulate in the fetch buffer before returning results.
132+
133+ Default : 1
134+
135+ # ## `batch.max_bytes`
136+
137+ Maximum number of bytes to fetch in one go.
138+
139+ Default : 1048576 (1MB)
140+
141+ # ## `batch.max_wait`
142+
143+ Maximum time to wait before returning a fetch, even if `batch.min_bytes` isn’t reached.
144+
145+ Default : 250ms
146+
147+ # ## `batch.queue_size`
148+
149+ Maximum number of messages to buffer internally before processing.
150+
151+ Default : 100
152+
153+ # ## `batch.commit_interval`
154+
155+ Time interval between automatic commits of consumer offsets.
156+
157+ Default : 0 (commit after every fetch)
99158
100159# ## `source`
101160
@@ -104,4 +163,3 @@ Must be `kafka`
104163# # DSN and command-line
105164
106165This datasource does not support acquisition from the command line.
107-
0 commit comments