Merge pull request #467 from raytung/docs/fix-up-docs

docs(out_kafka2): added examples and added `topic` parameter to README

Author: ashie

README.md

@@ -187,6 +187,10 @@ If `ruby-kafka` doesn't fit your kafka environment, check `rdkafka2` plugin inst
       @type kafka2
 
       brokers               <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
+
+      # Kafka topic, placerholders are supported. Chunk keys are required in the Buffer section inorder for placeholders
+      # to work.
+      topic                 (string) :default => nil
       topic_key             (string) :default => 'topic'
       partition_key         (string) :default => 'partition'
       partition_key_key     (string) :default => 'partition_key'
@@ -243,13 +247,12 @@ ruby-kafka's log is routed to fluentd log so you can see ruby-kafka's log in flu
 
 Supports following ruby-kafka's producer options.
 
-- max_send_retries - default: 1 - Number of times to retry sending of messages to a leader.
+- max_send_retries - default: 2 - Number of times to retry sending of messages to a leader.
 - required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
 - ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
 - compression_codec - default: nil - The codec the producer uses to compress messages.
 - max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
 - discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
-- monitoring_list - default: [] - library to be used to monitor. statsd and datadog are supported
 
 If you want to know about detail of monitoring, see also https://github.com/zendesk/ruby-kafka#monitoring
 
@@ -420,6 +423,16 @@ Support of fluentd v0.12 has ended. `kafka_buffered` will be an alias of `kafka2
       monitoring_list              (array)   :default => []
     </match>
 
+`kafka_buffered` supports the following `ruby-kafka` parameters:
+
+- max_send_retries - default: 2 - Number of times to retry sending of messages to a leader.
+- required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
+- ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
+- compression_codec - default: nil - The codec the producer uses to compress messages.
+- max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
+- discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
+- monitoring_list - default: [] - library to be used to monitor. statsd and datadog are supported
+
 `kafka_buffered` has two additional parameters:
 
 - kafka_agg_max_bytes - default: 4096 - Maximum value of total message size to be included in one batch transmission.

examples/README.md

@@ -0,0 +1,3 @@
+# Examples
+
+This directory contains example Fluentd config for this plugin

examples/out_kafka2/dynamic_topic_based_on_tag.conf

@@ -0,0 +1,32 @@
+<source>
+    @type sample
+    sample {"hello": "world"}
+    rate 7000
+    tag sample.hello.world
+</source>
+
+<match sample.**>
+    @type kafka2
+
+    brokers "broker:29092"
+
+    # Writes to topic `events.sample.hello.world`
+    topic "events.${tag}"
+
+    # Writes to topic `hello.world`
+    # topic "${tag[1]}.${tag[2]}"
+
+    <format>
+        @type json
+    </format>
+
+    <buffer tag>
+      flush_at_shutdown true
+      flush_mode interval
+      flush_interval 1s
+      chunk_limit_size 3MB
+      chunk_full_threshold 1
+      total_limit_size 1024MB
+      overflow_action block
+    </buffer>
+</match>

examples/out_kafka2/protobuf-formatter.conf

@@ -0,0 +1,23 @@
+<source>
+    @type sample
+    sample {"hello": "world", "some_record":{"event":"message"}}
+    rate 7000
+    tag sample.hello.world
+</source>
+
+<match sample.**>
+    @type kafka2
+
+    brokers "broker:29092"
+
+    record_key "some_record"
+    default_topic "events"
+
+    <format>
+        # requires the fluent-plugin-formatter-protobuf gem
+        # see its docs for full usage
+        @type protobuf
+        class_name SomeRecord
+        include_paths ["/opt/fluent-plugin-formatter-protobuf/some_record_pb.rb"]
+    </format>
+</match>

examples/out_kafka2/record_key.conf

@@ -0,0 +1,31 @@
+<source>
+    @type sample
+    sample {"hello": "world", "some_record":{"event":"message"}}
+    rate 7000
+    tag sample.hello.world
+</source>
+
+<match sample.**>
+    @type kafka2
+
+    brokers "broker:29092"
+
+    # {"event": "message"} will be formatted and sent to Kafka
+    record_key "some_record"
+
+    default_topic "events"
+
+    <format>
+        @type json
+    </format>
+
+    <buffer>
+      flush_at_shutdown true
+      flush_mode interval
+      flush_interval 1s
+      chunk_limit_size 3MB
+      chunk_full_threshold 1
+      total_limit_size 1024MB
+      overflow_action block
+    </buffer>
+</match>