Added role role_kafka_mirrormaker

Author: emielkuiper

Makefile

@@ -1,7 +1,7 @@
 .PHONY: all up-dc1 up-dc2 up provision-dc1 provision-dc2 provision restart-dc1 restart-dc2 restart brokers topics demo elastic_health elastic_indices elastic_allocation elastic all-dc1 all-dc2
 .DEFAULT_GOAL := all
 
-all: all-dc1 all-dc2 kafka-demo 
+all: all-dc1 all-dc2 kafka-demo
 
 all-dc1: up-dc1 provision-dc1 restart-dc1
 
@@ -68,3 +68,6 @@ restart: restart-dc1 restart-dc2
 kafka-demo:
 	ansible-playbook -i vagrant/inventory/hosts vagrant/ansible-playbook_demo_script.yml  --limit "dc1"
 	ansible-playbook -i vagrant/inventory/hosts vagrant/ansible-playbook_demo_script.yml  --limit "dc2"
+
+mirrormaker:
+	ansible-playbook -i vagrant/inventory/hosts vagrant/ansible-playbook.yml  --tags "mirrormaker"

README.md

@@ -27,7 +27,7 @@ This Makefile contains several options to control the environment.
 The most common options have been described below:
 
 * make (without arguments)
-Will spin up a single DC setup. After spinning up the VMs for a single DC setup Ansible will be automatically called to provision the VMs.
+Will spin up a dual DC setup. After spinning up the VMs for a the DCs Ansible will be automatically called to provision the VMs.
 
 * make [up-dc1 | up-dc2]
 Will only spin up either dc1 or dc2. No Ansible scripting will be done.
@@ -42,6 +42,9 @@ Requires Vms to already be up.
 * make provision
 Will provision both dc1 and dc2.
 
+* make mirrormaker
+Will provision a MirrorMaker instance on the TARGET DC. In this case the process will be started on DC2. It will consume from DC1 and act as a producer for DC2.
+
 
 
 # Regarding port numbers that Kafka uses
@@ -92,3 +95,30 @@ Hint: Some lines were ellipsized, use -l to show in full.
 [vagrant@kafka_broker_01 ~]$
 
 ```
+
+
+# MirrorMaker
+
+An additional service "confluence-mirrormaker" has been added. This newly added service can be managed through the regular systemctl commands.
+
+Example:
+
+```
+[vagrant@dc2_kafka_workernode_01 ~]$ sudo systemctl status confluent-mirrormaker -l
+● confluent-mirrormaker.service - Apache Kafka - MirrorMaker
+   Loaded: loaded (/usr/lib/systemd/system/confluent-mirrormaker.service; static; vendor preset: disabled)
+   Active: active (running) since Fri 2018-09-21 11:28:37 UTC; 2min 52s ago
+ Main PID: 14189 (java)
+   CGroup: /system.slice/confluent-mirrormaker.service
+           └─14189 java -Xmx256M -server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+ExplicitGCInvokesConcurrent -Djava.awt.headless=true -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dkafka.logs.dir=/var/log/kafka -Dlog4j.configuration=file:/etc/kafka/tools-log4j.properties -cp /bin/../share/java/kafka/*:/bin/../share/java/confluent-support-metrics/*:/usr/share/java/confluent-support-metrics/* kafka.tools.MirrorMaker --consumer.config /etc/kafka/consumer_dc1_to_dc2.properties --producer.config /etc/kafka/producer_dc1_to_dc2.properties --num.streams 3 --abort.on.send.failure true --whitelist test test2 test3 test4 test5
+
+Sep 21 11:28:37 dc2_kafka_workernode_01 systemd[1]: Started Apache Kafka - MirrorMaker.
+Sep 21 11:28:37 dc2_kafka_workernode_01 systemd[1]: Starting Apache Kafka - MirrorMaker...
+[vagrant@dc2_kafka_workernode_01 ~]$
+
+```
+
+*NOTE!!!*
+Do NOT start MirrorMaker on BOTH DCs. This will cause a neverending replication loop between both Kafka clusters!
+MirrorMaker does NOT have support for preventing this.
+Confluent Replicator is needed if you want to replicate both ways! 

vagrant/ansible-playbook.yml

@@ -43,3 +43,8 @@
 - import_playbook: ansible-playbook_import_elasticsearch_singlenode.yml
   tags:
     - elasticsearch
+
+# 8) Setting up MirrorMaker
+- import_playbook: ansible-playbook_import_mirrormaker.yml
+  tags:
+    - mirrormaker

vagrant/ansible-playbook_import_mirrormaker.yml

@@ -0,0 +1,13 @@
+###
+### Set up the hostname
+### Done in Ansible because of Vagrant race condition between setting both IP addresses and hostname
+
+- hosts:         kafka_brokers, kafka_worker_nodes
+  remote_user:   vagrant
+  become:        true
+  become_user:   root
+  become_method: sudo
+  gather_facts:  true
+
+  roles:
+    - role: role_kafka_mirrormaker

vagrant/group_vars/all

@@ -38,7 +38,7 @@ zookeeper_datadir: "/var/lib/zookeeper"
 kafka_zookeeper_connect: "
 {%- for node in vars['ansible_play_hosts'] | intersect(groups['kafka_brokers']) -%}
 {{ node }}:{{ port_zookeeper }}{% if not loop.last %},{% endif %}
-{% endfor%}
+{%- endfor%}
 "
 
 # Zookeeper multiline server config in zookeeper.properties file
@@ -55,21 +55,21 @@ zookeeper_properties_server_multiline_config: "{% for server in vars['ansible_pl
 kafka_bootstrap_servers: "
 {%- for node in vars['ansible_play_hosts'] | intersect(groups['kafka_brokers']) -%}
 {{ node }}:{{ port_kafka_broker }}{% if not loop.last %},{% endif %}
-{% endfor%}
+{%- endfor %}
 "
 
 # Example content: "172.28.129.201, 172.28.129.202, 172.28.129.203  "
 kafka_broker_ips: "
 {%- for node in vars['ansible_play_hosts'] | intersect(groups['kafka_brokers']) -%}
 {{ node }}{% if not loop.last %},{% endif %}
-{% endfor%}
+{%- endfor %}
 "
 
 # Example content: "172.28.129.204, 172.28.129.205  "
 kafka_workernode_ips: "
 {%- for node in vars['ansible_play_hosts'] | intersect(groups['kafka_worker_nodes']) -%}
 {{ node }}{% if not loop.last %},{% endif %}
-{% endfor%}
+{%- endfor %}
 "
 
 ###

vagrant/group_vars/kafka_worker_nodes

@@ -61,15 +61,64 @@ elastic_sink_name:       'elastic_sink_connector'
 
 
 ###
-### MirrorMaker
+### MirrorMaker "confluent-mirrormaker" systemctl service
+### Note: only needs to run at the TARGET destination
 ###
 
-### Note:
-### only needs to run at the TARGET destination
-
 ### Config files
 mirrormaker_consumer_config: "/etc/kafka/consumer_dc1_to_dc2.properties"
 mirrormaker_producer_config: "/etc/kafka/producer_dc1_to_dc2.properties"
 
 ### Space separated list of topics needed to be replicated
-mirrormaker_topic_whitelist: "test"
+mirrormaker_topic_whitelist: "test test2 test3 test4 test5"
+
+# Number of parallel streams
+mirrormaker_num_streams: "3"
+mirrormaker_abort_on_failure: "true"
+
+
+###
+### MirrorMaker consumer & producer config files
+### Note: only needs to run at the TARGET destination
+###
+
+mirrormaker_log4j_rootLogger: "WARN, stderr, file"
+
+### Variables for SOURCE target (aka: dc1)
+
+# Example content: "172.28.129.201:9092,172.28.129.202:9092,172.28.129.203:9092"
+mirrormaker_source_bootstrap_servers: "
+{%- for node in groups['dc1_kafka_brokers'] -%}
+{{ node }}:{{ port_kafka_broker }}{% if not loop.last %},{% endif %}
+{%- endfor %}
+"
+
+mirrormaker_source_group_id: "mirrormaker-group-source"
+mirrormaker_source_client_id: "mirror_maker_consumer"
+mirrormaker_source_exclude_internal_topics: "true"
+
+
+### Variables for DESTINATION target (aka: dc2)
+
+# Example content: "172.28.129.221:9092,172.28.129.222:9092,172.28.129.223:9092"
+mirrormaker_target_bootstrap_servers: "
+{%- for node in groups['dc2_kafka_brokers'] -%}
+{{ node }}:{{ port_kafka_broker }}{% if not loop.last %},{% endif %}
+{%- endfor %}
+"
+
+# (valid values are none, gzip, snappy, or lz4)
+mirrormaker_target_compression_type: "none"
+mirrormaker_target_client_id: "mirror_maker_producer"
+mirrormaker_target_max_in_flight_requests_per_connection: "1"
+mirrormaker_target_retries: "Int.MaxValue"
+mirrormaker_target_acks: "all"
+mirrormaker_target_block_on_buffer_full: "true"
+
+# Note that enabling idempotence requires
+#   max.in.flight.requests.per.connection to be less than or equal to 5,
+#   retries to be greater than 0 and
+#   acks must be 'all'.
+# If these values are not explicitly set by the user, suitable values will be chosen.
+# If incompatible values are set, a ConfigException will be thrown.
+mirrormaker_target_enable_idempotence: "true"

vagrant/roles/role_kafka_mirrormaker/defaults/main.yml

@@ -0,0 +1 @@
+