Skip to content

Commit 11eb44a

Browse files
emielkuiperemielkuiper
committed
modified: Vagrantfile
 modified: vagrant/ansible-playbook.yml modified: vagrant/templates/kafka/server.properties.j2
1 parent a77f510 commit 11eb44a

File tree

3 files changed

+134
-51
lines changed

3 files changed

+134
-51
lines changed

‎Vagrantfile‎

Lines changed: 48 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,54 @@
11
Vagrant.configure("2") do |config|
22

3-
# Define first Kafka node
4-
config.vm.define "kafka_node_01" do |kafka_node_01|
3+
# Define first Kafka broker node
4+
config.vm.define "kafka_broker_01" do |kafka_broker_01|
55
# synced_folder only used in ansible_local setting
6-
# kafka_node_01.vm.synced_folder "./vagrant", "/vagrant"
7-
kafka_node_01.vm.box = "ol7-latest"
8-
kafka_node_01.vm.hostname = 'kafka-node-01'
9-
kafka_node_01.vm.box_url = "https://yum.oracle.com/boxes/oraclelinux/latest/ol7-latest.box"
10-
kafka_node_01.vm.network :"private_network", type: "dhcp"
11-
kafka_node_01.vm.network "forwarded_port", guest: 9092, host: 9092, protocol: "tcp"
12-
kafka_node_01.vm.network "forwarded_port", guest: 9021, host: 9021, protocol: "tcp"
13-
kafka_node_01.vm.network "forwarded_port", guest: 8083, host: 8083, protocol: "tcp"
14-
kafka_node_01.vm.network "forwarded_port", guest: 8082, host: 8082, protocol: "tcp"
15-
kafka_node_01.vm.network "forwarded_port", guest: 8081, host: 8081, protocol: "tcp"
16-
kafka_node_01.vm.network "forwarded_port", guest: 2181, host: 2181, protocol: "tcp"
17-
kafka_node_01.vm.provider :virtualbox do |v|
6+
# kafka_broker_01.vm.synced_folder "./vagrant", "/vagrant"
7+
kafka_broker_01.vm.box = "ol7-latest"
8+
kafka_broker_01.vm.hostname = 'kafka-broker-01'
9+
kafka_broker_01.vm.box_url = "https://yum.oracle.com/boxes/oraclelinux/latest/ol7-latest.box"
10+
kafka_broker_01.vm.network :"private_network", type: "dhcp"
11+
kafka_broker_01.vm.network "forwarded_port", guest: 9092, host: 9092, protocol: "tcp"
12+
kafka_broker_01.vm.network "forwarded_port", guest: 9021, host: 9021, protocol: "tcp"
13+
kafka_broker_01.vm.network "forwarded_port", guest: 8083, host: 8083, protocol: "tcp"
14+
kafka_broker_01.vm.network "forwarded_port", guest: 8082, host: 8082, protocol: "tcp"
15+
kafka_broker_01.vm.network "forwarded_port", guest: 8081, host: 8081, protocol: "tcp"
16+
kafka_broker_01.vm.network "forwarded_port", guest: 2181, host: 2181, protocol: "tcp"
17+
kafka_broker_01.vm.provider :virtualbox do |v|
1818
v.customize ["modifyvm", :id, "--memory", 1024]
1919
v.customize ["modifyvm", :id, "--cpus", "2"]
2020
v.customize ["modifyvm", :id, "--cpuexecutioncap", "50"]
2121
v.customize ["modifyvm", :id, "--usb", "off"]
2222
v.customize ["modifyvm", :id, "--audio", "none"]
23-
v.customize ["modifyvm", :id, "--name", "kafka_node_01"]
24-
end # End of "kafka_node_01.vm.provider"
25-
end # End of config.vm.define "kafka_node_01"
23+
v.customize ["modifyvm", :id, "--name", "kafka_broker_01"]
24+
end # End of "kafka_broker_01.vm.provider"
25+
end # End of config.vm.define "kafka_broker_01"
2626

2727

28+
# Define second Kafka broker node
29+
config.vm.define "kafka_broker_02" do |kafka_broker_02|
30+
# synced_folder only used in ansible_local setting
31+
# kafka_broker_02.vm.synced_folder "./vagrant", "/vagrant"
32+
kafka_broker_02.vm.box = "ol7-latest"
33+
kafka_broker_02.vm.hostname = 'kafka-broker-02'
34+
kafka_broker_02.vm.box_url = "https://yum.oracle.com/boxes/oraclelinux/latest/ol7-latest.box"
35+
kafka_broker_02.vm.network :"private_network", type: "dhcp"
36+
kafka_broker_02.vm.network "forwarded_port", guest: 9092, host: 29092, protocol: "tcp"
37+
kafka_broker_02.vm.network "forwarded_port", guest: 9021, host: 29021, protocol: "tcp"
38+
kafka_broker_02.vm.network "forwarded_port", guest: 8083, host: 28083, protocol: "tcp"
39+
kafka_broker_02.vm.network "forwarded_port", guest: 8082, host: 28082, protocol: "tcp"
40+
kafka_broker_02.vm.network "forwarded_port", guest: 8081, host: 28081, protocol: "tcp"
41+
kafka_broker_02.vm.network "forwarded_port", guest: 2181, host: 22181, protocol: "tcp"
42+
kafka_broker_02.vm.provider :virtualbox do |v|
43+
v.customize ["modifyvm", :id, "--memory", 1024]
44+
v.customize ["modifyvm", :id, "--cpus", "2"]
45+
v.customize ["modifyvm", :id, "--cpuexecutioncap", "50"]
46+
v.customize ["modifyvm", :id, "--usb", "off"]
47+
v.customize ["modifyvm", :id, "--audio", "none"]
48+
v.customize ["modifyvm", :id, "--name", "kafka_broker_02"]
49+
end # End of "kafka_broker_02.vm.provider"
50+
end # End of config.vm.define "kafka_broker_02"
51+
2852

2953
# Define Kafka worker node
3054
config.vm.define "kafka_workernode_01" do |kafka_workernode_01|
@@ -54,10 +78,13 @@ Vagrant.configure("2") do |config|
5478
ansible.playbook = "vagrant/ansible-playbook.yml"
5579
ansible.limit = "all"
5680
ansible.groups = {
57-
"all" => ["kafka_node_01", "kafka_workernode_01"],
58-
"kafkabrokers" => ["kafka_node_01", "kafka_workernode_01"],
59-
"kafkabrokers:vars" => {"variable1" => "example1",
60-
"variable2" => "example2"}
81+
"kafka_all" => ["kafka_broker_01", "kafka_broker_02", "kafka_workernode_01"],
82+
"kafka_brokers" => ["kafka_broker_01", "kafka_broker_02"],
83+
"kafka_brokers:vars" => {"variable1" => "example1",
84+
"variable2" => "example2"},
85+
"kafka_worker_nodes" => ["kafka_workernode_01"],
86+
"kafka_worker_nodes:vars" => {"variable1" => "example1",
87+
"variable2" => "example2"}
6188
}
6289
end # End of "config.vm.provision"
6390

‎vagrant/ansible-playbook.yml‎

Lines changed: 76 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,10 @@
1-
- hosts: kafkabrokers
1+
- hosts: kafka_all
22
remote_user: vagrant
33
become: true
44
become_user: root
55
become_method: sudo
66
gather_facts: true
77

8-
98
#############################################################################
109
###
1110
### Variables
@@ -28,17 +27,34 @@
2827
kafka_groupname: "kafka"
2928
kafka_username: "kafka"
3029

31-
# /etc/kafka/server.properties
30+
# Network ports
31+
port_kafka_broker: 9092
32+
port_kafka_connect: 8083
33+
port_kafka_rest_proxy: 8082
34+
port_kafka_schema_registry: 8081
35+
port_zookeeper: 2181
36+
37+
### /etc/kafka/server.properties
38+
#
39+
# broker.id: will be dynamically derived from the hostname
3240
#
33-
# Kafka broker.id (kafka_server_broker_id) will be dynamically derived from the hostname
41+
# num.partitions: The default number of log partitions per topic.
42+
# More partitions allow greater parallelism for consumption.
3443
kafka_server_num_partitions: 2
44+
#
45+
# listeners: The address the socket server listens on.
3546
kafka_server_listeners: "PLAINTEXT://:9092"
47+
#
48+
# zookeeper.connect: Zookeeper connection string (see zookeeper docs for details).
49+
# This is a comma separated host:port pairs, each corresponding to a zk
50+
# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
3651
kafka_server_zookeeper_connect: "{{ kafka_service_ip_address }}:2181"
3752
#
3853
# NOTE: Confluent Proactive Support - requires support contract
3954
kafka_server_confluent_support_metrics_enable: "false"
4055

41-
# /etc/kafka/consumer.properties
56+
57+
### /etc/kafka/consumer.properties
4258
#
4359
kafka_consumer_bootstrap_servers: "{{ kafka_service_ip_address }}:9092"
4460
kafka_consumer_group_id: "test-consumer-group"
@@ -63,11 +79,21 @@
6379

6480
tasks:
6581

66-
- name: All the hosts known in the inventory file group kafkabrokers
82+
- name: All the hosts known in the inventory file group kafka_brokers
83+
debug:
84+
msg="{{ item }}"
85+
with_items:
86+
- "{{ groups['kafka_brokers'] }}"
87+
when:
88+
- inventory_hostname in groups['kafka_brokers']
89+
90+
- name: All the hosts known in the inventory file group kafka_worker_nodes
6791
debug:
6892
msg="{{ item }}"
6993
with_items:
70-
- "{{ groups['kafkabrokers'] }}"
94+
- "{{ groups['kafka_worker_nodes'] }}"
95+
when:
96+
- inventory_hostname in groups['kafka_worker_nodes']
7197

7298
- name: Kafka Service IP address
7399
debug:
@@ -81,6 +107,14 @@
81107
with_items:
82108
- "{{ kafka_mgmt_ip_address }}"
83109

110+
- name: Setting host fact for service IP address
111+
set_fact:
112+
service_ip: >
113+
{{ hostvars[inventory_hostname]['ansible_' + kafka_service_interface].ipv4.address }}
114+
115+
- name: Display fact
116+
debug:
117+
msg="{{ service_ip }}"
84118

85119
#############################################################################
86120
###
@@ -256,10 +290,14 @@
256290
register: kafka_server_broker_id
257291
tags:
258292
- kafka-broker-id
293+
when:
294+
- inventory_hostname in groups['kafka_brokers']
259295

260296
- name: Give broker-id number
261297
debug:
262298
msg: "Broker ID is {{ kafka_server_broker_id.stdout }}"
299+
when:
300+
- inventory_hostname in groups['kafka_brokers']
263301

264302

265303
#############################################################################
@@ -278,6 +316,8 @@
278316
mode: 0755
279317
tags:
280318
- kafka-configfiles
319+
when:
320+
- inventory_hostname in groups['kafka_brokers']
281321

282322
- name: Copying producer.properties
283323
template:
@@ -379,8 +419,6 @@
379419
- kafka-configfiles
380420

381421

382-
383-
384422
#############################################################################
385423
###
386424
### Create control script in /etc/init.d
@@ -396,25 +434,6 @@
396434
tags:
397435
- kafka-controlscript
398436

399-
# - name: Updating startup sequence
400-
# debug:
401-
# msg: "Getting the various services to start at boot time..."
402-
#
403-
# - name: Setting up execute permissions on rc local script
404-
# file:
405-
# path: "/etc/rc.d/rc.local"
406-
# mode: u+x
407-
# state: touch
408-
# tags:
409-
# - kafka-controlscript
410-
#
411-
# - name: Inserting control script entry in rc local script
412-
# blockinfile:
413-
# dest: "/etc/rc.d/rc.local"
414-
# content: '/etc/init.d/kafka-control start'
415-
# tags:
416-
# - kafka-controlscript
417-
418437

419438
#############################################################################
420439
###
@@ -460,7 +479,6 @@
460479
tags:
461480
- kafka-systemctl
462481

463-
464482
- name: Copy systemctl service file zookeeper.service
465483
copy:
466484
src: files/usr/lib/systemd/system/zookeeper.service
@@ -487,13 +505,16 @@
487505
tags:
488506
- kafka-systemctl
489507

508+
490509
#############################################################################
491510
###
492511
### Enabling and starting Kafka services at boot
493512
###
494513

495514
- debug:
496-
msg: "Getting the various Kafka services up and running..."
515+
msg: "Starting services for Kafka brokers..."
516+
when:
517+
- inventory_hostname in groups['kafka_brokers']
497518

498519
- name: Start and enable services at boot time
499520
systemd:
@@ -510,7 +531,32 @@
510531
when:
511532
- ansible_os_family == "RedHat"
512533
- ansible_distribution_major_version == "7"
534+
- inventory_hostname in groups['kafka_brokers']
513535
tags:
514536
- kafka-systemctl
515537

538+
539+
- debug:
540+
msg: "Starting services for Kafka worker nodes..."
541+
when:
542+
- inventory_hostname in groups['kafka_worker_nodes']
543+
544+
- name: Start and enable services at boot time
545+
systemd:
546+
name: "{{ item }}"
547+
state: started
548+
enabled: True
549+
daemon_reload: yes
550+
with_items:
551+
- zookeeper
552+
- kafka-rest
553+
- kafka-connector
554+
when:
555+
- ansible_os_family == "RedHat"
556+
- ansible_distribution_major_version == "7"
557+
- inventory_hostname in groups['kafka_worker_nodes']
558+
tags:
559+
- kafka-systemctl
560+
561+
516562
#############################################################################

‎vagrant/templates/kafka/server.properties.j2‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,16 @@ log.retention.check.interval.ms=300000
120120
# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
121121
# You can also append an optional chroot string to the urls to specify the
122122
# root directory for all kafka znodes.
123+
124+
125+
# EXP:
126+
# werkt
127+
# zookeeper.connect={% for server in groups['kafka_brokers'] %}{{ hostvars[server]['ansible_nodename'] }}:{{ port_zookeeper }}{% if not loop.last %},{% endif %}{% endfor %}
128+
129+
# nog niet
130+
# zookeeper.connect={% for server in groups['kafka_brokers'] %}{{ hostvars[server] '{{ port_zookeeper }}' }}{% if not loop.last %},{% endif %}{% endfor %}
131+
132+
123133
zookeeper.connect=localhost:2181
124134

125135
# Timeout in ms for connecting to zookeeper

0 commit comments

Comments
 (0)