RabbitMQ handler refactored to restart services in serial

Change-Id: I1ff4cbdf3f60cb7fd5fe5d3c5d498e05fe2df79a
Closes-Bug: #1904702
(cherry picked from commit 4cc4ba5)

Author: chembervint

ansible/roles/rabbitmq/handlers/main.yml

@@ -1,47 +1,10 @@
 ---
-- name: Restart rabbitmq container (first node)
+- name: Restart rabbitmq container
   vars:
     service_name: "rabbitmq"
     service: "{{ rabbitmq_services[service_name] }}"
-  become: true
-  kolla_docker:
-    action: "recreate_or_restart_container"
-    common_options: "{{ docker_common_options }}"
-    name: "{{ service.container_name }}"
-    image: "{{ service.image }}"
-    volumes: "{{ service.volumes }}"
-    environment: "{{ service.environment }}"
-    dimensions: "{{ service.dimensions }}"
+  include_tasks: 'restart_services.yml'
   when:
     - kolla_action != "config"
-    - inventory_hostname == groups[service.group]|first
-  notify:
-    - Waiting for rabbitmq to start on first node
-  listen: Restart rabbitmq container
-
-- name: Waiting for rabbitmq to start on first node
-  vars:
-    service_name: "rabbitmq"
-    service: "{{ rabbitmq_services[service_name] }}"
-  become: true
-  command: "docker exec {{ service.container_name }} rabbitmqctl wait {{ rabbitmq_pid_file }}"
-  when:
-    - inventory_hostname == groups[service.group]|first
-
-- name: Restart rabbitmq container (rest of nodes)
-  vars:
-    service_name: "rabbitmq"
-    service: "{{ rabbitmq_services[service_name] }}"
-  become: true
-  kolla_docker:
-    action: "recreate_or_restart_container"
-    common_options: "{{ docker_common_options }}"
-    name: "{{ service.container_name }}"
-    image: "{{ service.image }}"
-    volumes: "{{ service.volumes }}"
-    environment: "{{ service.environment }}"
-    dimensions: "{{ service.dimensions }}"
-  when:
-    - kolla_action != "config"
-    - inventory_hostname != groups[service.group]|first
-  listen: Restart rabbitmq container
+    - inventory_hostname == item
+  loop: "{{ groups[service.group] }}"

ansible/roles/rabbitmq/tasks/restart_services.yml

@@ -0,0 +1,21 @@
+---
+- name: Restart rabbitmq container
+  vars:
+    service_name: "rabbitmq"
+    service: "{{ rabbitmq_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+    environment: "{{ service.environment }}"
+    dimensions: "{{ service.dimensions }}"
+
+- name: Waiting for rabbitmq to start
+  vars:
+    service_name: "rabbitmq"
+    service: "{{ rabbitmq_services[service_name] }}"
+  become: true
+  command: "docker exec {{ service.container_name }} rabbitmqctl wait {{ rabbitmq_pid_file }}"

releasenotes/notes/bug-1904702-7451dd8c4caa309b.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    RabbitMQ services are now restarted serially to avoid a split brain.
+    `LP#1904702 <https://launchpad.net/bugs/1904702>`__