Removing volumes

Author: YuryHrytsuk

services/rabbit/.gitignore

@@ -1,4 +1,5 @@
 docker-compose.yml
+docker-compose*.yaml
 !erlang.cookie.secret.template
 rabbitmq.conf
 haproxy.cfg

services/rabbit/Makefile

@@ -2,7 +2,7 @@ REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
 
 include ${REPO_BASE_DIR}/scripts/common-services.Makefile
 # common-services.Makefile should be included first as common.Makefile
-#relies on STACK_NAME var which is defined in common-services.Makefile
+# relies on STACK_NAME var which is defined in common-services.Makefile
 include ${REPO_BASE_DIR}/scripts/common.Makefile
 
 .PHONY: up
@@ -36,3 +36,20 @@ haproxy.cfg: haproxy.cfg.j2 .env venv
 
 docker-compose.yml: docker-compose.yml.j2 .env rabbitmq.conf erlang.cookie.secret haproxy.cfg venv $(VENV_BIN)/j2
 	@$(call jinja, $<, .env, $@)
+
+#
+# Deleting volumes (data)
+#
+
+TEMP_COMPOSE_OPERATOR := .stack.${STACK_NAME}.operator.yaml
+STACK_NAME_OPERATOR := ${STACK_NAME}-operator
+
+docker-compose.operator.yaml: docker-compose.operator.yaml.j2 .env
+	@$(call jinja, $<, .env, $@)
+
+${TEMP_COMPOSE_OPERATOR}: docker-compose.operator.yaml .env
+	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
+
+destroy-cluster-with-data: ${TEMP_COMPOSE_OPERATOR} .env down
+	@docker stack rm ${STACK_NAME_OPERATOR}
+	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE_OPERATOR} ${STACK_NAME_OPERATOR}

services/rabbit/README.md

@@ -4,7 +4,7 @@ Make sure all nodes have joined the cluster before using it. Otherwise, number o
 
 ## Updating rabbitmq.conf / advanced.config (zero-downtime)
 
-We do not support this automated. But manually this can be achieved in case needed. `rabbitmq.conf` and `advanced.config` changes take effect after a node restart. This can be performed with zero-downtime when RabbitMQ is clustered (have multiple nodes). This can be achieved by stopping and starting rabbitmq nodes one by one
+We do not support this automated (except starting from scratch with empty volumes). But manually this can be achieved in case needed. `rabbitmq.conf` and `advanced.config` changes take effect after a node restart. This can be performed with zero-downtime when RabbitMQ is clustered (have multiple nodes). This can be achieved by stopping and starting rabbitmq nodes one by one
 * `docker exec -it <container-id> bash`
 * (inside container) `rabbitmqctl stop_app` and wait some time until node is stopped (can be seen in management ui)
 * (inside container) `rabbitmqctl start_app`

services/rabbit/docker-compose.operator.yaml.j2

@@ -0,0 +1,38 @@
+{% set NODE_IXS = range(1, (RABBIT_CLUSTER_NODE_COUNT | int) + 1) -%}
+
+services:
+
+{% for ix in NODE_IXS %}
+  clean_rabbit0{{ ix }}_volume:
+    image: docker:25.0.3-cli
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    init: true
+    deploy:
+      placement:
+        constraints:
+          - node.labels.rabbit0{{ ix }} == true
+      restart_policy:
+        condition: none
+      resources:
+        limits:
+          cpus: "1.0"
+          memory: "1G"
+        reservations:
+          cpus: "0.1"
+          memory: "128M"
+    configs:
+      - source: delete_docker_volume_on_node_script
+        target: /app/delete_docker_volume_on_node.sh
+        mode: 0755
+    environment:
+      VOLUME: ${STACK_NAME}0{{ ix }}_data
+      TIMEOUT: 120
+      INTERVAL: 5
+    entrypoint: ["/app/delete_docker_volume_on_node.sh"]
+{% endfor %}
+
+configs:
+  delete_docker_volume_on_node_script:
+    file: ./operator/delete_docker_volume_on_node.sh
+    name: ${STACK_NAME}_delete_docker_volume_on_node_script_{{ "./operator/delete_docker_volume_on_node.sh" | sha256file | substring(0,10) }}

services/rabbit/operator/delete_docker_volume_on_node.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env sh
+set -eu
+
+VOLUME="${VOLUME:-}"
+TIMEOUT="${TIMEOUT:-60}"
+INTERVAL="${INTERVAL:-5}"
+
+if [ -z "$VOLUME" ]; then
+  echo "ERROR: VOLUME not set. Usage: VOLUME=name [TIMEOUT=60] [INTERVAL=5] $0"
+  exit 1
+fi
+
+echo "Deleting volume '$VOLUME'"
+echo "Timeout: ${TIMEOUT}s"
+echo "Interval: ${INTERVAL}s"
+
+if ! docker volume inspect "$VOLUME" >/dev/null 2>&1; then
+  echo "Volume '$VOLUME' does not exist. Nothing to do."
+  exit 0
+fi
+
+elapsed=0
+while [ "$elapsed" -lt "$TIMEOUT" ]; do
+  if docker volume rm "$VOLUME" >/dev/null 2>&1; then
+    echo "Volume '$VOLUME' removed successfully."
+    exit 0
+  fi
+  remaining=$(( TIMEOUT - elapsed ))
+  echo "Volume '$VOLUME' still in use. Retrying in ${INTERVAL}s. Time left: ${remaining}s."
+  sleep "$INTERVAL"
+  elapsed=$(( elapsed + INTERVAL ))
+done
+
+echo "Timeout reached. Exiting."
+exit 1