Robust volume clean up

+ haproxy extra configuration

Author: YuryHrytsuk

services/rabbit/Makefile

@@ -5,6 +5,10 @@ include ${REPO_BASE_DIR}/scripts/common-services.Makefile
 # relies on STACK_NAME var which is defined in common-services.Makefile
 include ${REPO_BASE_DIR}/scripts/common.Makefile
 
+#
+# Running cluster
+#
+
 .PHONY: up
 up: ${TEMP_COMPOSE} prune-docker-stack-configs prune-docker-stack-secrets ## Deploys metabase stack
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE} ${STACK_NAME}
@@ -22,6 +26,10 @@ up-public: up
 ${TEMP_COMPOSE}: docker-compose.yml .env
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
 
+#
+# Config / Secret files
+#
+
 erlang.cookie.secret: erlang.cookie.secret.template .env
 	@set -a; source .env; set +a; \
 	envsubst < $< > $@
@@ -41,15 +49,27 @@ docker-compose.yml: docker-compose.yml.j2 .env rabbitmq.conf erlang.cookie.secre
 # Deleting volumes (data)
 #
 
-TEMP_COMPOSE_OPERATOR := .stack.${STACK_NAME}.operator.yaml
-STACK_NAME_OPERATOR := ${STACK_NAME}-operator
+CLEAN_VOLUMES_TEMP_COMPOSE := .stack.${STACK_NAME}.clean-volumes.yaml
+CLEAN_VOLUMES_STACK_NAME := ${STACK_NAME}-clean-volumes
 
-docker-compose.operator.yaml: docker-compose.operator.yaml.j2 .env
+docker-compose.clean-volumes.yaml: docker-compose.clean-volumes.yaml.j2 .env
 	@$(call jinja, $<, .env, $@)
 
-${TEMP_COMPOSE_OPERATOR}: docker-compose.operator.yaml .env
+${CLEAN_VOLUMES_TEMP_COMPOSE}: docker-compose.clean-volumes.yaml .env
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
 
-destroy-cluster-with-data: ${TEMP_COMPOSE_OPERATOR} .env down
-	@docker stack rm ${STACK_NAME_OPERATOR}
-	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE_OPERATOR} ${STACK_NAME_OPERATOR}
+TIMEOUT = 2m
+delete-volumes: ${CLEAN_VOLUMES_TEMP_COMPOSE}
+	## avoid accumulating tasks (in case they fail). Always start fresh new
+	@docker stack rm --detach=false ${CLEAN_VOLUMES_STACK_NAME}
+	## use --detach=false to wait until all jobs complete successfully
+	@timeout ${TIMEOUT} \
+	docker stack deploy --detach=false --with-registry-auth --prune --compose-file ${CLEAN_VOLUMES_TEMP_COMPOSE} ${CLEAN_VOLUMES_STACK_NAME}
+	@docker stack rm ${CLEAN_VOLUMES_STACK_NAME}
+
+#
+# Destroy cluster with data (to start fresh new)
+#
+
+destroy-cluster-with-data: down delete-volumes
+	@echo "Cluster has been destroyed. Data has been deleted"

services/rabbit/docker-compose.clean-volumes.yaml.j2

@@ -0,0 +1,43 @@
+{% set NODE_IXS = range(1, (RABBIT_CLUSTER_NODE_COUNT | int) + 1) -%}
+
+services:
+
+{% for ix in NODE_IXS %}
+  clean_rabbit0{{ ix }}_volume:
+    image: docker:25.0.3-cli
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    init: true
+    deploy:
+      mode: replicated-job
+      replicas: 1
+      restart_policy:
+        # do not restart jobs to avoid running these jobs in background
+        # unawared. It may lead to unexpected volume removal once rabbit
+        # cluster is not running
+        condition: none
+      placement:
+        constraints:
+          - node.labels.rabbit0{{ ix }} == true
+      resources:
+        limits:
+          cpus: "0.5"
+          memory: "512M"
+        reservations:
+          cpus: "0.1"
+          memory: "128M"
+    configs:
+      - source: delete_rabbit_docker_volume_on_node_script
+        target: /app/delete_rabbit_docker_volume_on_node.sh
+        mode: 0755
+    environment:
+      VOLUME: ${STACK_NAME}0{{ ix }}_data
+      TIMEOUT_MINUTES: 1
+      INTERVAL_SECONDS: 10
+    entrypoint: ["/app/delete_rabbit_docker_volume_on_node.sh"]
+{% endfor %}
+
+configs:
+  delete_rabbit_docker_volume_on_node_script:
+    file: ./scripts/delete_rabbit_docker_volume_on_node.sh
+    name: ${STACK_NAME}_delete_rabbit_docker_volume_on_node_script_{{ "./scripts/delete_rabbit_docker_volume_on_node.sh" | sha256file | substring(0,10) }}

services/rabbit/docker-compose.operator.yaml.j2

@@ -4,6 +4,11 @@ services:
   loadbalancer:
     image: haproxy:3.2
     deploy:
+      update_config:
+        order: start-first
+        parallelism: 1
+        delay: 30s
+        failure_action: rollback
       # https://discourse.haproxy.org/t/haproxy-high-availability-configuration/11983
       replicas: ${RABBIT_LB_REPLICAS}
       # necessary to preserve client ip
@@ -26,11 +31,6 @@ services:
       timeout: 2s
       retries: 2
       interval: 10s
-    ports:
-      - target: 15672
-        published: 15672
-        protocol: tcp
-        mode: host
     networks:
       - rabbit
     configs:
@@ -100,7 +100,7 @@ services:
   subscriber:
     image: python:3.11
     deploy:
-      replicas: 0
+      replicas: 1
     command: sh -c "pip install pika && python /app/sub.py"
     environment:
       - RABBIT_HOSTS=rabbit_loadbalancer
@@ -114,7 +114,7 @@ services:
   publisher:
     image: python:3.11
     deploy:
-      replicas: 0
+      replicas: 1
     command: sh -c "pip install pika && python /app/pub.py"
     environment:
       - RABBIT_HOSTS=rabbit_loadbalancer

services/rabbit/docker-compose.yml.j2

@@ -68,3 +68,4 @@ backend rabbit_dashboard_backends
     server rabbit0{{ ix }} rabbit_rabbit0{{ ix }}:{{ RABBIT_MANAGEMENT_PORT }} check resolvers dockerdns init-addr libc,none inter 5s rise 2 fall 3
 {%- endfor %}
 # keep new line in the end to avoid "Missing LF on last line" error
+# random comment

services/rabbit/haproxy.cfg.j2

@@ -0,0 +1,46 @@
+#!/usr/bin/env sh
+set -eu
+
+VOLUME="${VOLUME:-}"
+TIMEOUT_MINUTES="${TIMEOUT_MINUTES:-2}"
+INTERVAL_SECONDS="${INTERVAL_SECONDS:-5}"
+
+if [ -z "$VOLUME" ]; then
+  echo "ERROR: VOLUME not set. Usage: VOLUME=name [TIMEOUT_MINUTES=2] [INTERVAL_SECONDS=5] $0"
+  exit 1
+fi
+
+case "$VOLUME" in
+  *rabbit*) ;;
+  *)
+    echo "[SAFEGUARD] ERROR: VOLUME name must contain 'rabbit'."
+    exit 1
+    ;;
+esac
+
+echo "Deleting volume '$VOLUME'"
+echo "Timeout: ${TIMEOUT_MINUTES}m"
+echo "Interval: ${INTERVAL_SECONDS},"
+
+if ! docker volume inspect "$VOLUME" >/dev/null 2>&1; then
+  echo "Volume '$VOLUME' does not exist. Nothing to do."
+  exit 0
+fi
+
+elapsed=0
+timeout_seconds=$(( TIMEOUT_MINUTES * 60 ))
+while [ "$elapsed" -lt "$timeout_seconds" ]; do
+  if docker volume rm "$VOLUME" >/dev/null 2>&1; then
+    echo "Volume '$VOLUME' removed successfully."
+    exit 0
+  fi
+
+  remaining=$(( timeout_seconds - elapsed ))
+  echo "Volume '$VOLUME' still in use. Retrying in ${INTERVAL_SECONDS}s. Time left: ${remaining}s."
+  sleep "$INTERVAL_SECONDS"
+
+  elapsed=$(( elapsed + INTERVAL_SECONDS ))
+done
+
+echo "Timeout reached. Exiting."
+exit 1