Set up RabbitMQ as Celery broker

.envs/.ci/.django

@@ -19,3 +19,9 @@ MINIO_BROWSER_REDIRECT_URL=http://minio:9001
 
 DEFAULT_PROCESSING_SERVICE_NAME=Test Processing Service
 DEFAULT_PROCESSING_SERVICE_ENDPOINT=http://ml_backend:2000
+
+# RabbitMQ
+# ------------------------------------------------------------------------------
+CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672//
+RABBITMQ_DEFAULT_USER=rabbituser
+RABBITMQ_DEFAULT_PASS=rabbitpass

.envs/.local/.django

@@ -16,6 +16,11 @@ REDIS_URL=redis://redis:6379/0
 CELERY_FLOWER_USER=QSocnxapfMvzLqJXSsXtnEZqRkBtsmKT
 CELERY_FLOWER_PASSWORD=BEQgmCtgyrFieKNoGTsux9YIye0I7P5Q7vEgfJD2C4jxmtHDetFaE2jhS7K7rxaf
 
+# RabbitMQ
+CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672//
+RABBITMQ_DEFAULT_USER=rabbituser
+RABBITMQ_DEFAULT_PASS=rabbitpass
+
 # Attempting to keep Flower from showing workers as offline
 # FLOWER_BROKER_API=REDIS_URL
 FLOWER_PERSISTENT=True

compose/local/django/celery/worker/start

@@ -4,4 +4,4 @@ set -o errexit
 set -o nounset
 
 
-exec watchfiles --filter python celery.__main__.main --args '-A config.celery_app worker -l INFO'
+exec watchfiles --filter python celery.__main__.main --args '-A config.celery_app worker --queues=antenna -l INFO'

config/settings/base.py

@@ -260,6 +260,7 @@
         },
     }
 }
+REDIS_URL = env("REDIS_URL", default=None)
 
 # ADMIN
 # ------------------------------------------------------------------------------
@@ -299,6 +300,8 @@
 if USE_TZ:
     # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-timezone
     CELERY_TIMEZONE = TIME_ZONE
+
+CELERY_TASK_DEFAULT_QUEUE = "antenna"
 # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-broker_url
 CELERY_BROKER_URL = env("CELERY_BROKER_URL")
 # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-result_backend

config/settings/local.py

@@ -83,10 +83,9 @@
 # ------------------------------------------------------------------------------
 # https://django-extensions.readthedocs.io/en/latest/installation_instructions.html#configuration
 INSTALLED_APPS += ["django_extensions"]  # noqa: F405
+
 # Celery
 # ------------------------------------------------------------------------------
-
+CELERY_TASK_DEFAULT_QUEUE = "antenna"
 # https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-eager-propagates
 CELERY_TASK_EAGER_PROPAGATES = True
-# Your stuff...
-# ------------------------------------------------------------------------------

docker-compose.yml

@@ -24,6 +24,7 @@ services:
       - redis
       - minio-init
       - ml_backend
+      - rabbitmq
     volumes:
       - .:/app:z
     env_file:
@@ -94,7 +95,9 @@ services:
     # Also make sure to install debugpy in your requirements/local.txt
     ports:
       - "5678:5678"
-    command: python -m debugpy --listen 0.0.0.0:5678 -m celery -A config.celery_app worker -l INFO
+    command: python -m debugpy --listen 0.0.0.0:5678 -m celery -A config.celery_app worker --queues=antenna -l INFO
+    depends_on:
+      - rabbitmq
 
   celerybeat:
     <<: *django
@@ -111,6 +114,17 @@ services:
     volumes:
       - ./data/flower/:/data/
 
+  rabbitmq:
+    image: rabbitmq:3-management
+    ports:
+      - "5672:5672"
+      - "15672:15672"
+    environment:
+      RABBITMQ_DEFAULT_USER: rabbituser
+      RABBITMQ_DEFAULT_PASS: rabbitpass
+    networks:
+      - antenna_network
+
   minio:
     image: minio/minio:RELEASE.2024-11-07T00-52-20Z
     command: minio server --console-address ":9001" /data