diff --git a/.envs/.ci/.django b/.envs/.ci/.django index 833bb7ec7..bec17501b 100644 --- a/.envs/.ci/.django +++ b/.envs/.ci/.django @@ -23,5 +23,6 @@ DEFAULT_PROCESSING_SERVICE_ENDPOINT=http://ml_backend:2000 # RabbitMQ # ------------------------------------------------------------------------------ CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672/ +CELERY_RESULT_BACKEND=rpc:// # Use RabbitMQ for results backend RABBITMQ_DEFAULT_USER=rabbituser RABBITMQ_DEFAULT_PASS=rabbitpass diff --git a/.envs/.local/.django b/.envs/.local/.django index 59586664e..e646e46eb 100644 --- a/.envs/.local/.django +++ b/.envs/.local/.django @@ -18,6 +18,7 @@ CELERY_FLOWER_PASSWORD=BEQgmCtgyrFieKNoGTsux9YIye0I7P5Q7vEgfJD2C4jxmtHDetFaE2jhS # RabbitMQ CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672/ +CELERY_RESULT_BACKEND=rpc:// # Use RabbitMQ for results backend RABBITMQ_DEFAULT_USER=rabbituser RABBITMQ_DEFAULT_PASS=rabbitpass diff --git a/config/settings/base.py b/config/settings/base.py index 95845d034..03124d41a 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -305,7 +305,8 @@ # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-broker_url CELERY_BROKER_URL = env("CELERY_BROKER_URL") # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-result_backend -CELERY_RESULT_BACKEND = env("CELERY_RESULT_BACKEND", default=env("REDIS_URL", default=None)) +# "rpc://" means use RabbitMQ for results backend by default +CELERY_RESULT_BACKEND = env("CELERY_RESULT_BACKEND", default="rpc://") # type: ignore[no-untyped-call] # https://docs.celeryq.dev/en/stable/userguide/configuration.html#result-extended CELERY_RESULT_EXTENDED = True # https://docs.celeryq.dev/en/stable/userguide/configuration.html#result-backend-always-retry @@ -334,7 +335,8 @@ # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std-setting-task_send_sent_event CELERY_TASK_SEND_SENT_EVENT = True -# Health checking and retries, specific to Redis +# Health checking and retries if using Redis as results backend +# https://docs.celeryq.dev/en/stable/userguide/configuration.html#redis CELERY_REDIS_MAX_CONNECTIONS = 50 # Total connection pool limit for results backend CELERY_REDIS_SOCKET_TIMEOUT = 120 # Match Redis timeout CELERY_REDIS_SOCKET_KEEPALIVE = True @@ -346,15 +348,19 @@ CELERY_WORKER_PREFETCH_MULTIPLIER = 1 CELERY_WORKER_ENABLE_PREFETCH_COUNT_REDUCTION = True +# Cancel & return to queue if connection is lost +# https://docs.celeryq.dev/en/latest/userguide/configuration.html#worker-cancel-long-running-tasks-on-connection-loss +CELERY_WORKER_CANCEL_LONG_RUNNING_TASKS_ON_CONNECTION_LOSS = True + # RabbitMQ broker connection settings # These settings improve reliability for long-running workers with intermittent network issues CELERY_BROKER_TRANSPORT_OPTIONS = { "socket_timeout": 120, # Socket read/write timeout (seconds) - "socket_connect_timeout": 30, # Max time to establish connection (seconds) + "socket_connect_timeout": 40, # Max time to establish connection (seconds) "socket_keepalive": True, # Enable TCP keepalive probes "retry_on_timeout": True, # Retry operations on timeout "max_connections": 20, # Per-process connection pool limit - "heartbeat": 60, # RabbitMQ heartbeat interval (seconds) - detects broken connections + "heartbeat": 30, # RabbitMQ heartbeat interval (seconds) - detects broken connections } # Broker connection retry settings