Replace celery with dramatiq in docker

Signed-off-by: Federico Busetti <[email protected]>

Author: febus982

Dockerfile

@@ -51,8 +51,8 @@ RUN make dev-dependencies
 # when signals are propagated, we disable it in dev image default CMD
 CMD ["uvicorn", "http_app:create_app", "--host", "0.0.0.0", "--port", "8000", "--factory", "--reload"]
 
-# Installs requirements to run production celery application
-FROM base_builder AS celery_builder
+# Installs requirements to run production dramatiq application
+FROM base_builder AS dramatiq_builder
 RUN poetry install --no-root
 
 # Installs requirements to run production http application
@@ -79,9 +79,10 @@ COPY --chown=nonroot:nonroot src/http_app ./http_app
 # Run CMD using array syntax, so it's uses `exec` and runs as PID1
 CMD ["opentelemetry-instrument", "uvicorn", "http_app:create_app", "--host", "0.0.0.0", "--port", "8000", "--factory"]
 
-# Copy the celery python package and requirements from relevant builder
-FROM base_app AS celery_app
-COPY --from=celery_builder /poetryvenvs /poetryvenvs
-COPY --chown=nonroot:nonroot src/celery_worker ./celery_worker
+# Copy the dramatiq python package and requirements from relevant builder
+FROM base_app AS dramatiq_app
+COPY --from=dramatiq_builder /poetryvenvs /poetryvenvs
+COPY --chown=nonroot:nonroot src/dramatiq_worker ./dramatiq_worker
 # Run CMD using array syntax, so it's uses `exec` and runs as PID1
-CMD ["opentelemetry-instrument", "celery", "-A", "celery_worker:app", "worker", "-l", "INFO"]
+# TODO: Review processes/threads
+CMD ["opentelemetry-instrument", "dramatiq", "-p", "1", "-t", "1", "dramatiq_worker"]

docker-compose.yaml

@@ -8,34 +8,16 @@ services:
   redis:
     image: redis
 
-  celery-worker:
+  dramatiq:
     build:
       dockerfile: Dockerfile
       context: .
-      target: celery_app
+      target: dramatiq_app
     environment:
-      OTEL_SERVICE_NAME: "bootstrap-fastapi-worker"
-      OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4317"
-      CELERY__broker_url: "redis://redis:6379/0"
-      CELERY__result_backend: "redis://redis:6379/1"
-    working_dir: "/app/src"
-    volumes:
-      - '.:/app'
-    depends_on:
-      - redis
-      - otel-collector
-      - celery-beat
-
-  celery-beat:
-    build:
-      dockerfile: Dockerfile
-      context: .
-      target: celery_app
-    environment:
-      OTEL_SERVICE_NAME: "bootstrap-fastapi-worker"
+      WATCHFILES_FORCE_POLLING: true
+      OTEL_SERVICE_NAME: "bootstrap-fastapi-dramatiq-worker"
       OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4317"
-      CELERY__broker_url: "redis://redis:6379/0"
-      CELERY__result_backend: "redis://redis:6379/1"
+      DRAMATIQ__REDIS_URL: "redis://redis:6379/0"
     working_dir: "/app/src"
     volumes:
       - '.:/app'
@@ -44,12 +26,14 @@ services:
       - otel-collector
     command:
       - opentelemetry-instrument
-      - celery
-      - -A
-      - celery_worker:app
-      - beat
-      - -l
-      - INFO
+      - dramatiq
+      - --watch
+      - .
+      - -p
+      - "1"
+      - -t
+      - "1"
+      - dramatiq_worker
 
   dev:
     build: