celery interface fix.

Author: andrade94

.env

@@ -47,6 +47,9 @@ MONGO_DB=socialmediadb
 RABBITMQ_USER=rabbitmquser
 RABBITMQ_PASSWORD=rabbitmqpassword
 
+# Celery
+CELERY_BROKER=amqp://rabbitmquser:rabbitmqpassword@rabbitmq:5672//
+
 # Add for local development when not using Docker
 # These will be automatically set in Docker based on service names
 # MONGO_SERVER=localhost

backend/Dockerfile

@@ -40,4 +40,7 @@ COPY ./app /app/app
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv sync
 
+# Install all required dependencies directly
+RUN pip install celery==5.3.0 flower==1.2.0 redis==4.6.0 pydantic>=2.0 pydantic-settings>=2.0 fastapi>=0.114.2 pymongo>=4.5.0 motor>=3.2.0
+
 CMD ["fastapi", "run", "--workers", "4", "app/main.py"]

backend/Dockerfile.celery

@@ -0,0 +1,14 @@
+FROM python:3.10
+
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH=/app
+
+WORKDIR /app/
+
+COPY ./pyproject.toml /app/
+COPY ./app /app/app
+
+# Install all required dependencies directly
+RUN pip install celery==5.3.0 flower==1.2.0 redis==4.6.0 pydantic>=2.0 pydantic-settings>=2.0 fastapi>=0.114.2 pymongo>=4.5.0 motor>=3.2.0 pika>=1.3.2 email-validator>=2.1.0 python-multipart>=0.0.7 passlib[bcrypt]>=1.7.4 bcrypt==4.0.1 pyjwt>=2.8.0 sqlmodel>=0.0.21 psycopg[binary]>=3.1.13 alembic>=1.12.1 tenacity>=8.2.3
+
+CMD ["celery", "-A", "app.tasks.celery_app", "worker", "--loglevel=info"] 

backend/app/core/config.py

@@ -81,6 +81,9 @@ def SQLALCHEMY_DATABASE_URI(self) -> PostgresDsn:
     EMAILS_FROM_EMAIL: EmailStr | None = None
     EMAILS_FROM_NAME: EmailStr | None = None
 
+    # Celery settings
+    CELERY_BROKER: str = "amqp://guest:guest@localhost:5672//"
+
     @model_validator(mode="after")
     def _set_default_emails_from(self) -> Self:
         if not self.EMAILS_FROM_NAME:

backend/app/db/session.py

@@ -1,4 +1,6 @@
 from typing import Generator
+import os
+from pymongo import MongoClient
 
 from sqlmodel import Session, create_engine
 
@@ -11,4 +13,16 @@
 def get_session() -> Generator[Session, None, None]:
     """Get a SQLModel session for PostgreSQL database operations."""
     with Session(engine) as session:
-        yield session 
+        yield session
+
+# MongoDB Connection
+mongodb_server = os.environ.get("MONGODB_SERVER", "localhost")
+mongodb_port = int(os.environ.get("MONGODB_PORT", "27017"))
+mongodb_user = os.environ.get("MONGODB_USER", "mongo")
+mongodb_password = os.environ.get("MONGODB_PASSWORD", "mongo")
+mongodb_database = os.environ.get("MONGODB_DATABASE", "political_analysis")
+
+# Create MongoDB client
+mongodb_uri = f"mongodb://{mongodb_user}:{mongodb_password}@{mongodb_server}:{mongodb_port}/{mongodb_database}"
+mongodb_client = MongoClient(mongodb_uri)
+mongodb = mongodb_client[mongodb_database] 

backend/app/tasks/celery_app.py

@@ -1,11 +1,23 @@
 from celery import Celery
+import os
 
 from app.core.config import settings
 
+# Get Redis connection details from environment variables
+redis_server = os.environ.get("REDIS_SERVER", "localhost")
+redis_port = os.environ.get("REDIS_PORT", "6379")
+redis_url = f"redis://{redis_server}:{redis_port}/0"
+
+# Get RabbitMQ connection details from environment variables
+rabbitmq_user = os.environ.get("RABBITMQ_USER", "guest")
+rabbitmq_password = os.environ.get("RABBITMQ_PASSWORD", "guest")
+rabbitmq_server = os.environ.get("RABBITMQ_SERVER", "localhost")
+broker_url = f"amqp://{rabbitmq_user}:{rabbitmq_password}@{rabbitmq_server}:5672//"
+
 celery_app = Celery(
     "worker",
-    broker=settings.CELERY_BROKER,
-    backend=settings.CELERY_BROKER,
+    broker=broker_url,
+    backend=redis_url,
 )
 
 # Configure Celery

backend/celerybeat-schedule

@@ -188,6 +188,7 @@ services:
   celery-worker:
     build:
       context: ./backend
+      dockerfile: Dockerfile.celery
     restart: "no"
     command: celery -A app.tasks.celery_app worker --loglevel=info
     volumes:
@@ -204,6 +205,7 @@ services:
       RABBITMQ_USER: ${RABBITMQ_USER:-guest}
       RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD:-guest}
       KAFKA_BOOTSTRAP_SERVERS: "kafka:9093,localhost:9092"
+      CELERY_BROKER: "amqp://${RABBITMQ_USER:-guest}:${RABBITMQ_PASSWORD:-guest}@rabbitmq:5672//"
     depends_on:
       - backend
       - redis
@@ -213,6 +215,7 @@ services:
   celery-beat:
     build:
       context: ./backend
+      dockerfile: Dockerfile.celery
     restart: "no"
     command: celery -A app.tasks.celery_app beat --loglevel=info
     volumes:
@@ -229,14 +232,16 @@ services:
       RABBITMQ_USER: ${RABBITMQ_USER:-guest}
       RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD:-guest}
       KAFKA_BOOTSTRAP_SERVERS: "kafka:9093,localhost:9092"
+      CELERY_BROKER: "amqp://${RABBITMQ_USER:-guest}:${RABBITMQ_PASSWORD:-guest}@rabbitmq:5672//"
     depends_on:
       - celery-worker
 
   celery-flower:
     build:
       context: ./backend
+      dockerfile: Dockerfile.celery
     restart: "no"
-    command: celery -A app.tasks.celery_app flower --port=5555
+    command: celery flower --broker=amqp://${RABBITMQ_USER:-guest}:${RABBITMQ_PASSWORD:-guest}@rabbitmq:5672// --port=5555
     ports:
       - "5555:5555"
     environment:
@@ -250,6 +255,15 @@ services:
       RABBITMQ_SERVER: "rabbitmq"
       RABBITMQ_USER: ${RABBITMQ_USER:-guest}
       RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD:-guest}
+      CELERY_BROKER: "amqp://${RABBITMQ_USER:-guest}:${RABBITMQ_PASSWORD:-guest}@rabbitmq:5672//"
+      PROJECT_NAME: "Political Social Media Analysis Platform"
+      POSTGRES_SERVER: "db"
+      POSTGRES_PORT: "5432"
+      POSTGRES_DB: "app"
+      POSTGRES_USER: "postgres"
+      POSTGRES_PASSWORD: "changethis"
+      FIRST_SUPERUSER: "[email protected]"
+      FIRST_SUPERUSER_PASSWORD: "password"
     depends_on:
       - celery-worker