From ac67395c537985c5e1f7ad6219bb6a8a23ee7835 Mon Sep 17 00:00:00 2001 From: Vanessa Mac Date: Sat, 8 Nov 2025 13:00:29 -0500 Subject: [PATCH 01/10] feat: rabbitmq as celery broker, default antenna queue --- .envs/.ci/.django | 6 ++++++ .envs/.local/.django | 5 +++++ compose/local/django/celery/worker/start | 2 +- config/settings/base.py | 3 +++ config/settings/local.py | 5 ++--- docker-compose.yml | 16 +++++++++++++++- 6 files changed, 32 insertions(+), 5 deletions(-) diff --git a/.envs/.ci/.django b/.envs/.ci/.django index f0adf722b..13dd5be16 100644 --- a/.envs/.ci/.django +++ b/.envs/.ci/.django @@ -19,3 +19,9 @@ MINIO_BROWSER_REDIRECT_URL=http://minio:9001 DEFAULT_PROCESSING_SERVICE_NAME=Test Processing Service DEFAULT_PROCESSING_SERVICE_ENDPOINT=http://ml_backend:2000 + +# RabbitMQ +# ------------------------------------------------------------------------------ +CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672// +RABBITMQ_DEFAULT_USER=rabbituser +RABBITMQ_DEFAULT_PASS=rabbitpass diff --git a/.envs/.local/.django b/.envs/.local/.django index 76a69e68e..ccc898b4c 100644 --- a/.envs/.local/.django +++ b/.envs/.local/.django @@ -16,6 +16,11 @@ REDIS_URL=redis://redis:6379/0 CELERY_FLOWER_USER=QSocnxapfMvzLqJXSsXtnEZqRkBtsmKT CELERY_FLOWER_PASSWORD=BEQgmCtgyrFieKNoGTsux9YIye0I7P5Q7vEgfJD2C4jxmtHDetFaE2jhS7K7rxaf +# RabbitMQ +CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672// +RABBITMQ_DEFAULT_USER=rabbituser +RABBITMQ_DEFAULT_PASS=rabbitpass + # Attempting to keep Flower from showing workers as offline # FLOWER_BROKER_API=REDIS_URL FLOWER_PERSISTENT=True diff --git a/compose/local/django/celery/worker/start b/compose/local/django/celery/worker/start index 183a80159..73a9f015b 100644 --- a/compose/local/django/celery/worker/start +++ b/compose/local/django/celery/worker/start @@ -4,4 +4,4 @@ set -o errexit set -o nounset -exec watchfiles --filter python celery.__main__.main --args '-A config.celery_app worker -l INFO' +exec watchfiles --filter python celery.__main__.main --args '-A config.celery_app worker --queues=antenna -l INFO' diff --git a/config/settings/base.py b/config/settings/base.py index aa2783c7f..68f5d4130 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -260,6 +260,7 @@ }, } } +REDIS_URL = env("REDIS_URL", default=None) # ADMIN # ------------------------------------------------------------------------------ @@ -299,6 +300,8 @@ if USE_TZ: # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-timezone CELERY_TIMEZONE = TIME_ZONE + +CELERY_TASK_DEFAULT_QUEUE = "antenna" # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-broker_url CELERY_BROKER_URL = env("CELERY_BROKER_URL") # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-result_backend diff --git a/config/settings/local.py b/config/settings/local.py index c2f58afa0..84b48701e 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -83,10 +83,9 @@ # ------------------------------------------------------------------------------ # https://django-extensions.readthedocs.io/en/latest/installation_instructions.html#configuration INSTALLED_APPS += ["django_extensions"] # noqa: F405 + # Celery # ------------------------------------------------------------------------------ - +CELERY_TASK_DEFAULT_QUEUE = "antenna" # https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-eager-propagates CELERY_TASK_EAGER_PROPAGATES = True -# Your stuff... -# ------------------------------------------------------------------------------ diff --git a/docker-compose.yml b/docker-compose.yml index ff9d125f0..031e4b84c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,6 +24,7 @@ services: - redis - minio-init - ml_backend + - rabbitmq volumes: - .:/app:z env_file: @@ -94,7 +95,9 @@ services: # Also make sure to install debugpy in your requirements/local.txt ports: - "5678:5678" - command: python -m debugpy --listen 0.0.0.0:5678 -m celery -A config.celery_app worker -l INFO + command: python -m debugpy --listen 0.0.0.0:5678 -m celery -A config.celery_app worker --queues=antenna -l INFO + depends_on: + - rabbitmq celerybeat: <<: *django @@ -111,6 +114,17 @@ services: volumes: - ./data/flower/:/data/ + rabbitmq: + image: rabbitmq:3-management + ports: + - "5672:5672" + - "15672:15672" + environment: + RABBITMQ_DEFAULT_USER: rabbituser + RABBITMQ_DEFAULT_PASS: rabbitpass + networks: + - antenna_network + minio: image: minio/minio:RELEASE.2024-11-07T00-52-20Z command: minio server --console-address ":9001" /data From 974226337f0c9857a1378545c861900318f036e6 Mon Sep 17 00:00:00 2001 From: Vanessa Mac Date: Sat, 8 Nov 2025 14:24:07 -0500 Subject: [PATCH 02/10] feat: separate CI stack with rabbitmq --- .envs/.ci/.django | 2 +- .envs/.local/.django | 2 +- config/settings/local.py | 5 +++-- docker-compose.ci.yml | 25 ++++++++++++++++++++++++- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/.envs/.ci/.django b/.envs/.ci/.django index 13dd5be16..f2a7a6e7e 100644 --- a/.envs/.ci/.django +++ b/.envs/.ci/.django @@ -22,6 +22,6 @@ DEFAULT_PROCESSING_SERVICE_ENDPOINT=http://ml_backend:2000 # RabbitMQ # ------------------------------------------------------------------------------ -CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672// +CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672 RABBITMQ_DEFAULT_USER=rabbituser RABBITMQ_DEFAULT_PASS=rabbitpass diff --git a/.envs/.local/.django b/.envs/.local/.django index ccc898b4c..6c82303cb 100644 --- a/.envs/.local/.django +++ b/.envs/.local/.django @@ -17,7 +17,7 @@ CELERY_FLOWER_USER=QSocnxapfMvzLqJXSsXtnEZqRkBtsmKT CELERY_FLOWER_PASSWORD=BEQgmCtgyrFieKNoGTsux9YIye0I7P5Q7vEgfJD2C4jxmtHDetFaE2jhS7K7rxaf # RabbitMQ -CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672// +CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672 RABBITMQ_DEFAULT_USER=rabbituser RABBITMQ_DEFAULT_PASS=rabbitpass diff --git a/config/settings/local.py b/config/settings/local.py index 84b48701e..c2f58afa0 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -83,9 +83,10 @@ # ------------------------------------------------------------------------------ # https://django-extensions.readthedocs.io/en/latest/installation_instructions.html#configuration INSTALLED_APPS += ["django_extensions"] # noqa: F405 - # Celery # ------------------------------------------------------------------------------ -CELERY_TASK_DEFAULT_QUEUE = "antenna" + # https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-eager-propagates CELERY_TASK_EAGER_PROPAGATES = True +# Your stuff... +# ------------------------------------------------------------------------------ diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index c622cdbc1..b220ac6fb 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -1,3 +1,12 @@ +# Ensure this launches a separate stack for CI testing to avoid conflicts with local dev setup: +name: antenna-ci + +volumes: + ami_ci_postgres_data: + driver: local + minio_ci_data: + driver: local + services: django: &django build: @@ -9,6 +18,7 @@ services: - redis - minio-init - ml_backend + - rabbitmq env_file: - ./.envs/.ci/.django - ./.envs/.ci/.postgres @@ -18,6 +28,8 @@ services: build: context: . dockerfile: ./compose/local/postgres/Dockerfile + volumes: + - ami_ci_postgres_data:/var/lib/postgresql/data env_file: - ./.envs/.ci/.postgres @@ -27,11 +39,20 @@ services: celeryworker: <<: *django image: ami_ci_celeryworker - command: /start-celeryworker + depends_on: + - rabbitmq + command: python -m celery -A config.celery_app worker --queues=antenna -l INFO + + rabbitmq: + image: rabbitmq:3-management + env_file: + - ./.envs/.ci/.django minio: image: minio/minio:RELEASE.2024-11-07T00-52-20Z command: minio server --console-address ":9001" /data + volumes: + - "minio_ci_data:/data" env_file: - ./.envs/.ci/.django healthcheck: @@ -53,6 +74,8 @@ services: ml_backend: build: context: ./processing_services/minimal + depends_on: + - rabbitmq volumes: - ./processing_services/minimal/:/app networks: From 1c337cd73b67e96bbe12333bffdaa37504783929 Mon Sep 17 00:00:00 2001 From: Vanessa Mac Date: Sat, 8 Nov 2025 14:56:25 -0500 Subject: [PATCH 03/10] fix: broker url trailing slash, database url for CI stack --- .envs/.ci/.django | 2 +- .envs/.ci/.postgres | 1 + .envs/.local/.django | 2 +- .envs/.local/.postgres | 1 + docker-compose.ci.yml | 2 -- 5 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.envs/.ci/.django b/.envs/.ci/.django index f2a7a6e7e..833bb7ec7 100644 --- a/.envs/.ci/.django +++ b/.envs/.ci/.django @@ -22,6 +22,6 @@ DEFAULT_PROCESSING_SERVICE_ENDPOINT=http://ml_backend:2000 # RabbitMQ # ------------------------------------------------------------------------------ -CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672 +CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672/ RABBITMQ_DEFAULT_USER=rabbituser RABBITMQ_DEFAULT_PASS=rabbitpass diff --git a/.envs/.ci/.postgres b/.envs/.ci/.postgres index f5b543e75..73888ccd5 100644 --- a/.envs/.ci/.postgres +++ b/.envs/.ci/.postgres @@ -3,3 +3,4 @@ POSTGRES_PORT=5432 POSTGRES_DB=ami-ci POSTGRES_USER=4JXkOnTAeDmDyIapSRrGEE POSTGRES_PASSWORD=d4xojpnJU3OzPQ0apSCLP1oHR1TYvyMzAlF5KpE9HFL6MPlnbDibwI +DATABASE_URL=postgresql://4JXkOnTAeDmDyIapSRrGEE:d4xojpnJU3OzPQ0apSCLP1oHR1TYvyMzAlF5KpE9HFL6MPlnbDibwI@postgres:5432/ami-ci diff --git a/.envs/.local/.django b/.envs/.local/.django index 6c82303cb..59586664e 100644 --- a/.envs/.local/.django +++ b/.envs/.local/.django @@ -17,7 +17,7 @@ CELERY_FLOWER_USER=QSocnxapfMvzLqJXSsXtnEZqRkBtsmKT CELERY_FLOWER_PASSWORD=BEQgmCtgyrFieKNoGTsux9YIye0I7P5Q7vEgfJD2C4jxmtHDetFaE2jhS7K7rxaf # RabbitMQ -CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672 +CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672/ RABBITMQ_DEFAULT_USER=rabbituser RABBITMQ_DEFAULT_PASS=rabbitpass diff --git a/.envs/.local/.postgres b/.envs/.local/.postgres index 7fd1e8e22..598c1604e 100644 --- a/.envs/.local/.postgres +++ b/.envs/.local/.postgres @@ -3,3 +3,4 @@ POSTGRES_PORT=5432 POSTGRES_DB=ami POSTGRES_USER=xekSryPnqczJXkOnTAeDmDyIapSRrGEE POSTGRES_PASSWORD=iMRQjJEGflj5xojpnJU3OzPQ0apSCLP1oHR1TYvyMzAlF5KpE9HFL6MPlnbDibwI +DATABASE_URL=postgres://xekSryPnqczJXkOnTAeDmDyIapSRrGEE:iMRQjJEGflj5xojpnJU3OzPQ0apSCLP1oHR1TYvyMzAlF5KpE9HFL6MPlnbDibwI@postgres:5432/ami diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index b220ac6fb..b2e4e667c 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -74,8 +74,6 @@ services: ml_backend: build: context: ./processing_services/minimal - depends_on: - - rabbitmq volumes: - ./processing_services/minimal/:/app networks: From 53d13eb007438facf71da647b1f21eb0056519a0 Mon Sep 17 00:00:00 2001 From: Vanessa Mac Date: Mon, 10 Nov 2025 13:07:05 -0500 Subject: [PATCH 04/10] Add missing CI changes --- docker-compose.ci.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index b2e4e667c..446ae6ac7 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -12,7 +12,10 @@ services: build: context: . dockerfile: ./compose/local/django/Dockerfile - image: ami_ci_django + volumes: + - .:/app:z + extra_hosts: + - "host.docker.internal:host-gateway" depends_on: - postgres - redis @@ -38,7 +41,6 @@ services: celeryworker: <<: *django - image: ami_ci_celeryworker depends_on: - rabbitmq command: python -m celery -A config.celery_app worker --queues=antenna -l INFO @@ -66,7 +68,8 @@ services: env_file: - ./.envs/.ci/.django depends_on: - - minio + minio: + condition: service_healthy volumes: - ./compose/local/minio/init.sh:/etc/minio/init.sh entrypoint: /etc/minio/init.sh From 539fd3571d4e27d5fa38ba78b50a5d053aa658a4 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 13 Nov 2025 14:43:58 -0800 Subject: [PATCH 05/10] chore: share env vars with django app --- docker-compose.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 031e4b84c..cd3334f4a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -119,9 +119,8 @@ services: ports: - "5672:5672" - "15672:15672" - environment: - RABBITMQ_DEFAULT_USER: rabbituser - RABBITMQ_DEFAULT_PASS: rabbitpass + env_file: + - ./.envs/.local/.django networks: - antenna_network From 133515d0f92808ed6dbd1ccb0f7b6614cdbda8e4 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 13 Nov 2025 15:04:18 -0800 Subject: [PATCH 06/10] feat: move celery & debugbpy start command out of compose files --- compose/local/django/celery/worker/start | 8 ++++++-- docker-compose.ci.yml | 2 +- docker-compose.yml | 4 +++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/compose/local/django/celery/worker/start b/compose/local/django/celery/worker/start index 73a9f015b..5b270a5d7 100644 --- a/compose/local/django/celery/worker/start +++ b/compose/local/django/celery/worker/start @@ -3,5 +3,9 @@ set -o errexit set -o nounset - -exec watchfiles --filter python celery.__main__.main --args '-A config.celery_app worker --queues=antenna -l INFO' +# Support optional debug mode via CELERY_DEBUG environment variable +if [ "${CELERY_DEBUG:-0}" = "1" ]; then + exec python -m debugpy --listen 0.0.0.0:5678 -m celery -A config.celery_app worker --queues=antenna -l INFO +else + exec watchfiles --filter python celery.__main__.main --args '-A config.celery_app worker --queues=antenna -l INFO' +fi diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 446ae6ac7..8e93b684d 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -43,7 +43,7 @@ services: <<: *django depends_on: - rabbitmq - command: python -m celery -A config.celery_app worker --queues=antenna -l INFO + command: /start-celeryworker rabbitmq: image: rabbitmq:3-management diff --git a/docker-compose.yml b/docker-compose.yml index cd3334f4a..a64e306b3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -95,7 +95,9 @@ services: # Also make sure to install debugpy in your requirements/local.txt ports: - "5678:5678" - command: python -m debugpy --listen 0.0.0.0:5678 -m celery -A config.celery_app worker --queues=antenna -l INFO + environment: + - CELERY_DEBUG=1 + command: /start-celeryworker depends_on: - rabbitmq From edbd7d2a6795d3d0e900f5dc7085cc8ecd4dc04f Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 13 Nov 2025 15:08:36 -0800 Subject: [PATCH 07/10] feat: update production compose file for rabbitmq & default queue --- compose/production/django/celery/worker/start | 2 +- docker-compose.production.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/compose/production/django/celery/worker/start b/compose/production/django/celery/worker/start index 9d41926e7..6b372e854 100644 --- a/compose/production/django/celery/worker/start +++ b/compose/production/django/celery/worker/start @@ -4,4 +4,4 @@ set -o errexit set -o pipefail set -o nounset -exec newrelic-admin run-program celery -A config.celery_app worker -l INFO +exec newrelic-admin run-program celery -A config.celery_app worker --queues=antenna -l INFO diff --git a/docker-compose.production.yml b/docker-compose.production.yml index 099464277..83e2799da 100644 --- a/docker-compose.production.yml +++ b/docker-compose.production.yml @@ -16,8 +16,9 @@ services: - ./config:/app/config ports: - "5001:5000" - extra_hosts: + extra_hosts: # hostname aliases to the IPs of persistant services running outside docker network - "db:${DATABASE_IP}" + - "rabbitmq:${RABBITMQ_IP}" - "redis:${REDIS_IP}" command: /start scale: 1 # Can't scale until the load balancer is within the compose config From efec9f37c098295bb2dadec85e46cf1574b80383 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 13 Nov 2025 15:10:54 -0800 Subject: [PATCH 08/10] chore: explain redundant DATABASE_URL env var --- .envs/.ci/.postgres | 1 + .envs/.local/.postgres | 1 + 2 files changed, 2 insertions(+) diff --git a/.envs/.ci/.postgres b/.envs/.ci/.postgres index 73888ccd5..9d24e90fd 100644 --- a/.envs/.ci/.postgres +++ b/.envs/.ci/.postgres @@ -3,4 +3,5 @@ POSTGRES_PORT=5432 POSTGRES_DB=ami-ci POSTGRES_USER=4JXkOnTAeDmDyIapSRrGEE POSTGRES_PASSWORD=d4xojpnJU3OzPQ0apSCLP1oHR1TYvyMzAlF5KpE9HFL6MPlnbDibwI +# DATABASE_URL is for optional convenience with certain tools (e.g., psql), not used by Django DATABASE_URL=postgresql://4JXkOnTAeDmDyIapSRrGEE:d4xojpnJU3OzPQ0apSCLP1oHR1TYvyMzAlF5KpE9HFL6MPlnbDibwI@postgres:5432/ami-ci diff --git a/.envs/.local/.postgres b/.envs/.local/.postgres index 598c1604e..1654270bd 100644 --- a/.envs/.local/.postgres +++ b/.envs/.local/.postgres @@ -3,4 +3,5 @@ POSTGRES_PORT=5432 POSTGRES_DB=ami POSTGRES_USER=xekSryPnqczJXkOnTAeDmDyIapSRrGEE POSTGRES_PASSWORD=iMRQjJEGflj5xojpnJU3OzPQ0apSCLP1oHR1TYvyMzAlF5KpE9HFL6MPlnbDibwI +# DATABASE_URL is for optional convenience with certain tools (e.g., psql), not used by Django DATABASE_URL=postgres://xekSryPnqczJXkOnTAeDmDyIapSRrGEE:iMRQjJEGflj5xojpnJU3OzPQ0apSCLP1oHR1TYvyMzAlF5KpE9HFL6MPlnbDibwI@postgres:5432/ami From e8d10ab48bccb40d79bc9f45e94e927ad8f39e6d Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 13 Nov 2025 15:57:27 -0800 Subject: [PATCH 09/10] feat: use docker override file to activate debuggers --- .agents/AGENTS.md | 39 +++++++++++++++++++++ .gitignore | 3 ++ .vscode/launch.json | 39 +++++++++++++++++++++ README.md | 44 ++++++++++++++++++++++++ compose/local/django/celery/worker/start | 4 +-- compose/local/django/start | 9 +++-- docker-compose.yml | 6 ---- 7 files changed, 134 insertions(+), 10 deletions(-) diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index 875dbd652..e29b255a5 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -13,6 +13,16 @@ Every call to the AI model API incurs a cost and requires electricity. Be smart - Always prefer command line tools to avoid expensive API requests (e.g., use git and jq instead of reading whole files) - Use bulk operations and prefetch patterns to minimize database queries +**Performance Optimization:** +- django-cachalot handles automatic query caching - don't add manual caching layers on top +- Focus on optimizing cold queries first before adding caching +- When ordering by annotated fields, pagination COUNT queries include those annotations - use `.values('pk')` to strip them +- For large tables (>10k rows), consider fuzzy counting using PostgreSQL's pg_class.reltuples + +**Git Commit Guidelines:** +- Do NOT include "Generated with Claude Code" in commit messages +- ALWAYS include "Co-Authored-By: Claude " at the end of commit messages + ## Project Overview Antenna is an Automated Monitoring of Insects ML Platform. It's a collaborative platform for processing and reviewing images from automated insect monitoring stations, maintaining metadata, and orchestrating multiple machine learning pipelines for analysis. @@ -66,6 +76,35 @@ Stop all services: docker compose down ``` +### Debugging with VS Code + +Enable remote debugging with debugpy for Django and Celery services: + +```bash +# One-time setup: copy the override example file +cp docker-compose.override.yml.example docker-compose.override.yml + +# Start services (debugpy will be enabled automatically) +docker compose up + +# In VS Code, attach debugger: +# - "Attach: Django" for web server debugging (port 5678) +# - "Attach: Celeryworker" for task debugging (port 5679) +# - "Attach: Django + Celery" for simultaneous debugging +``` + +**How it works:** +- The `docker-compose.override.yml` file sets `DEBUGGER=1` environment variable +- Start scripts (`compose/local/django/start` and `compose/local/django/celery/worker/start`) detect this and launch with debugpy +- VS Code launch configurations in `.vscode/launch.json` connect to the exposed ports +- The override file is git-ignored for local customization + +**Disable debugging:** +```bash +rm docker-compose.override.yml +docker compose restart django celeryworker +``` + ### Backend (Django) Run tests: diff --git a/.gitignore b/.gitignore index 88e8e173a..7d50dcb5c 100644 --- a/.gitignore +++ b/.gitignore @@ -277,5 +277,8 @@ sandbox/ # Other flower +# Docker Compose override for local debugging (copy from .example file) +docker-compose.override.yml + # Temporary files in current work session .agents/todo diff --git a/.vscode/launch.json b/.vscode/launch.json index f85af6b31..2c60646c8 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,6 +1,38 @@ { "version": "0.2.0", "configurations": [ + { + "name": "Attach: Django", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "/app" + } + ], + "justMyCode": false + }, + { + "name": "Attach: Celeryworker", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5679 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "/app" + } + ], + "justMyCode": false + }, { "name": "Python Debugger: Remote Attach", "type": "debugpy", @@ -16,5 +48,12 @@ } ] } + ], + "compounds": [ + { + "name": "Attach: Django + Celery", + "configurations": ["Attach: Django", "Attach: Celeryworker"], + "stopAll": true + } ] } diff --git a/README.md b/README.md index 31be030b1..7d3774b48 100644 --- a/README.md +++ b/README.md @@ -258,3 +258,47 @@ The local environment uses a local PostgreSQL database in a Docker container. ### Load fixtures with test data docker compose run --rm django python manage.py migrate + +## Debugging with VS Code + +Antenna supports remote debugging with debugpy for both Django and Celery services. + +### Setup + +1. Copy the override example file: + ```bash + cp docker-compose.override.yml.example docker-compose.override.yml + ``` + +2. Start services normally: + ```bash + docker compose up + ``` + +3. In VS Code, open the Debug panel (Ctrl+Shift+D) and select one of: + - **Attach: Django** - Debug the Django web server (port 5678) + - **Attach: Celeryworker** - Debug the Celery worker (port 5679) + - **Attach: Django + Celery** - Debug both simultaneously + +4. Click the green play button or press F5 to attach the debugger + +### Setting Breakpoints + +- Set breakpoints in your Python code by clicking in the left margin of the editor +- When the code executes, the debugger will pause at your breakpoints +- Use the Debug Console to inspect variables and execute expressions + +### Troubleshooting + +- **Connection refused**: Make sure you copied `docker-compose.override.yml.example` to `docker-compose.override.yml` +- **Debugger not stopping**: Verify breakpoints are set in code that actually executes +- **Port conflicts**: Check that ports 5678 and 5679 aren't already in use on your host machine + +### Disabling Debug Mode + +To disable debugging and return to normal operation: + +```bash +rm docker-compose.override.yml +docker compose restart django celeryworker +``` diff --git a/compose/local/django/celery/worker/start b/compose/local/django/celery/worker/start index 5b270a5d7..5b5ee2cbf 100644 --- a/compose/local/django/celery/worker/start +++ b/compose/local/django/celery/worker/start @@ -3,8 +3,8 @@ set -o errexit set -o nounset -# Support optional debug mode via CELERY_DEBUG environment variable -if [ "${CELERY_DEBUG:-0}" = "1" ]; then +# Support optional debug mode via DEBUGGER environment variable +if [ "${DEBUGGER:-0}" = "1" ]; then exec python -m debugpy --listen 0.0.0.0:5678 -m celery -A config.celery_app worker --queues=antenna -l INFO else exec watchfiles --filter python celery.__main__.main --args '-A config.celery_app worker --queues=antenna -l INFO' diff --git a/compose/local/django/start b/compose/local/django/start index 1549d1334..e4360b6f8 100644 --- a/compose/local/django/start +++ b/compose/local/django/start @@ -4,6 +4,11 @@ set -o errexit set -o pipefail set -o nounset - python manage.py migrate -exec uvicorn config.asgi:application --host 0.0.0.0 --reload --reload-include '*.html' + +# Support optional debug mode via DEBUGGER environment variable +if [ "${DEBUGGER:-0}" = "1" ]; then + exec python -m debugpy --listen 0.0.0.0:5678 -m uvicorn config.asgi:application --host 0.0.0.0 --reload --reload-include '*.html' +else + exec uvicorn config.asgi:application --host 0.0.0.0 --reload --reload-include '*.html' +fi diff --git a/docker-compose.yml b/docker-compose.yml index a64e306b3..71734e977 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -91,12 +91,6 @@ services: <<: *django image: ami_local_celeryworker scale: 1 - # For remote debugging with debugpy, should get overridden for production - # Also make sure to install debugpy in your requirements/local.txt - ports: - - "5678:5678" - environment: - - CELERY_DEBUG=1 command: /start-celeryworker depends_on: - rabbitmq From 82fdfd8c288bcdd976825d33d05cc919d5c95746 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 13 Nov 2025 19:05:21 -0800 Subject: [PATCH 10/10] chore: tell agent docs we are using RabbitMQ instead of Redis now --- .agents/AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index c76fad170..8acceeb4d 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -283,7 +283,7 @@ Location: `processing_services/` directory contains example implementations ### Celery Task Queue -**Broker & Result Backend:** Redis +**Broker & Result Backend:** RabbitMQ **Key Tasks:** - `ami.jobs.tasks.run_job` - Main ML processing workflow