Merge pull request #1219 from NASA-IMPACT/1182-ml-classification-queue

1182 ml classification queue

Author: CarsonDavis

CHANGELOG.md

@@ -12,6 +12,7 @@ For each PR made, an entry should be added to this changelog. It should contain
   - etc.
 
 ## Changelog
+
 - 1209-bug-fix-document-type-creator-form
   - Description: The dropdown on the pattern creation form needs to be set as multi as the default option since this is why the doc type creator form is used for the majority of multi-URL pattern creations. This should be applied to doc types, division types, and titles as well.
   - Changes:

compose/local/django/start

@@ -1,3 +1,4 @@
+#compose/local/django/start
 #!/bin/bash
 
 set -o errexit

compose/production/django/Dockerfile

@@ -1,3 +1,4 @@
+# compose/production/django/Dockerfile
 # define an alias for the specfic python version used in this file.
 FROM python:3.10.14-slim-bullseye AS python
 

compose/production/django/start

@@ -1,3 +1,4 @@
+# compose/production/django/start
 #!/bin/bash
 
 set -o errexit

compose/production/traefik/traefik.yml

@@ -1,3 +1,4 @@
+# compose/production/traefik/traefik.yml
 log:
   level: INFO
 

config/celery.py

@@ -0,0 +1,24 @@
+# config/celery.py
+import os
+
+from celery import Celery
+from celery.schedules import crontab
+
+# Set the default Django settings module
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
+
+app = Celery("cosmos")
+
+# Configure Celery using Django settings
+app.config_from_object("django.conf:settings", namespace="CELERY")
+
+# Load task modules from all registered Django app configs
+app.autodiscover_tasks()
+
+app.conf.beat_schedule = {
+    "process-inference-queue": {
+        "task": "inference.tasks.process_inference_job_queue",
+        # Only run between 6pm and 7am
+        "schedule": crontab(minute="*/5", hour="18-23,0-6"),
+    },
+}

config/settings/base.py

@@ -84,6 +84,7 @@
     "feedback",
     "sde_collections",
     "sde_indexing_helper.users",
+    "inference",
 ]
 
 # https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
@@ -289,11 +290,9 @@
 # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std:setting-result_serializer
 CELERY_RESULT_SERIALIZER = "json"
 # https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-time-limit
-# TODO: set to whatever value is adequate in your circumstances
-CELERY_TASK_TIME_LIMIT = 5 * 60
+CELERY_TASK_TIME_LIMIT = 30 * 60
 # https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-soft-time-limit
-# TODO: set to whatever value is adequate in your circumstances
-CELERY_TASK_SOFT_TIME_LIMIT = 60
+CELERY_TASK_SOFT_TIME_LIMIT = 25 * 60
 # https://docs.celeryq.dev/en/stable/userguide/configuration.html#beat-scheduler
 CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler"
 # https://docs.celeryq.dev/en/stable/userguide/configuration.html#worker-send-task-events
@@ -350,3 +349,5 @@
 LRM_QA_PASSWORD = env("LRM_QA_PASSWORD")
 LRM_DEV_TOKEN = env("LRM_DEV_TOKEN")
 XLI_TOKEN = env("XLI_TOKEN")
+INFERENCE_API_URL = env("INFERENCE_API_URL", default="http://host.docker.internal:8000")
+TDAMM_CLASSIFICATION_THRESHOLD = env("TDAMM_CLASSIFICATION_THRESHOLD", default="0.5")

inference/__init__.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class InferenceConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "inference"
+    verbose_name = "Inference"