Merge pull request #1270 from NASA-IMPACT/1269-fix-celery-trigger

Updated process_inference_queue to run on the schedule

Author: dhanur-sharma

config/celery.py

@@ -2,7 +2,6 @@
 import os
 
 from celery import Celery
-from celery.schedules import crontab
 
 # Set the default Django settings module
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
@@ -14,11 +13,3 @@
 
 # Load task modules from all registered Django app configs
 app.autodiscover_tasks()
-
-app.conf.beat_schedule = {
-    "process-inference-queue": {
-        "task": "inference.tasks.process_inference_job_queue",
-        # Only run between 6pm and 7am
-        "schedule": crontab(minute="*/5", hour="18-23,0-6"),
-    },
-}

inference/__init__.py

@@ -0,0 +1 @@
+default_app_config = "inference.apps.InferenceConfig"

inference/apps.py

@@ -5,3 +5,6 @@ class InferenceConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
     name = "inference"
     verbose_name = "Inference"
+
+    def ready(self):
+        import inference.signals  # noqa F401

inference/signals.py

@@ -0,0 +1,24 @@
+from django.db.models.signals import post_migrate
+from django.dispatch import receiver
+
+
+@receiver(post_migrate)
+def create_periodic_tasks(sender, **kwargs):
+    if sender.name == "inference":
+        from django_celery_beat.models import CrontabSchedule, PeriodicTask
+
+        # Create schedule for every 5 minutes between 6pm-7am
+        crontab, _ = CrontabSchedule.objects.get_or_create(
+            minute="*/5",
+            hour="18-23,0-6",
+            day_of_week="*",
+            day_of_month="*",
+            month_of_year="*",
+        )
+
+        # Create the periodic task if it doesn't exist
+        PeriodicTask.objects.get_or_create(
+            crontab=crontab,
+            name="Process inference queue (6pm-7am)",
+            task="inference.tasks.process_inference_job_queue",
+        )

inference/tests/test_batch.py

@@ -36,7 +36,7 @@ def mock_url_large(self):
         """Returns a mock URL object with large text content"""
         url = Mock()
         url.id = 2
-        url.scraped_text = "X" * 10010  # Exceeds default max size
+        url.scraped_text = "X" * 12000  # Exceeds default max size
         url.scraped_title = "Large Content Page"
         url.url = "https://example.com/large-page"
         return url
@@ -191,7 +191,7 @@ def test_iter_url_batches_mix_normal_and_oversized(self, processor):
         # Normal URL
         url1 = Mock(id=1, scraped_text="X" * 2000, scraped_title="Title 1", url="https://example.com/1")
         # Oversized URL
-        url2 = Mock(id=2, scraped_text="X" * 11000, scraped_title="Title 2", url="https://example.com/2")
+        url2 = Mock(id=2, scraped_text="X" * 15000, scraped_title="Title 2", url="https://example.com/2")
         # Another normal URL
         url3 = Mock(id=3, scraped_text="X" * 3000, scraped_title="Title 3", url="https://example.com/3")