Add manager method to delete old jobs

Author: j4mie

django_dbq/models.py

@@ -3,13 +3,17 @@
 from django_dbq.tasks import get_next_task_name, get_failure_hook_name, get_creation_hook_name
 from jsonfield import JSONField
 from model_utils import Choices
+import datetime
 import logging
 import uuid
 
 
 logger = logging.getLogger(__name__)
 
 
+DELETE_JOBS_AFTER_HOURS = 24
+
+
 class JobManager(models.Manager):
 
     def get_ready_or_none(self, queue_name, max_retries=3):
@@ -38,6 +42,15 @@ def get_ready_or_none(self, queue_name, max_retries=3):
                 retries_left -= 1
                 logger.warn("Caught %s when looking for a READY job, retrying %s more times", str(e), retries_left)
 
+    def delete_old(self):
+        """
+        Delete all jobs older than DELETE_JOBS_AFTER_HOURS
+        """
+        delete_jobs_in_states = [Job.STATES.FAILED, Job.STATES.COMPLETE]
+        delete_jobs_created_before = datetime.datetime.utcnow() - datetime.timedelta(hours=DELETE_JOBS_AFTER_HOURS)
+        logger.info("Deleting all job in states %s created before %s", ", ".join(delete_jobs_in_states), delete_jobs_created_before.isoformat())
+        Job.objects.filter(state__in=delete_jobs_in_states, created__lte=delete_jobs_created_before).delete()
+
 
 class Job(models.Model):
 

django_dbq/tests.py

@@ -171,3 +171,30 @@ def test_failure_hook(self):
         job = Job.objects.get()
         self.assertEqual(job.state, Job.STATES.FAILED)
         self.assertEqual(job.workspace['output'], 'failure hook ran')
+
+
+@override_settings(JOBS={'testjob': {'tasks': ['a']}})
+class DeleteOldJobsTestCase(TestCase):
+
+    def test_delete_old_jobs(self):
+        two_days_ago = datetime.utcnow() - timedelta(days=2)
+
+        j1 = Job.objects.create(name='testjob', state=Job.STATES.COMPLETE)
+        j1.created = two_days_ago
+        j1.save()
+
+        j2 = Job.objects.create(name='testjob', state=Job.STATES.FAILED)
+        j2.created = two_days_ago
+        j2.save()
+
+        j3 = Job.objects.create(name='testjob', state=Job.STATES.NEW)
+        j3.created = two_days_ago
+        j3.save()
+
+        j4 = Job.objects.create(name='testjob', state=Job.STATES.COMPLETE)
+
+        Job.objects.delete_old()
+
+        self.assertEqual(Job.objects.count(), 2)
+        self.assertTrue(j3 in Job.objects.all())
+        self.assertTrue(j4 in Job.objects.all())