Merge pull request #7 from dabapps/delete-old-jobs

Management command to delete old jobs

Author: j4mie

README.md

@@ -109,6 +109,11 @@ Jobs have a `state` field which can have one of the following values:
 
 #### Management commands
 
+There is a management command, `manage.py delete_old_jobs`, which deletes any
+jobs from the database which are in state `COMPLETE` or `FAILED` and were
+created more than 24 hours ago. This could be run, for example, as a cron task,
+to ensure the jobs table remains at a reasonable size.
+
 For debugging/development purposes, a simple management command is supplied to create jobs:
 
     manage.py create_job <job_name> --queue_name 'my_queue_name' --workspace '{"key": "value"}'

django_dbq/management/commands/delete_old_jobs.py

@@ -0,0 +1,11 @@
+from django.core.management.base import BaseCommand
+from django_dbq.models import Job
+
+
+class Command(BaseCommand):
+
+    help = "Delete old jobs"
+
+    def handle(self, *args, **options):
+        Job.objects.delete_old()
+        self.stdout.write('Deleted old jobs')

django_dbq/models.py

@@ -3,13 +3,17 @@
 from django_dbq.tasks import get_next_task_name, get_failure_hook_name, get_creation_hook_name
 from jsonfield import JSONField
 from model_utils import Choices
+import datetime
 import logging
 import uuid
 
 
 logger = logging.getLogger(__name__)
 
 
+DELETE_JOBS_AFTER_HOURS = 24
+
+
 class JobManager(models.Manager):
 
     def get_ready_or_none(self, queue_name, max_retries=3):
@@ -38,6 +42,15 @@ def get_ready_or_none(self, queue_name, max_retries=3):
                 retries_left -= 1
                 logger.warn("Caught %s when looking for a READY job, retrying %s more times", str(e), retries_left)
 
+    def delete_old(self):
+        """
+        Delete all jobs older than DELETE_JOBS_AFTER_HOURS
+        """
+        delete_jobs_in_states = [Job.STATES.FAILED, Job.STATES.COMPLETE]
+        delete_jobs_created_before = datetime.datetime.utcnow() - datetime.timedelta(hours=DELETE_JOBS_AFTER_HOURS)
+        logger.info("Deleting all job in states %s created before %s", ", ".join(delete_jobs_in_states), delete_jobs_created_before.isoformat())
+        Job.objects.filter(state__in=delete_jobs_in_states, created__lte=delete_jobs_created_before).delete()
+
 
 class Job(models.Model):
 

django_dbq/tests.py

@@ -171,3 +171,30 @@ def test_failure_hook(self):
         job = Job.objects.get()
         self.assertEqual(job.state, Job.STATES.FAILED)
         self.assertEqual(job.workspace['output'], 'failure hook ran')
+
+
+@override_settings(JOBS={'testjob': {'tasks': ['a']}})
+class DeleteOldJobsTestCase(TestCase):
+
+    def test_delete_old_jobs(self):
+        two_days_ago = datetime.utcnow() - timedelta(days=2)
+
+        j1 = Job.objects.create(name='testjob', state=Job.STATES.COMPLETE)
+        j1.created = two_days_ago
+        j1.save()
+
+        j2 = Job.objects.create(name='testjob', state=Job.STATES.FAILED)
+        j2.created = two_days_ago
+        j2.save()
+
+        j3 = Job.objects.create(name='testjob', state=Job.STATES.NEW)
+        j3.created = two_days_ago
+        j3.save()
+
+        j4 = Job.objects.create(name='testjob', state=Job.STATES.COMPLETE)
+
+        Job.objects.delete_old()
+
+        self.assertEqual(Job.objects.count(), 2)
+        self.assertTrue(j3 in Job.objects.all())
+        self.assertTrue(j4 in Job.objects.all())