From f72b398fd8aa0db456375c28bb2c48b852438dd1 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 8 Oct 2024 10:47:20 -0400 Subject: [PATCH 01/47] feat:unify models --- scheduler/admin/__init__.py | 3 +- scheduler/admin/task_admin.py | 280 ++++++++++++++++++ scheduler/models/__init__.py | 1 + scheduler/models/scheduled_task.py | 2 +- scheduler/models/task.py | 458 +++++++++++++++++++++++++++++ 5 files changed, 742 insertions(+), 2 deletions(-) create mode 100644 scheduler/admin/task_admin.py create mode 100644 scheduler/models/task.py diff --git a/scheduler/admin/__init__.py b/scheduler/admin/__init__.py index 237e1c8..fb58874 100644 --- a/scheduler/admin/__init__.py +++ b/scheduler/admin/__init__.py @@ -1,2 +1,3 @@ -from .task_models import TaskAdmin # noqa: F401 +from .task_models import TaskAdmin as OldTaskAdmin # noqa: F401 from .ephemeral_models import QueueAdmin, WorkerAdmin # noqa: F401 +from .task_admin import TaskAdmin # noqa: F401 \ No newline at end of file diff --git a/scheduler/admin/task_admin.py b/scheduler/admin/task_admin.py new file mode 100644 index 0000000..80265f4 --- /dev/null +++ b/scheduler/admin/task_admin.py @@ -0,0 +1,280 @@ +import redis +import valkey +from django.contrib import admin, messages +from django.contrib.contenttypes.admin import GenericStackedInline +from django.utils.translation import gettext_lazy as _ + +from scheduler import tools +from scheduler.models import TaskArg, TaskKwarg, Task +from scheduler.settings import SCHEDULER_CONFIG, logger +from scheduler.tools import get_job_executions_for_task + + +class HiddenMixin(object): + class Media: + js = [ + "admin/js/jquery.init.js", + ] + + +class JobArgInline(HiddenMixin, GenericStackedInline): + model = TaskArg + extra = 0 + fieldsets = ( + ( + None, + { + "fields": ( + ( + "arg_type", + "val", + ), + ), + }, + ), + ) + + +class JobKwargInline(HiddenMixin, GenericStackedInline): + model = TaskKwarg + extra = 0 + fieldsets = ( + ( + None, + { + "fields": ( + ("key",), + ( + "arg_type", + "val", + ), + ), + }, + ), + ) + + +_LIST_DISPLAY_EXTRA = dict( + CronTask=( + "cron_string", + "next_run", + "successful_runs", + "last_successful_run", + "failed_runs", + "last_failed_run", + ), + ScheduledTask=("scheduled_time",), + RepeatableTask=( + "scheduled_time", + "interval_display", + "successful_runs", + "last_successful_run", + "failed_runs", + "last_failed_run", + ), + Task=( + "scheduled_time", + "interval_display", + "cron_string", + "next_run", + "successful_runs", + "last_successful_run", + "failed_runs", + "last_failed_run", + ), +) +_FIELDSET_EXTRA = dict( + CronTask=( + "cron_string", + "timeout", + "result_ttl", + ( + "successful_runs", + "last_successful_run", + ), + ( + "failed_runs", + "last_failed_run", + ), + ), + ScheduledTask=("scheduled_time", "timeout", "result_ttl"), + RepeatableTask=( + "scheduled_time", + ( + "interval", + "interval_unit", + ), + "repeat", + "timeout", + "result_ttl", + ( + "successful_runs", + "last_successful_run", + ), + ( + "failed_runs", + "last_failed_run", + ), + ), + Task=( + "scheduled_time", + "cron_string", + ( + "interval", + "interval_unit", + ), + "repeat", + "timeout", + "result_ttl", + ( + "successful_runs", + "last_successful_run", + ), + ( + "failed_runs", + "last_failed_run", + ), + ), +) + + +@admin.register(Task) +class TaskAdmin(admin.ModelAdmin): + """TaskAdmin admin view for all task models. + Using the _LIST_DISPLAY_EXTRA and _FIELDSET_EXTRA additional data for each model. + """ + + save_on_top = True + change_form_template = "admin/scheduler/change_form.html" + actions = [ + "disable_selected", + "enable_selected", + "enqueue_job_now", + ] + inlines = [ + JobArgInline, + JobKwargInline, + ] + list_filter = ("enabled",) + list_display = ( + "enabled", + "name", + "job_id", + "function_string", + "is_scheduled", + "queue", + ) + list_display_links = ("name",) + readonly_fields = ("job_id",) + fieldsets = ( + ( + None, + { + "fields": ( + "name", + "callable", + "enabled", + "at_front", + ), + }, + ), + ( + _("RQ Settings"), + { + "fields": ( + "queue", + "job_id", + ), + }, + ), + ) + + def get_list_display(self, request): + if self.model.__name__ not in _LIST_DISPLAY_EXTRA: + raise ValueError(f"Unrecognized model {self.model}") + return TaskAdmin.list_display + _LIST_DISPLAY_EXTRA[self.model.__name__] + + def get_fieldsets(self, request, obj=None): + if self.model.__name__ not in _FIELDSET_EXTRA: + raise ValueError(f"Unrecognized model {self.model}") + return TaskAdmin.fieldsets + ( + ( + _("Scheduling"), + { + "fields": _FIELDSET_EXTRA[self.model.__name__], + }, + ), + ) + + @admin.display(description="Next run") + def next_run(self, o: Task): + return tools.get_next_cron_time(o.cron_string) + + def change_view(self, request, object_id, form_url="", extra_context=None): + extra = extra_context or {} + obj = self.get_object(request, object_id) + try: + execution_list = get_job_executions_for_task(obj.queue, obj) + except (redis.ConnectionError, valkey.ConnectionError) as e: + logger.warn(f"Could not get job executions: {e}") + execution_list = list() + paginator = self.get_paginator(request, execution_list, SCHEDULER_CONFIG.EXECUTIONS_IN_PAGE) + page_number = request.GET.get("p", 1) + page_obj = paginator.get_page(page_number) + page_range = paginator.get_elided_page_range(page_obj.number) + + extra.update( + { + "pagination_required": paginator.count > SCHEDULER_CONFIG.EXECUTIONS_IN_PAGE, + "executions": page_obj, + "page_range": page_range, + "page_var": "p", + } + ) + + return super(TaskAdmin, self).change_view(request, object_id, form_url, extra_context=extra) + + def delete_queryset(self, request, queryset): + for job in queryset: + job.unschedule() + super(TaskAdmin, self).delete_queryset(request, queryset) + + def delete_model(self, request, obj): + obj.unschedule() + super(TaskAdmin, self).delete_model(request, obj) + + @admin.action(description=_("Disable selected %(verbose_name_plural)s"), permissions=("change",)) + def disable_selected(self, request, queryset): + rows_updated = 0 + for obj in queryset.filter(enabled=True).iterator(): + obj.enabled = False + obj.unschedule() + rows_updated += 1 + + message_bit = "1 job was" if rows_updated == 1 else f"{rows_updated} jobs were" + + level = messages.WARNING if not rows_updated else messages.INFO + self.message_user(request, f"{message_bit} successfully disabled and unscheduled.", level=level) + + @admin.action(description=_("Enable selected %(verbose_name_plural)s"), permissions=("change",)) + def enable_selected(self, request, queryset): + rows_updated = 0 + for obj in queryset.filter(enabled=False).iterator(): + obj.enabled = True + obj.save() + rows_updated += 1 + + message_bit = "1 job was" if rows_updated == 1 else f"{rows_updated} jobs were" + level = messages.WARNING if not rows_updated else messages.INFO + self.message_user(request, f"{message_bit} successfully enabled and scheduled.", level=level) + + @admin.action(description="Enqueue now", permissions=("change",)) + def enqueue_job_now(self, request, queryset): + task_names = [] + for task in queryset: + task.enqueue_to_run() + task_names.append(task.name) + self.message_user( + request, + f"The following jobs have been enqueued: {', '.join(task_names)}", + ) diff --git a/scheduler/models/__init__.py b/scheduler/models/__init__.py index b05c19a..59baf5a 100644 --- a/scheduler/models/__init__.py +++ b/scheduler/models/__init__.py @@ -1,3 +1,4 @@ from .args import TaskKwarg, TaskArg, BaseTaskArg # noqa: F401 from .queue import Queue # noqa: F401 from .scheduled_task import BaseTask, ScheduledTask, RepeatableTask, CronTask # noqa: F401 +from .task import Task # noqa: F401 diff --git a/scheduler/models/scheduled_task.py b/scheduler/models/scheduled_task.py index 46fa882..d0251f4 100644 --- a/scheduler/models/scheduled_task.py +++ b/scheduler/models/scheduled_task.py @@ -395,7 +395,7 @@ class ScheduledTask(ScheduledTimeMixin, BaseTask): def ready_for_schedule(self) -> bool: return super(ScheduledTask, self).ready_for_schedule() and ( - self.scheduled_time is None or self.scheduled_time >= timezone.now() + self.scheduled_time is None or self.scheduled_time >= timezone.now() ) class Meta: diff --git a/scheduler/models/task.py b/scheduler/models/task.py new file mode 100644 index 0000000..0cc41c5 --- /dev/null +++ b/scheduler/models/task.py @@ -0,0 +1,458 @@ +import math +import uuid +from datetime import timedelta +from typing import Dict + +import croniter +from django.apps import apps +from django.conf import settings as django_settings +from django.contrib import admin +from django.contrib.contenttypes.fields import GenericRelation +from django.core.exceptions import ValidationError +from django.core.mail import mail_admins +from django.db import models +from django.templatetags.tz import utc +from django.urls import reverse +from django.utils import timezone +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from scheduler import settings +from scheduler import tools +from scheduler.models.args import TaskArg, TaskKwarg +from scheduler.queues import get_queue +from scheduler.rq_classes import DjangoQueue +from scheduler.settings import QUEUES +from scheduler.settings import logger + +SCHEDULER_INTERVAL = settings.SCHEDULER_CONFIG.SCHEDULER_INTERVAL + + +def failure_callback(job, connection, result, *args, **kwargs): + task_type = job.meta.get("task_type", None) + if task_type is None: + return + task = Task.objects.filter(job_id=job.id).first() + if task is None: + logger.warn(f"Could not find task for job {job.id}") + return + mail_admins( + f"Task {task.id}/{task.name} has failed", + "See django-admin for logs", + ) + task.job_id = None + task.failed_runs += 1 + task.last_failed_run = timezone.now() + task.save(schedule_job=True) + + +def success_callback(job, connection, result, *args, **kwargs): + model_name = job.meta.get("task_type", None) + if model_name is None: + return + model = apps.get_model(app_label="scheduler", model_name=model_name) + task = model.objects.filter(job_id=job.id).first() + if task is None: + return + task.job_id = None + task.successful_runs += 1 + task.last_successful_run = timezone.now() + task.save(schedule_job=True) + + +def get_queue_choices(): + return [(queue, queue) for queue in QUEUES.keys()] + + +class Task(models.Model): + class TaskType(models.TextChoices): + CRON = "CronTask", _("Cron Task") + REPEATABLE = "RepeatableTask", _("Repeatable Task") + ONCE = "OnceTask", _("Run once") + + class TimeUnits(models.TextChoices): + SECONDS = "seconds", _("seconds") + MINUTES = "minutes", _("minutes") + HOURS = "hours", _("hours") + DAYS = "days", _("days") + WEEKS = "weeks", _("weeks") + + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + name = models.CharField(_("name"), max_length=128, unique=True, help_text=_("Name of the job")) + task_type = models.CharField(_("Task type"), max_length=32, choices=TaskType.choices, default=TaskType.ONCE) + callable = models.CharField(_("callable"), max_length=2048) + callable_args = GenericRelation(TaskArg, related_query_name="args") + callable_kwargs = GenericRelation(TaskKwarg, related_query_name="kwargs") + enabled = models.BooleanField( + _("enabled"), + default=True, + help_text=_( + "Should job be scheduled? This field is useful to keep past jobs that should no longer be scheduled" + ), + ) + queue = models.CharField(_("queue"), max_length=255, choices=get_queue_choices, help_text=_("Queue name")) + job_id = models.CharField( + _("job id"), max_length=128, editable=False, blank=True, null=True, + help_text=_("Current job_id on queue") + ) + at_front = models.BooleanField( + _("At front"), + default=False, + blank=True, + null=True, + help_text=_("When queuing the job, add it in the front of the queue"), + ) + timeout = models.IntegerField( + _("timeout"), + blank=True, + null=True, + help_text=_( + "Timeout specifies the maximum runtime, in seconds, for the job " + "before it'll be considered 'lost'. Blank uses the default " + "timeout." + ), + ) + result_ttl = models.IntegerField( + _("result ttl"), + blank=True, + null=True, + help_text=mark_safe( + """The TTL value (in seconds) of the job result.
+ -1: Result never expires, you should delete jobs manually.
+ 0: Result gets deleted immediately.
+ >0: Result expires after n seconds.""" + ), + ) + failed_runs = models.PositiveIntegerField( + _("failed runs"), + default=0, + help_text=_("Number of times the task has failed"), + ) + successful_runs = models.PositiveIntegerField( + _("successful runs"), + default=0, + help_text=_("Number of times the task has succeeded"), + ) + last_successful_run = models.DateTimeField( + _("last successful run"), + blank=True, + null=True, + help_text=_("Last time the task has succeeded"), + ) + last_failed_run = models.DateTimeField( + _("last failed run"), + blank=True, + null=True, + help_text=_("Last time the task has failed"), + ) + interval = models.PositiveIntegerField( + _("interval"), blank=True, null=True, + help_text=_("Interval for repeatable task"), ) + interval_unit = models.CharField( + _("interval unit"), max_length=12, choices=TimeUnits.choices, default=TimeUnits.HOURS, blank=True, null=True, + ) + repeat = models.PositiveIntegerField( + _("repeat"), + blank=True, + null=True, + help_text=_("Number of times to run the job. Leaving this blank means it will run forever."), + ) + scheduled_time = models.DateTimeField(_("scheduled time")) + cron_string = models.CharField( + _("cron string"), + max_length=64, blank=True, null=True, + help_text=mark_safe( + """Define the schedule in a crontab like syntax. + Times are in UTC. Use crontab.guru to create a cron string.""" + ), + ) + + def callable_func(self): + """Translate callable string to callable""" + return tools.callable_func(self.callable) + + @admin.display(boolean=True, description=_("is scheduled?")) + def is_scheduled(self) -> bool: + """Check whether a next job for this task is queued/scheduled to be executed""" + if self.job_id is None: # no job_id => is not scheduled + return False + # check whether job_id is in scheduled/queued/active jobs + scheduled_jobs = self.rqueue.scheduled_job_registry.get_job_ids() + enqueued_jobs = self.rqueue.get_job_ids() + active_jobs = self.rqueue.started_job_registry.get_job_ids() + res = (self.job_id in scheduled_jobs) or (self.job_id in enqueued_jobs) or (self.job_id in active_jobs) + # If the job_id is not scheduled/queued/started, + # update the job_id to None. (The job_id belongs to a previous run which is completed) + if not res: + self.job_id = None + super(Task, self).save() + return res + + @admin.display(description="Callable") + def function_string(self) -> str: + args = self.parse_args() + args_list = [repr(arg) for arg in args] + kwargs = self.parse_kwargs() + kwargs_list = [k + "=" + repr(v) for (k, v) in kwargs.items()] + return self.callable + f"({', '.join(args_list + kwargs_list)})" + + def parse_args(self): + """Parse args for running the job""" + args = self.callable_args.all() + return [arg.value() for arg in args] + + def parse_kwargs(self): + """Parse kwargs for running the job""" + kwargs = self.callable_kwargs.all() + return dict([kwarg.value() for kwarg in kwargs]) + + def _next_job_id(self): + addition = uuid.uuid4().hex[-10:] + name = self.name.replace("/", ".") + return f"{self.queue}:{name}:{addition}" + + def _enqueue_args(self) -> Dict: + """Args for DjangoQueue.enqueue. + Set all arguments for DjangoQueue.enqueue/enqueue_at. + Particularly: + - set job timeout and ttl + - ensure a callback to reschedule the job next iteration. + - Set job-id to proper format + - set job meta + """ + res = dict( + meta=dict( + task_type=self.task_type, + scheduled_task_id=self.id, + ), + on_success=success_callback, + on_failure=failure_callback, + job_id=self._next_job_id(), + ) + if self.at_front: + res["at_front"] = self.at_front + if self.timeout: + res["job_timeout"] = self.timeout + if self.result_ttl is not None: + res["result_ttl"] = self.result_ttl + if self.task_type == self.TaskType.REPEATABLE: + res["meta"]["interval"] = self.interval_seconds() + res["meta"]["repeat"] = self.repeat + return res + + @property + def rqueue(self) -> DjangoQueue: + """Returns django-queue for job""" + return get_queue(self.queue) + + def ready_for_schedule(self) -> bool: + """Is the task ready to be scheduled? + + If the task is already scheduled or disabled, then it is not + ready to be scheduled. + + :returns: True if the task is ready to be scheduled. + """ + if self.is_scheduled(): + logger.debug(f"Task {self.name} already scheduled") + return False + if not self.enabled: + logger.debug(f"Task {str(self)} disabled, enable task before scheduling") + return False + if self.task_type == Task.TaskType.REPEATABLE and self._schedule_time() < timezone.now(): + return False + return True + + def schedule(self) -> bool: + """Schedule the next execution for the task to run. + :returns: True if a job was scheduled, False otherwise. + """ + if not self.ready_for_schedule(): + return False + schedule_time = self._schedule_time() + kwargs = self._enqueue_args() + job = self.rqueue.enqueue_at( + schedule_time, + tools.run_task, + args=(self.task_type, self.id), + **kwargs, + ) + self.job_id = job.id + super(Task, self).save() + return True + + def enqueue_to_run(self) -> bool: + """Enqueue task to run now.""" + kwargs = self._enqueue_args() + job = self.rqueue.enqueue( + tools.run_task, + args=(self.task_type, self.id), + **kwargs, + ) + self.job_id = job.id + self.save(schedule_job=False) + return True + + def unschedule(self) -> bool: + """Remove a job from django-queue. + + If a job is queued to be executed or scheduled to be executed, it will remove it. + """ + queue = self.rqueue + if self.job_id is None: + return True + queue.remove(self.job_id) + queue.scheduled_job_registry.remove(self.job_id) + self.job_id = None + self.save(schedule_job=False) + return True + + def _schedule_time(self): + if self.task_type == self.TaskType.CRON: + self.scheduled_time = tools.get_next_cron_time(self.cron_string) + elif self.task_type == self.TaskType.REPEATABLE: + _now = timezone.now() + if self.scheduled_time >= _now: + return utc(self.scheduled_time) if django_settings.USE_TZ else self.scheduled_time + gap = math.ceil((_now.timestamp() - self.scheduled_time.timestamp()) / self.interval_seconds()) + if self.repeat is None or self.repeat >= gap: + self.scheduled_time += timedelta(seconds=self.interval_seconds() * gap) + self.repeat = (self.repeat - gap) if self.repeat is not None else None + return utc(self.scheduled_time) if django_settings.USE_TZ else self.scheduled_time + + def to_dict(self) -> Dict: + """Export model to dictionary, so it can be saved as external file backup""" + res = dict( + model=self.task_type, + name=self.name, + callable=self.callable, + callable_args=[ + dict( + arg_type=arg.arg_type, + val=arg.val, + ) + for arg in self.callable_args.all() + ], + callable_kwargs=[ + dict( + arg_type=arg.arg_type, + key=arg.key, + val=arg.val, + ) + for arg in self.callable_kwargs.all() + ], + enabled=self.enabled, + queue=self.queue, + repeat=getattr(self, "repeat", None), + at_front=self.at_front, + timeout=self.timeout, + result_ttl=self.result_ttl, + cron_string=getattr(self, "cron_string", None), + scheduled_time=self._schedule_time().isoformat(), + interval=getattr(self, "interval", None), + interval_unit=getattr(self, "interval_unit", None), + successful_runs=getattr(self, "successful_runs", None), + failed_runs=getattr(self, "failed_runs", None), + last_successful_run=getattr(self, "last_successful_run", None), + last_failed_run=getattr(self, "last_failed_run", None), + ) + return res + + def get_absolute_url(self): + model = self._meta.model.__name__.lower() + return reverse( + f"admin:scheduler_{model}_change", + args=[ + self.id, + ], + ) + + def __str__(self): + func = self.function_string() + return f"{self.task_type}[{self.name}={func}]" + + def save(self, **kwargs): + schedule_job = kwargs.pop("schedule_job", True) + update_fields = kwargs.get("update_fields", None) + if update_fields is not None: + kwargs["update_fields"] = set(update_fields).union({"modified"}) + super(Task, self).save(**kwargs) + if schedule_job: + self.schedule() + super(Task, self).save() + + def delete(self, **kwargs): + self.unschedule() + super(Task, self).delete(**kwargs) + + def interval_display(self): + return "{} {}".format(self.interval, self.get_interval_unit_display()) + + def interval_seconds(self): + kwargs = { + self.interval_unit: self.interval, + } + return timedelta(**kwargs).total_seconds() + + def clean_callable(self): + try: + tools.callable_func(self.callable) + except Exception: + raise ValidationError( + {"callable": ValidationError(_("Invalid callable, must be importable"), code="invalid")} + ) + + def clean_queue(self): + queue_keys = settings.QUEUES.keys() + if self.queue not in queue_keys: + raise ValidationError( + { + "queue": ValidationError( + _("Invalid queue, must be one of: {}".format(", ".join(queue_keys))), code="invalid" + ) + } + ) + + def clean_interval_unit(self): + if SCHEDULER_INTERVAL > self.interval_seconds(): + raise ValidationError( + _("Job interval is set lower than %(queue)r queue's interval. " "minimum interval is %(interval)"), + code="invalid", + params={"queue": self.queue, "interval": SCHEDULER_INTERVAL}, + ) + if self.interval_seconds() % SCHEDULER_INTERVAL: + raise ValidationError( + _("Job interval is not a multiple of rq_scheduler's interval frequency: %(interval)ss"), + code="invalid", + params={"interval": SCHEDULER_INTERVAL}, + ) + + def clean_result_ttl(self) -> None: + """Throws an error if there are repeats left to run and the result_ttl won't last until the next scheduled time. + :return: None + """ + if self.result_ttl and self.result_ttl != -1 and self.result_ttl < self.interval_seconds() and self.repeat: + raise ValidationError( + _( + "Job result_ttl must be either indefinite (-1) or " + "longer than the interval, %(interval)s seconds, to ensure rescheduling." + ), + code="invalid", + params={"interval": self.interval_seconds()}, + ) + + def clean_cron_string(self): + try: + croniter.croniter(self.cron_string) + except ValueError as e: + raise ValidationError({"cron_string": ValidationError(_(str(e)), code="invalid")}) + + def clean(self): + self.clean_queue() + self.clean_callable() + if self.task_type == self.TaskType.CRON: + self.clean_cron_string() + if self.task_type == self.TaskType.REPEATABLE: + self.clean_interval_unit() + self.clean_result_ttl() From 8cd41d803189117134d39661dd64c7982d07977f Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 8 Oct 2024 11:02:29 -0400 Subject: [PATCH 02/47] black --- scheduler/admin/__init__.py | 2 +- scheduler/management/commands/rqworker.py | 5 +---- scheduler/models/scheduled_task.py | 2 +- scheduler/models/task.py | 21 +++++++++++++++------ scheduler/queues.py | 2 +- scheduler/rq_classes.py | 15 ++++++++------- 6 files changed, 27 insertions(+), 20 deletions(-) diff --git a/scheduler/admin/__init__.py b/scheduler/admin/__init__.py index fb58874..47abb5b 100644 --- a/scheduler/admin/__init__.py +++ b/scheduler/admin/__init__.py @@ -1,3 +1,3 @@ from .task_models import TaskAdmin as OldTaskAdmin # noqa: F401 from .ephemeral_models import QueueAdmin, WorkerAdmin # noqa: F401 -from .task_admin import TaskAdmin # noqa: F401 \ No newline at end of file +from .task_admin import TaskAdmin # noqa: F401 diff --git a/scheduler/management/commands/rqworker.py b/scheduler/management/commands/rqworker.py index 99f0573..14b9ae8 100644 --- a/scheduler/management/commands/rqworker.py +++ b/scheduler/management/commands/rqworker.py @@ -118,10 +118,7 @@ def handle(self, **options): try: # Instantiate a worker - w = create_worker( - *queues, - **init_options - ) + w = create_worker(*queues, **init_options) # Close any opened DB connection before any fork reset_db_connections() diff --git a/scheduler/models/scheduled_task.py b/scheduler/models/scheduled_task.py index d0251f4..46fa882 100644 --- a/scheduler/models/scheduled_task.py +++ b/scheduler/models/scheduled_task.py @@ -395,7 +395,7 @@ class ScheduledTask(ScheduledTimeMixin, BaseTask): def ready_for_schedule(self) -> bool: return super(ScheduledTask, self).ready_for_schedule() and ( - self.scheduled_time is None or self.scheduled_time >= timezone.now() + self.scheduled_time is None or self.scheduled_time >= timezone.now() ) class Meta: diff --git a/scheduler/models/task.py b/scheduler/models/task.py index 0cc41c5..534d5ba 100644 --- a/scheduler/models/task.py +++ b/scheduler/models/task.py @@ -93,8 +93,7 @@ class TimeUnits(models.TextChoices): ) queue = models.CharField(_("queue"), max_length=255, choices=get_queue_choices, help_text=_("Queue name")) job_id = models.CharField( - _("job id"), max_length=128, editable=False, blank=True, null=True, - help_text=_("Current job_id on queue") + _("job id"), max_length=128, editable=False, blank=True, null=True, help_text=_("Current job_id on queue") ) at_front = models.BooleanField( _("At front"), @@ -147,10 +146,18 @@ class TimeUnits(models.TextChoices): help_text=_("Last time the task has failed"), ) interval = models.PositiveIntegerField( - _("interval"), blank=True, null=True, - help_text=_("Interval for repeatable task"), ) + _("interval"), + blank=True, + null=True, + help_text=_("Interval for repeatable task"), + ) interval_unit = models.CharField( - _("interval unit"), max_length=12, choices=TimeUnits.choices, default=TimeUnits.HOURS, blank=True, null=True, + _("interval unit"), + max_length=12, + choices=TimeUnits.choices, + default=TimeUnits.HOURS, + blank=True, + null=True, ) repeat = models.PositiveIntegerField( _("repeat"), @@ -161,7 +168,9 @@ class TimeUnits(models.TextChoices): scheduled_time = models.DateTimeField(_("scheduled time")) cron_string = models.CharField( _("cron string"), - max_length=64, blank=True, null=True, + max_length=64, + blank=True, + null=True, help_text=mark_safe( """Define the schedule in a crontab like syntax. Times are in UTC. Use crontab.guru to create a cron string.""" diff --git a/scheduler/queues.py b/scheduler/queues.py index 636f67b..3cff9fb 100644 --- a/scheduler/queues.py +++ b/scheduler/queues.py @@ -87,7 +87,7 @@ def get_connection(queue_settings, use_strict_redis=False): def get_queue( - name="default", default_timeout=None, is_async=None, autocommit=None, connection=None, **kwargs + name="default", default_timeout=None, is_async=None, autocommit=None, connection=None, **kwargs ) -> DjangoQueue: """Returns an DjangoQueue using parameters defined in `SCHEDULER_QUEUES`""" from .settings import QUEUES diff --git a/scheduler/rq_classes.py b/scheduler/rq_classes.py index 1a91c48..4050c6c 100644 --- a/scheduler/rq_classes.py +++ b/scheduler/rq_classes.py @@ -62,8 +62,9 @@ def is_scheduled_task(self) -> bool: return self.meta.get("scheduled_task_id", None) is not None def is_execution_of(self, task: "ScheduledTask") -> bool: # noqa: F821 - return (self.meta.get("task_type", None) == task.TASK_TYPE - and self.meta.get("scheduled_task_id", None) == task.id) + return ( + self.meta.get("task_type", None) == task.TASK_TYPE and self.meta.get("scheduled_task_id", None) == task.id + ) def stop_execution(self, connection: ConnectionType): send_stop_job_command(connection, self.id) @@ -91,11 +92,11 @@ def __str__(self): return f"{self.name}/{','.join(self.queue_names())}" def _start_scheduler( - self, - burst: bool = False, - logging_level: str = "INFO", - date_format: str = "%H:%M:%S", - log_format: str = "%(asctime)s %(message)s", + self, + burst: bool = False, + logging_level: str = "INFO", + date_format: str = "%H:%M:%S", + log_format: str = "%(asctime)s %(message)s", ) -> None: """Starts the scheduler process. This is specifically designed to be run by the worker when running the `work()` method. From 8444a51d94051b5d3949b4024d626d791e34583a Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 8 Oct 2024 14:07:31 -0400 Subject: [PATCH 03/47] black --- scheduler/admin/task_admin.py | 174 ++++++---------------------------- scheduler/models/task.py | 2 - 2 files changed, 29 insertions(+), 147 deletions(-) diff --git a/scheduler/admin/task_admin.py b/scheduler/admin/task_admin.py index 80265f4..95221d3 100644 --- a/scheduler/admin/task_admin.py +++ b/scheduler/admin/task_admin.py @@ -20,129 +20,18 @@ class Media: class JobArgInline(HiddenMixin, GenericStackedInline): model = TaskArg extra = 0 - fieldsets = ( - ( - None, - { - "fields": ( - ( - "arg_type", - "val", - ), - ), - }, - ), - ) + fieldsets = ((None, dict(fields=("arg_type", "val"))),) class JobKwargInline(HiddenMixin, GenericStackedInline): model = TaskKwarg extra = 0 - fieldsets = ( - ( - None, - { - "fields": ( - ("key",), - ( - "arg_type", - "val", - ), - ), - }, - ), - ) - - -_LIST_DISPLAY_EXTRA = dict( - CronTask=( - "cron_string", - "next_run", - "successful_runs", - "last_successful_run", - "failed_runs", - "last_failed_run", - ), - ScheduledTask=("scheduled_time",), - RepeatableTask=( - "scheduled_time", - "interval_display", - "successful_runs", - "last_successful_run", - "failed_runs", - "last_failed_run", - ), - Task=( - "scheduled_time", - "interval_display", - "cron_string", - "next_run", - "successful_runs", - "last_successful_run", - "failed_runs", - "last_failed_run", - ), -) -_FIELDSET_EXTRA = dict( - CronTask=( - "cron_string", - "timeout", - "result_ttl", - ( - "successful_runs", - "last_successful_run", - ), - ( - "failed_runs", - "last_failed_run", - ), - ), - ScheduledTask=("scheduled_time", "timeout", "result_ttl"), - RepeatableTask=( - "scheduled_time", - ( - "interval", - "interval_unit", - ), - "repeat", - "timeout", - "result_ttl", - ( - "successful_runs", - "last_successful_run", - ), - ( - "failed_runs", - "last_failed_run", - ), - ), - Task=( - "scheduled_time", - "cron_string", - ( - "interval", - "interval_unit", - ), - "repeat", - "timeout", - "result_ttl", - ( - "successful_runs", - "last_successful_run", - ), - ( - "failed_runs", - "last_failed_run", - ), - ), -) + fieldsets = ((None, dict(fields=("key", ("arg_type", "val")))),) @admin.register(Task) class TaskAdmin(admin.ModelAdmin): - """TaskAdmin admin view for all task models. - Using the _LIST_DISPLAY_EXTRA and _FIELDSET_EXTRA additional data for each model. - """ + """TaskAdmin admin view for all task models.""" save_on_top = True change_form_template = "admin/scheduler/change_form.html" @@ -163,49 +52,44 @@ class TaskAdmin(admin.ModelAdmin): "function_string", "is_scheduled", "queue", + "scheduled_time", + "interval_display", + "cron_string", + "next_run", + "successful_runs", + "last_successful_run", + "failed_runs", + "last_failed_run", ) list_display_links = ("name",) - readonly_fields = ("job_id",) + readonly_fields = ( + "job_id", + "successful_runs", + "last_successful_run", + "failed_runs", + "last_failed_run", + ) + radio_fields = {"task_type": admin.HORIZONTAL} fieldsets = ( ( None, - { - "fields": ( + dict( + fields=( "name", "callable", - "enabled", - "at_front", - ), - }, + "task_type", + ("enabled", "timeout", "result_ttl"), + ("scheduled_time", "cron_string", "interval", "interval_unit", "repeat"), + ) + ), ), + (_("RQ Settings"), dict(fields=(("queue", "at_front"), "job_id"))), ( - _("RQ Settings"), - { - "fields": ( - "queue", - "job_id", - ), - }, + _("Previous runs info"), + dict(fields=(("successful_runs", "last_successful_run"), ("failed_runs", "last_failed_run"))), ), ) - def get_list_display(self, request): - if self.model.__name__ not in _LIST_DISPLAY_EXTRA: - raise ValueError(f"Unrecognized model {self.model}") - return TaskAdmin.list_display + _LIST_DISPLAY_EXTRA[self.model.__name__] - - def get_fieldsets(self, request, obj=None): - if self.model.__name__ not in _FIELDSET_EXTRA: - raise ValueError(f"Unrecognized model {self.model}") - return TaskAdmin.fieldsets + ( - ( - _("Scheduling"), - { - "fields": _FIELDSET_EXTRA[self.model.__name__], - }, - ), - ) - @admin.display(description="Next run") def next_run(self, o: Task): return tools.get_next_cron_time(o.cron_string) diff --git a/scheduler/models/task.py b/scheduler/models/task.py index 534d5ba..71abdc6 100644 --- a/scheduler/models/task.py +++ b/scheduler/models/task.py @@ -98,8 +98,6 @@ class TimeUnits(models.TextChoices): at_front = models.BooleanField( _("At front"), default=False, - blank=True, - null=True, help_text=_("When queuing the job, add it in the front of the queue"), ) timeout = models.IntegerField( From b2c2263fbddfa24b14612733de9ade555cd1edf9 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 8 Oct 2024 15:00:53 -0400 Subject: [PATCH 04/47] wip --- scheduler/management/commands/export.py | 2 +- scheduler/management/commands/import.py | 33 ++++++++++++++++--------- scheduler/rq_classes.py | 4 +-- scheduler/tools.py | 6 +++-- testproject/testproject/settings.py | 14 +++++------ 5 files changed, 36 insertions(+), 23 deletions(-) diff --git a/scheduler/management/commands/export.py b/scheduler/management/commands/export.py index 6a83595..bb2b249 100644 --- a/scheduler/management/commands/export.py +++ b/scheduler/management/commands/export.py @@ -54,7 +54,7 @@ def handle(self, *args, **options): if options.get("format") == "json": import json - click.echo(json.dumps(res, indent=2), file=file) + click.echo(json.dumps(res, indent=2, default=str), file=file) return if options.get("format") == "yaml": diff --git a/scheduler/management/commands/import.py b/scheduler/management/commands/import.py index b7bbfe9..a6b9b20 100644 --- a/scheduler/management/commands/import.py +++ b/scheduler/management/commands/import.py @@ -8,7 +8,7 @@ from django.core.management.base import BaseCommand from django.utils import timezone -from scheduler.models import TaskArg, TaskKwarg +from scheduler.models import TaskArg, TaskKwarg, Task from scheduler.tools import MODEL_NAMES @@ -17,10 +17,20 @@ def job_model_str(model_str: str) -> str: return model_str[:-3] + "Task" return model_str - -def create_job_from_dict(job_dict: Dict[str, Any], update): - model = apps.get_model(app_label="scheduler", model_name=job_model_str(job_dict["model"])) - existing_job = model.objects.filter(name=job_dict["name"]).first() +def get_task_type(model_str: str) -> Task.TaskType: + model_str = job_model_str(model_str) + if model_str not in MODEL_NAMES: + raise ValueError(f"Invalid model {model_str}") + if model_str == "CronTask": + return Task.TaskType.CRON + elif model_str == "RepeatableTask": + return Task.TaskType.REPEATABLE + elif model_str == "ScheduledTask": + return Task.TaskType.ONCE + +def create_task_from_dict(task_dict: Dict[str, Any], update): + existing_job = Task.objects.filter(name=task_dict["name"]).first() + task_type = get_task_type(task_dict["model"]) if existing_job: if update: click.echo(f'Found existing job "{existing_job}, removing it to be reinserted"') @@ -28,7 +38,8 @@ def create_job_from_dict(job_dict: Dict[str, Any], update): else: click.echo(f'Found existing job "{existing_job}", skipping') return - kwargs = dict(job_dict) + kwargs = dict(task_dict) + kwargs["task_type"] = task_type del kwargs["model"] del kwargs["callable_args"] del kwargs["callable_kwargs"] @@ -37,21 +48,21 @@ def create_job_from_dict(job_dict: Dict[str, Any], update): if not settings.USE_TZ and not timezone.is_naive(target): target = timezone.make_naive(target) kwargs["scheduled_time"] = target - model_fields = set(map(lambda field: field.attname, model._meta.get_fields())) + model_fields = set(map(lambda field: field.attname, Task._meta.get_fields())) keys_to_ignore = list(filter(lambda _k: _k not in model_fields, kwargs.keys())) for k in keys_to_ignore: del kwargs[k] - scheduled_job = model.objects.create(**kwargs) + scheduled_job = Task.objects.create(**kwargs) click.echo(f"Created job {scheduled_job}") content_type = ContentType.objects.get_for_model(scheduled_job) - for arg in job_dict["callable_args"]: + for arg in task_dict["callable_args"]: TaskArg.objects.create( content_type=content_type, object_id=scheduled_job.id, **arg, ) - for arg in job_dict["callable_kwargs"]: + for arg in task_dict["callable_kwargs"]: TaskKwarg.objects.create( content_type=content_type, object_id=scheduled_job.id, @@ -125,4 +136,4 @@ def handle(self, *args, **options): model.objects.all().delete() for job in jobs: - create_job_from_dict(job, update=options.get("update")) + create_task_from_dict(job, update=options.get("update")) diff --git a/scheduler/rq_classes.py b/scheduler/rq_classes.py index 4050c6c..2ed76da 100644 --- a/scheduler/rq_classes.py +++ b/scheduler/rq_classes.py @@ -61,9 +61,9 @@ def __eq__(self, other) -> bool: def is_scheduled_task(self) -> bool: return self.meta.get("scheduled_task_id", None) is not None - def is_execution_of(self, task: "ScheduledTask") -> bool: # noqa: F821 + def is_execution_of(self, task: "Task") -> bool: # noqa: F821 return ( - self.meta.get("task_type", None) == task.TASK_TYPE and self.meta.get("scheduled_task_id", None) == task.id + self.meta.get("task_type", None) == task.task_type and self.meta.get("scheduled_task_id", None) == task.id ) def stop_execution(self, connection: ConnectionType): diff --git a/scheduler/tools.py b/scheduler/tools.py index 3f5eac9..7aa615c 100644 --- a/scheduler/tools.py +++ b/scheduler/tools.py @@ -1,6 +1,6 @@ import importlib import os -from typing import List, Any, Callable +from typing import List, Any, Callable, Optional import croniter from django.apps import apps @@ -21,8 +21,10 @@ def callable_func(callable_str: str) -> Callable: return func -def get_next_cron_time(cron_string) -> timezone.datetime: +def get_next_cron_time(cron_string: Optional[str]) -> Optional[timezone.datetime]: """Calculate the next scheduled time by creating a crontab object with a cron string""" + if cron_string is None: + return None now = timezone.now() itr = croniter.croniter(cron_string, now) next_itr = itr.get_next(timezone.datetime) diff --git a/testproject/testproject/settings.py b/testproject/testproject/settings.py index 1ab283c..e076068 100644 --- a/testproject/testproject/settings.py +++ b/testproject/testproject/settings.py @@ -44,9 +44,9 @@ "default": { "BACKEND": "django.core.cache.backends.redis.RedisCache", "LOCATION": [ - "redis://127.0.0.1:6379", # leader + "redis://127.0.0.1:6379", ], - "OPTIONS": {"connection_class": FakeConnection}, + "BROKER": "fakeredis", } } TEMPLATES = [ @@ -115,19 +115,19 @@ STATIC_URL = "/static/" SCHEDULER_QUEUES = { "default": { - "URL": f"redis://localhost:${BROKER_PORT}/0", + "URL": f"redis://localhost:{BROKER_PORT}/0", }, "low": { - "URL": f"redis://localhost:${BROKER_PORT}/0", + "URL": f"redis://localhost:{BROKER_PORT}/0", }, "high": { - "URL": f"redis://localhost:${BROKER_PORT}/1", + "URL": f"redis://localhost:{BROKER_PORT}/1", }, "medium": { - "URL": f"redis://localhost:${BROKER_PORT}/1", + "URL": f"redis://localhost:{BROKER_PORT}/1", }, "another": { - "URL": f"redis://localhost:${BROKER_PORT}/1", + "URL": f"redis://localhost:{BROKER_PORT}/1", }, } DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" From 35f718c9675d9a8c7ab41cf27db4885f4a8be726 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 8 Oct 2024 17:52:58 -0400 Subject: [PATCH 05/47] wip --- scheduler/models/scheduled_task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scheduler/models/scheduled_task.py b/scheduler/models/scheduled_task.py index 46fa882..f5fa18e 100644 --- a/scheduler/models/scheduled_task.py +++ b/scheduler/models/scheduled_task.py @@ -294,8 +294,8 @@ def to_dict(self) -> Dict: scheduled_time=self._schedule_time().isoformat(), interval=getattr(self, "interval", None), interval_unit=getattr(self, "interval_unit", None), - successful_runs=getattr(self, "successful_runs", None), - failed_runs=getattr(self, "failed_runs", None), + successful_runs=getattr(self, "successful_runs", 0), + failed_runs=getattr(self, "failed_runs", 0), last_successful_run=getattr(self, "last_successful_run", None), last_failed_run=getattr(self, "last_failed_run", None), ) From bf6f0ee16104111df320abbac3cbcd6f05f247b9 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Fri, 11 Oct 2024 09:01:11 -0400 Subject: [PATCH 06/47] wip --- docs/changelog.md | 11 + pyproject.toml | 2 +- scheduler/management/commands/import.py | 9 +- scheduler/models/scheduled_task.py | 18 +- scheduler/models/task.py | 22 +- scheduler/tests/test_cron_task.py | 2 +- scheduler/tests/test_internals.py | 4 +- scheduler/tests/test_mgmt_cmds.py | 27 +- .../{test_models.py => test_old_models.py} | 8 +- scheduler/tests/test_repeatable_task.py | 2 +- scheduler/tests/test_task_model.py | 551 ++++++++++++++++++ scheduler/tests/testtools.py | 2 +- scheduler/tools.py | 2 +- 13 files changed, 611 insertions(+), 49 deletions(-) rename scheduler/tests/{test_models.py => test_old_models.py} (99%) create mode 100644 scheduler/tests/test_task_model.py diff --git a/docs/changelog.md b/docs/changelog.md index c164b3b..7ea1e0c 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,16 @@ # Changelog + +## v2.2.0 🌈 + +### 🚀 Features + +- Created a new `Task` model representing all kind of scheduled tasks. + - In future versions, `CronTask`, `ScheduledTask` and `RepeatableTask` will be removed. + - `Task` model has a `task_type` field to differentiate between the types of tasks. + - Old tasks in the database will be migrated to the new `Task` model automatically. + + ## v2.1.0 🌈 ### 🚀 Features diff --git a/pyproject.toml b/pyproject.toml index 2e375aa..a9cca6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "django-tasks-scheduler" packages = [ { include = "scheduler" }, ] -version = "2.1.0" +version = "2.2.0" description = "An async job scheduler for django using redis" readme = "README.md" keywords = ["redis", "django", "background-jobs", "job-queue", "task-queue", "redis-queue", "scheduled-jobs"] diff --git a/scheduler/management/commands/import.py b/scheduler/management/commands/import.py index a6b9b20..2e574cb 100644 --- a/scheduler/management/commands/import.py +++ b/scheduler/management/commands/import.py @@ -9,6 +9,7 @@ from django.utils import timezone from scheduler.models import TaskArg, TaskKwarg, Task +from scheduler.models.task import TaskType from scheduler.tools import MODEL_NAMES @@ -17,16 +18,16 @@ def job_model_str(model_str: str) -> str: return model_str[:-3] + "Task" return model_str -def get_task_type(model_str: str) -> Task.TaskType: +def get_task_type(model_str: str) -> TaskType: model_str = job_model_str(model_str) if model_str not in MODEL_NAMES: raise ValueError(f"Invalid model {model_str}") if model_str == "CronTask": - return Task.TaskType.CRON + return TaskType.CRON elif model_str == "RepeatableTask": - return Task.TaskType.REPEATABLE + return TaskType.REPEATABLE elif model_str == "ScheduledTask": - return Task.TaskType.ONCE + return TaskType.ONCE def create_task_from_dict(task_dict: Dict[str, Any], update): existing_job = Task.objects.filter(name=task_dict["name"]).first() diff --git a/scheduler/models/scheduled_task.py b/scheduler/models/scheduled_task.py index f5fa18e..7b8c658 100644 --- a/scheduler/models/scheduled_task.py +++ b/scheduler/models/scheduled_task.py @@ -70,7 +70,7 @@ def get_queue_choices(): class BaseTask(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - TASK_TYPE = "BaseTask" + task_type = "BaseTask" name = models.CharField( _("name"), max_length=128, @@ -180,7 +180,7 @@ def _enqueue_args(self) -> Dict: """ res = dict( meta=dict( - task_type=self.TASK_TYPE, + task_type=self.task_type, scheduled_task_id=self.id, ), on_success=success_callback, @@ -227,7 +227,7 @@ def schedule(self) -> bool: job = self.rqueue.enqueue_at( schedule_time, tools.run_task, - args=(self.TASK_TYPE, self.id), + args=(self.task_type, self.id), **kwargs, ) self.job_id = job.id @@ -239,7 +239,7 @@ def enqueue_to_run(self) -> bool: kwargs = self._enqueue_args() job = self.rqueue.enqueue( tools.run_task, - args=(self.TASK_TYPE, self.id), + args=(self.task_type, self.id), **kwargs, ) self.job_id = job.id @@ -266,7 +266,7 @@ def _schedule_time(self): def to_dict(self) -> Dict: """Export model to dictionary, so it can be saved as external file backup""" res = dict( - model=self.TASK_TYPE, + model=self.task_type, name=self.name, callable=self.callable, callable_args=[ @@ -312,7 +312,7 @@ def get_absolute_url(self): def __str__(self): func = self.function_string() - return f"{self.TASK_TYPE}[{self.name}={func}]" + return f"{self.task_type}[{self.name}={func}]" def save(self, **kwargs): schedule_job = kwargs.pop("schedule_job", True) @@ -391,7 +391,7 @@ class Meta: class ScheduledTask(ScheduledTimeMixin, BaseTask): - TASK_TYPE = "ScheduledTask" + task_type = "ScheduledTask" def ready_for_schedule(self) -> bool: return super(ScheduledTask, self).ready_for_schedule() and ( @@ -405,6 +405,7 @@ class Meta: class RepeatableTask(RepeatableMixin, ScheduledTimeMixin, BaseTask): + task_type = "RepeatableTask" class TimeUnits(models.TextChoices): SECONDS = "seconds", _("seconds") MINUTES = "minutes", _("minutes") @@ -422,7 +423,6 @@ class TimeUnits(models.TextChoices): null=True, help_text=_("Number of times to run the job. Leaving this blank means it will run forever."), ) - TASK_TYPE = "RepeatableTask" def clean(self): super(RepeatableTask, self).clean() @@ -497,7 +497,7 @@ class Meta: class CronTask(RepeatableMixin, BaseTask): - TASK_TYPE = "CronTask" + task_type = "CronTask" cron_string = models.CharField( _("cron string"), diff --git a/scheduler/models/task.py b/scheduler/models/task.py index 71abdc6..e70b916 100644 --- a/scheduler/models/task.py +++ b/scheduler/models/task.py @@ -64,12 +64,12 @@ def get_queue_choices(): return [(queue, queue) for queue in QUEUES.keys()] -class Task(models.Model): - class TaskType(models.TextChoices): - CRON = "CronTask", _("Cron Task") - REPEATABLE = "RepeatableTask", _("Repeatable Task") - ONCE = "OnceTask", _("Run once") +class TaskType(models.TextChoices): + CRON = "CronTask", _("Cron Task") + REPEATABLE = "RepeatableTask", _("Repeatable Task") + ONCE = "OnceTask", _("Run once") +class Task(models.Model): class TimeUnits(models.TextChoices): SECONDS = "seconds", _("seconds") MINUTES = "minutes", _("minutes") @@ -243,7 +243,7 @@ def _enqueue_args(self) -> Dict: res["job_timeout"] = self.timeout if self.result_ttl is not None: res["result_ttl"] = self.result_ttl - if self.task_type == self.TaskType.REPEATABLE: + if self.task_type == TaskType.REPEATABLE: res["meta"]["interval"] = self.interval_seconds() res["meta"]["repeat"] = self.repeat return res @@ -267,7 +267,7 @@ def ready_for_schedule(self) -> bool: if not self.enabled: logger.debug(f"Task {str(self)} disabled, enable task before scheduling") return False - if self.task_type == Task.TaskType.REPEATABLE and self._schedule_time() < timezone.now(): + if self.task_type == TaskType.REPEATABLE and self._schedule_time() < timezone.now(): return False return True @@ -316,9 +316,9 @@ def unschedule(self) -> bool: return True def _schedule_time(self): - if self.task_type == self.TaskType.CRON: + if self.task_type == TaskType.CRON: self.scheduled_time = tools.get_next_cron_time(self.cron_string) - elif self.task_type == self.TaskType.REPEATABLE: + elif self.task_type == TaskType.REPEATABLE: _now = timezone.now() if self.scheduled_time >= _now: return utc(self.scheduled_time) if django_settings.USE_TZ else self.scheduled_time @@ -458,8 +458,8 @@ def clean_cron_string(self): def clean(self): self.clean_queue() self.clean_callable() - if self.task_type == self.TaskType.CRON: + if self.task_type == TaskType.CRON: self.clean_cron_string() - if self.task_type == self.TaskType.REPEATABLE: + if self.task_type == TaskType.REPEATABLE: self.clean_interval_unit() self.clean_result_ttl() diff --git a/scheduler/tests/test_cron_task.py b/scheduler/tests/test_cron_task.py index a64f9b7..7d3bd4c 100644 --- a/scheduler/tests/test_cron_task.py +++ b/scheduler/tests/test_cron_task.py @@ -3,7 +3,7 @@ from scheduler import settings from scheduler.models import CronTask from scheduler.tools import create_worker -from .test_models import BaseTestCases +from .test_old_models import BaseTestCases from .testtools import task_factory from ..queues import get_queue diff --git a/scheduler/tests/test_internals.py b/scheduler/tests/test_internals.py index a8cb491..f3759c3 100644 --- a/scheduler/tests/test_internals.py +++ b/scheduler/tests/test_internals.py @@ -10,8 +10,8 @@ class TestInternals(SchedulerBaseCase): def test_get_scheduled_job(self): task = task_factory(ScheduledTask, scheduled_time=timezone.now() - timedelta(hours=1)) - self.assertEqual(task, get_scheduled_task(task.TASK_TYPE, task.id)) + self.assertEqual(task, get_scheduled_task(task.task_type, task.id)) with self.assertRaises(ValueError): - get_scheduled_task(task.TASK_TYPE, task.id + 1) + get_scheduled_task(task.task_type, task.id + 1) with self.assertRaises(ValueError): get_scheduled_task("UNKNOWN_JOBTYPE", task.id) diff --git a/scheduler/tests/test_mgmt_cmds.py b/scheduler/tests/test_mgmt_cmds.py index 6257935..46b66f3 100644 --- a/scheduler/tests/test_mgmt_cmds.py +++ b/scheduler/tests/test_mgmt_cmds.py @@ -7,12 +7,13 @@ from django.core.management import call_command from django.test import TestCase -from scheduler.models import ScheduledTask, RepeatableTask +from scheduler.models import ScheduledTask, RepeatableTask, Task from scheduler.queues import get_queue from scheduler.tests.jobs import failing_job, test_job from scheduler.tests.testtools import task_factory from . import test_settings # noqa from .test_views import BaseTestCase +from ..models.task import TaskType from ..tools import create_worker @@ -231,8 +232,8 @@ def test_import__should_schedule_job(self): # act call_command("import", filename=self.tmpfile.name) # assert - self.assertEqual(1, ScheduledTask.objects.count()) - db_job = ScheduledTask.objects.first() + self.assertEqual(1, Task.objects.filter(task_type=TaskType.ONCE).count()) + db_job = Task.objects.filter(task_type=TaskType.ONCE).first() attrs = ["name", "queue", "callable", "enabled", "timeout"] for attr in attrs: self.assertEqual(getattr(jobs[0], attr), getattr(db_job, attr)) @@ -247,8 +248,8 @@ def test_import__should_schedule_job_yaml(self): # act call_command("import", filename=self.tmpfile.name, format="yaml") # assert - self.assertEqual(1, ScheduledTask.objects.count()) - db_job = ScheduledTask.objects.first() + self.assertEqual(1, Task.objects.filter(task_type=TaskType.ONCE).count()) + db_job = Task.objects.filter(task_type=TaskType.ONCE).objects.first() attrs = ["name", "queue", "callable", "enabled", "timeout"] for attr in attrs: self.assertEqual(getattr(jobs[0], attr), getattr(db_job, attr)) @@ -282,13 +283,13 @@ def test_import__should_schedule_job_reset(self): reset=True, ) # assert - self.assertEqual(1, ScheduledTask.objects.count()) - db_job = ScheduledTask.objects.first() + self.assertEqual(1, Task.objects.filter(task_type=TaskType.ONCE).count()) + db_job = Task.objects.filter(task_type=TaskType.ONCE).first() attrs = ["name", "queue", "callable", "enabled", "timeout"] for attr in attrs: self.assertEqual(getattr(jobs[0], attr), getattr(db_job, attr)) - self.assertEqual(1, RepeatableTask.objects.count()) - db_job = RepeatableTask.objects.first() + self.assertEqual(1, Task.objects.filter(task_type=TaskType.REPEATABLE).count()) + db_job = Task.objects.filter(task_type=TaskType.REPEATABLE).first() attrs = ["name", "queue", "callable", "enabled", "timeout"] for attr in attrs: self.assertEqual(getattr(jobs[1], attr), getattr(db_job, attr)) @@ -307,8 +308,8 @@ def test_import__should_schedule_job_update_existing(self): update=True, ) # assert - self.assertEqual(2, ScheduledTask.objects.count()) - db_job = ScheduledTask.objects.get(name=jobs[0].name) + self.assertEqual(2, Task.objects.filter(task_type=TaskType.ONCE).count()) + db_job = Task.objects.filter(task_type=TaskType.ONCE).get(name=jobs[0].name) self.assertNotEqual(jobs[0].id, db_job.id) attrs = ["name", "queue", "callable", "enabled", "timeout"] for attr in attrs: @@ -327,8 +328,8 @@ def test_import__should_schedule_job_without_update_existing(self): filename=self.tmpfile.name, ) # assert - self.assertEqual(2, ScheduledTask.objects.count()) - db_job = ScheduledTask.objects.get(name=jobs[0].name) + self.assertEqual(2, Task.objects.filter(task_type=TaskType.ONCE).count()) + db_job = Task.objects.get(name=jobs[0].name) attrs = ["id", "name", "queue", "callable", "enabled", "timeout"] for attr in attrs: self.assertEqual(getattr(jobs[0], attr), getattr(db_job, attr)) diff --git a/scheduler/tests/test_models.py b/scheduler/tests/test_old_models.py similarity index 99% rename from scheduler/tests/test_models.py rename to scheduler/tests/test_old_models.py index 2a1873d..a20e0ea 100644 --- a/scheduler/tests/test_models.py +++ b/scheduler/tests/test_old_models.py @@ -376,9 +376,7 @@ def test_admin_change_view__bad_redis_connection(self): def test_admin_enqueue_job_now(self): # arrange self.client.login(username="admin", password="admin") - task = task_factory( - self.TaskModelClass, - ) + task = task_factory(self.TaskModelClass) self.assertIsNotNone(task.job_id) self.assertTrue(task.is_scheduled()) data = { @@ -396,7 +394,7 @@ def test_admin_enqueue_job_now(self): self.assertEqual(200, res.status_code) entry = _get_job_from_scheduled_registry(task) task_model, scheduled_task_id = entry.args - self.assertEqual(task_model, task.TASK_TYPE) + self.assertEqual(task_model, task.task_type) self.assertEqual(scheduled_task_id, task.id) self.assertEqual("scheduled", entry.get_status()) assert_has_execution_with_status(task, "queued") @@ -410,7 +408,7 @@ def test_admin_enqueue_job_now(self): # assert 2 entry = _get_job_from_scheduled_registry(task) - self.assertEqual(task_model, task.TASK_TYPE) + self.assertEqual(task_model, task.task_type) self.assertEqual(scheduled_task_id, task.id) assert_has_execution_with_status(task, "finished") diff --git a/scheduler/tests/test_repeatable_task.py b/scheduler/tests/test_repeatable_task.py index c55c0c0..43507e4 100644 --- a/scheduler/tests/test_repeatable_task.py +++ b/scheduler/tests/test_repeatable_task.py @@ -6,7 +6,7 @@ from scheduler import settings from scheduler.models import RepeatableTask -from scheduler.tests.test_models import BaseTestCases +from scheduler.tests.test_old_models import BaseTestCases from .testtools import task_factory, _get_job_from_scheduled_registry diff --git a/scheduler/tests/test_task_model.py b/scheduler/tests/test_task_model.py new file mode 100644 index 0000000..a20e0ea --- /dev/null +++ b/scheduler/tests/test_task_model.py @@ -0,0 +1,551 @@ +import zoneinfo +from datetime import datetime, timedelta + +from django.contrib.messages import get_messages +from django.core.exceptions import ValidationError +from django.test import override_settings +from django.urls import reverse +from django.utils import timezone +from freezegun import freeze_time + +from scheduler import settings +from scheduler.models import BaseTask, TaskArg, TaskKwarg, ScheduledTask +from scheduler.tools import run_task, create_worker +from . import jobs +from .testtools import ( + task_factory, + taskarg_factory, + _get_job_from_scheduled_registry, + SchedulerBaseCase, + _get_executions, +) +from ..queues import get_queue + + +def assert_response_has_msg(response, message): + messages = [m.message for m in get_messages(response.wsgi_request)] + assert message in messages, f'expected "{message}" in {messages}' + + +def assert_has_execution_with_status(task, status): + job_list = _get_executions(task) + job_list = [(j.id, j.get_status()) for j in job_list] + for job in job_list: + if job[1] == status: + return + raise AssertionError(f"{task} does not have an execution with status {status}: {job_list}") + + +class BaseTestCases: + class TestBaseTask(SchedulerBaseCase): + TaskModelClass = BaseTask + + def test_callable_func(self): + task = task_factory(self.TaskModelClass) + task.callable = "scheduler.tests.jobs.test_job" + func = task.callable_func() + self.assertEqual(jobs.test_job, func) + + def test_callable_func_not_callable(self): + task = task_factory(self.TaskModelClass) + task.callable = "scheduler.tests.jobs.test_non_callable" + with self.assertRaises(TypeError): + task.callable_func() + + def test_clean_callable(self): + task = task_factory(self.TaskModelClass) + task.callable = "scheduler.tests.jobs.test_job" + self.assertIsNone(task.clean_callable()) + + def test_clean_callable_invalid(self): + task = task_factory(self.TaskModelClass) + task.callable = "scheduler.tests.jobs.test_non_callable" + with self.assertRaises(ValidationError): + task.clean_callable() + + def test_clean_queue(self): + for queue in settings.QUEUES.keys(): + task = task_factory(self.TaskModelClass) + task.queue = queue + self.assertIsNone(task.clean_queue()) + + def test_clean_queue_invalid(self): + task = task_factory(self.TaskModelClass) + task.queue = "xxxxxx" + task.callable = "scheduler.tests.jobs.test_job" + with self.assertRaises(ValidationError): + task.clean() + + # next 2 check the above are included in job.clean() function + def test_clean_base(self): + task = task_factory(self.TaskModelClass) + task.queue = list(settings.QUEUES)[0] + task.callable = "scheduler.tests.jobs.test_job" + self.assertIsNone(task.clean()) + + def test_clean_invalid_callable(self): + task = task_factory(self.TaskModelClass) + task.queue = list(settings.QUEUES)[0] + task.callable = "scheduler.tests.jobs.test_non_callable" + with self.assertRaises(ValidationError): + task.clean() + + def test_clean_invalid_queue(self): + task = task_factory(self.TaskModelClass) + task.queue = "xxxxxx" + task.callable = "scheduler.tests.jobs.test_job" + with self.assertRaises(ValidationError): + task.clean() + + def test_is_schedulable_already_scheduled(self): + task = task_factory( + self.TaskModelClass, + ) + task.schedule() + self.assertTrue(task.is_scheduled()) + + def test_is_schedulable_disabled(self): + task = task_factory(self.TaskModelClass) + task.enabled = False + self.assertFalse(task.enabled) + + def test_schedule(self): + task = task_factory( + self.TaskModelClass, + ) + self.assertTrue(task.is_scheduled()) + self.assertIsNotNone(task.job_id) + + def test_unschedulable(self): + task = task_factory(self.TaskModelClass, enabled=False) + self.assertFalse(task.is_scheduled()) + self.assertIsNone(task.job_id) + + def test_unschedule(self): + task = task_factory( + self.TaskModelClass, + ) + self.assertTrue(task.unschedule()) + self.assertIsNone(task.job_id) + + def test_unschedule_not_scheduled(self): + task = task_factory(self.TaskModelClass, enabled=False) + self.assertTrue(task.unschedule()) + self.assertIsNone(task.job_id) + + def test_save_enabled(self): + task = task_factory( + self.TaskModelClass, + ) + self.assertIsNotNone(task.job_id) + + def test_save_disabled(self): + task = task_factory(self.TaskModelClass, enabled=False) + task.save() + self.assertIsNone(task.job_id) + + def test_save_and_schedule(self): + task = task_factory( + self.TaskModelClass, + ) + self.assertIsNotNone(task.job_id) + self.assertTrue(task.is_scheduled()) + + def test_schedule2(self): + task = task_factory(self.TaskModelClass) + task.queue = list(settings.QUEUES)[0] + task.enabled = False + task.scheduled_time = timezone.now() + timedelta(minutes=1) + self.assertFalse(task.schedule()) + + def test_delete_and_unschedule(self): + task = task_factory( + self.TaskModelClass, + ) + self.assertIsNotNone(task.job_id) + self.assertTrue(task.is_scheduled()) + task.delete() + self.assertFalse(task.is_scheduled()) + + def test_job_create(self): + prev_count = self.TaskModelClass.objects.count() + task_factory(self.TaskModelClass) + self.assertEqual(self.TaskModelClass.objects.count(), prev_count + 1) + + def test_str(self): + name = "test" + task = task_factory(self.TaskModelClass, name=name) + self.assertEqual(f"{self.TaskModelClass.__name__}[{name}={task.callable}()]", str(task)) + + def test_callable_passthrough(self): + task = task_factory(self.TaskModelClass) + entry = _get_job_from_scheduled_registry(task) + self.assertEqual(entry.func, run_task) + job_model, job_id = entry.args + self.assertEqual(job_model, self.TaskModelClass.__name__) + self.assertEqual(job_id, task.id) + + def test_timeout_passthrough(self): + task = task_factory(self.TaskModelClass, timeout=500) + entry = _get_job_from_scheduled_registry(task) + self.assertEqual(entry.timeout, 500) + + def test_at_front_passthrough(self): + task = task_factory(self.TaskModelClass, at_front=True) + queue = task.rqueue + jobs_to_schedule = queue.scheduled_job_registry.get_job_ids() + self.assertIn(task.job_id, jobs_to_schedule) + + def test_callable_result(self): + task = task_factory( + self.TaskModelClass, + ) + entry = _get_job_from_scheduled_registry(task) + self.assertEqual(entry.perform(), 2) + + def test_callable_empty_args_and_kwargs(self): + task = task_factory(self.TaskModelClass, callable="scheduler.tests.jobs.test_args_kwargs") + entry = _get_job_from_scheduled_registry(task) + self.assertEqual(entry.perform(), "test_args_kwargs()") + + def test_delete_args(self): + task = task_factory( + self.TaskModelClass, + ) + arg = taskarg_factory(TaskArg, val="one", content_object=task) + self.assertEqual(1, task.callable_args.count()) + arg.delete() + self.assertEqual(0, task.callable_args.count()) + + def test_delete_kwargs(self): + task = task_factory( + self.TaskModelClass, + ) + kwarg = taskarg_factory(TaskKwarg, key="key1", arg_type="str", val="one", content_object=task) + self.assertEqual(1, task.callable_kwargs.count()) + kwarg.delete() + self.assertEqual(0, task.callable_kwargs.count()) + + def test_parse_args(self): + task = task_factory( + self.TaskModelClass, + ) + date = timezone.now() + taskarg_factory(TaskArg, val="one", content_object=task) + taskarg_factory(TaskArg, arg_type="int", val=2, content_object=task) + taskarg_factory(TaskArg, arg_type="bool", val=True, content_object=task) + taskarg_factory(TaskArg, arg_type="bool", val=False, content_object=task) + taskarg_factory(TaskArg, arg_type="datetime", val=date, content_object=task) + self.assertEqual(task.parse_args(), ["one", 2, True, False, date]) + + def test_parse_kwargs(self): + job = task_factory( + self.TaskModelClass, + ) + date = timezone.now() + taskarg_factory(TaskKwarg, key="key1", arg_type="str", val="one", content_object=job) + taskarg_factory(TaskKwarg, key="key2", arg_type="int", val=2, content_object=job) + taskarg_factory(TaskKwarg, key="key3", arg_type="bool", val=True, content_object=job) + taskarg_factory(TaskKwarg, key="key4", arg_type="datetime", val=date, content_object=job) + kwargs = job.parse_kwargs() + self.assertEqual(kwargs, dict(key1="one", key2=2, key3=True, key4=date)) + + def test_callable_args_and_kwargs(self): + task = task_factory(self.TaskModelClass, callable="scheduler.tests.jobs.test_args_kwargs") + date = timezone.now() + taskarg_factory(TaskArg, arg_type="str", val="one", content_object=task) + taskarg_factory(TaskKwarg, key="key1", arg_type="int", val=2, content_object=task) + taskarg_factory(TaskKwarg, key="key2", arg_type="datetime", val=date, content_object=task) + taskarg_factory(TaskKwarg, key="key3", arg_type="bool", val=False, content_object=task) + task.save() + entry = _get_job_from_scheduled_registry(task) + self.assertEqual(entry.perform(), "test_args_kwargs('one', key1=2, key2={}, key3=False)".format(date)) + + def test_function_string(self): + task = task_factory( + self.TaskModelClass, + ) + date = timezone.now() + taskarg_factory(TaskArg, arg_type="str", val="one", content_object=task) + taskarg_factory(TaskArg, arg_type="int", val="1", content_object=task) + taskarg_factory(TaskArg, arg_type="datetime", val=date, content_object=task) + taskarg_factory(TaskArg, arg_type="bool", val=True, content_object=task) + taskarg_factory(TaskKwarg, key="key1", arg_type="str", val="one", content_object=task) + taskarg_factory(TaskKwarg, key="key2", arg_type="int", val=2, content_object=task) + taskarg_factory(TaskKwarg, key="key3", arg_type="datetime", val=date, content_object=task) + taskarg_factory(TaskKwarg, key="key4", arg_type="bool", val=False, content_object=task) + self.assertEqual( + task.function_string(), + f"scheduler.tests.jobs.test_job('one', 1, {repr(date)}, True, " + f"key1='one', key2=2, key3={repr(date)}, key4=False)", + ) + + def test_admin_list_view(self): + # arrange + self.client.login(username="admin", password="admin") + job = task_factory( + self.TaskModelClass, + ) + model = job._meta.model.__name__.lower() + url = reverse(f"admin:scheduler_{model}_changelist") + # act + res = self.client.get(url) + # assert + self.assertEqual(200, res.status_code) + + def test_admin_list_view_delete_model(self): + # arrange + self.client.login(username="admin", password="admin") + task = task_factory( + self.TaskModelClass, + ) + model = task._meta.model.__name__.lower() + url = reverse(f"admin:scheduler_{model}_changelist") + # act + res = self.client.post( + url, + data={ + "action": "delete_model", + "_selected_action": [ + task.pk, + ], + }, + ) + # assert + self.assertEqual(302, res.status_code) + + def test_admin_run_job_now_enqueues_job_at(self): + # arrange + self.client.login(username="admin", password="admin") + task = task_factory( + self.TaskModelClass, + ) + model = task._meta.model.__name__.lower() + url = reverse(f"admin:scheduler_{model}_changelist") + # act + res = self.client.post( + url, + data={ + "action": "enqueue_job_now", + "_selected_action": [ + task.pk, + ], + }, + ) + # assert + self.assertEqual(302, res.status_code) + task.refresh_from_db() + queue = get_queue(task.queue) + self.assertIn(task.job_id, queue.get_job_ids()) + + def test_admin_change_view(self): + # arrange + self.client.login(username="admin", password="admin") + task = task_factory( + self.TaskModelClass, + ) + model = task._meta.model.__name__.lower() + url = reverse( + f"admin:scheduler_{model}_change", + args=[ + task.pk, + ], + ) + # act + res = self.client.get(url) + # assert + self.assertEqual(200, res.status_code) + + def test_admin_change_view__bad_redis_connection(self): + # arrange + self.client.login(username="admin", password="admin") + task = task_factory(self.TaskModelClass, queue="test2", instance_only=True) + task.save(schedule_job=False) + model = task._meta.model.__name__.lower() + url = reverse( + f"admin:scheduler_{model}_change", + args=[ + task.pk, + ], + ) + # act + res = self.client.get(url) + # assert + self.assertEqual(200, res.status_code) + + def test_admin_enqueue_job_now(self): + # arrange + self.client.login(username="admin", password="admin") + task = task_factory(self.TaskModelClass) + self.assertIsNotNone(task.job_id) + self.assertTrue(task.is_scheduled()) + data = { + "action": "enqueue_job_now", + "_selected_action": [ + task.id, + ], + } + model = task._meta.model.__name__.lower() + url = reverse(f"admin:scheduler_{model}_changelist") + # act + res = self.client.post(url, data=data, follow=True) + + # assert part 1 + self.assertEqual(200, res.status_code) + entry = _get_job_from_scheduled_registry(task) + task_model, scheduled_task_id = entry.args + self.assertEqual(task_model, task.task_type) + self.assertEqual(scheduled_task_id, task.id) + self.assertEqual("scheduled", entry.get_status()) + assert_has_execution_with_status(task, "queued") + + # act 2 + worker = create_worker( + "default", + fork_job_execution=False, + ) + worker.work(burst=True) + + # assert 2 + entry = _get_job_from_scheduled_registry(task) + self.assertEqual(task_model, task.task_type) + self.assertEqual(scheduled_task_id, task.id) + assert_has_execution_with_status(task, "finished") + + def test_admin_enable_job(self): + # arrange + self.client.login(username="admin", password="admin") + task = task_factory(self.TaskModelClass, enabled=False) + self.assertIsNone(task.job_id) + self.assertFalse(task.is_scheduled()) + data = { + "action": "enable_selected", + "_selected_action": [ + task.id, + ], + } + model = task._meta.model.__name__.lower() + url = reverse(f"admin:scheduler_{model}_changelist") + # act + res = self.client.post(url, data=data, follow=True) + # assert + self.assertEqual(200, res.status_code) + task.refresh_from_db() + self.assertTrue(task.enabled) + self.assertTrue(task.is_scheduled()) + assert_response_has_msg(res, "1 job was successfully enabled and scheduled.") + + def test_admin_disable_job(self): + # arrange + self.client.login(username="admin", password="admin") + task = task_factory(self.TaskModelClass, enabled=True) + task.save() + data = { + "action": "disable_selected", + "_selected_action": [ + task.id, + ], + } + model = task._meta.model.__name__.lower() + url = reverse(f"admin:scheduler_{model}_changelist") + self.assertTrue(task.is_scheduled()) + # act + res = self.client.post(url, data=data, follow=True) + # assert + self.assertEqual(200, res.status_code) + task.refresh_from_db() + self.assertFalse(task.is_scheduled()) + self.assertFalse(task.enabled) + assert_response_has_msg(res, "1 job was successfully disabled and unscheduled.") + + def test_admin_single_delete(self): + # arrange + self.client.login(username="admin", password="admin") + prev_count = self.TaskModelClass.objects.count() + task = task_factory( + self.TaskModelClass, + ) + self.assertIsNotNone(task.job_id) + self.assertTrue(task.is_scheduled()) + prev = len(_get_executions(task)) + model = task._meta.model.__name__.lower() + url = reverse( + f"admin:scheduler_{model}_delete", + args=[ + task.pk, + ], + ) + data = { + "post": "yes", + } + # act + res = self.client.post(url, data=data, follow=True) + # assert + self.assertEqual(200, res.status_code) + self.assertEqual(prev_count, self.TaskModelClass.objects.count()) + self.assertEqual(prev - 1, len(_get_executions(task))) + + def test_admin_delete_selected(self): + # arrange + self.client.login(username="admin", password="admin") + task = task_factory(self.TaskModelClass, enabled=True) + task.save() + queue = get_queue(task.queue) + scheduled_jobs = queue.scheduled_job_registry.get_job_ids() + job_id = task.job_id + self.assertIn(job_id, scheduled_jobs) + data = { + "action": "delete_selected", + "_selected_action": [ + task.id, + ], + "post": "yes", + } + model = task._meta.model.__name__.lower() + url = reverse(f"admin:scheduler_{model}_changelist") + # act + res = self.client.post(url, data=data, follow=True) + # assert + self.assertEqual(200, res.status_code) + assert_response_has_msg(res, f"Successfully deleted 1 {self.TaskModelClass._meta.verbose_name}.") + self.assertIsNone(self.TaskModelClass.objects.filter(id=task.id).first()) + scheduled_jobs = queue.scheduled_job_registry.get_job_ids() + self.assertNotIn(job_id, scheduled_jobs) + + class TestSchedulableJob(TestBaseTask): + # Currently ScheduledJob and RepeatableJob + TaskModelClass = ScheduledTask + + @freeze_time("2016-12-25") + @override_settings(USE_TZ=False) + def test_schedule_time_no_tz(self): + task = task_factory(self.TaskModelClass) + task.scheduled_time = datetime(2016, 12, 25, 8, 0, 0, tzinfo=None) + self.assertEqual("2016-12-25T08:00:00", task._schedule_time().isoformat()) + + @freeze_time("2016-12-25") + @override_settings(USE_TZ=True) + def test_schedule_time_with_tz(self): + task = task_factory(self.TaskModelClass) + est = zoneinfo.ZoneInfo("US/Eastern") + task.scheduled_time = datetime(2016, 12, 25, 8, 0, 0, tzinfo=est) + self.assertEqual("2016-12-25T13:00:00+00:00", task._schedule_time().isoformat()) + + def test_result_ttl_passthrough(self): + job = task_factory(self.TaskModelClass, result_ttl=500) + entry = _get_job_from_scheduled_registry(job) + self.assertEqual(entry.result_ttl, 500) + + +class TestScheduledJob(BaseTestCases.TestSchedulableJob): + TaskModelClass = ScheduledTask + + def test_clean(self): + job = task_factory(self.TaskModelClass) + job.queue = list(settings.QUEUES)[0] + job.callable = "scheduler.tests.jobs.test_job" + self.assertIsNone(job.clean()) + + def test_unschedulable_old_job(self): + job = task_factory(self.TaskModelClass, scheduled_time=timezone.now() - timedelta(hours=1)) + self.assertFalse(job.is_scheduled()) diff --git a/scheduler/tests/testtools.py b/scheduler/tests/testtools.py index 561327f..0238a25 100644 --- a/scheduler/tests/testtools.py +++ b/scheduler/tests/testtools.py @@ -113,7 +113,7 @@ def setUp(self) -> None: queue.empty() def tearDown(self) -> None: - super(SchedulerBaseCase, self).setUp() + super(SchedulerBaseCase, self).tearDown() queue = get_queue("default") queue.empty() diff --git a/scheduler/tools.py b/scheduler/tools.py index 7aa615c..cac5316 100644 --- a/scheduler/tools.py +++ b/scheduler/tools.py @@ -33,7 +33,7 @@ def get_next_cron_time(cron_string: Optional[str]) -> Optional[timezone.datetime def get_scheduled_task(task_model: str, task_id: int) -> "BaseTask": # noqa: F821 if task_model not in MODEL_NAMES: - raise ValueError(f"Job Model {task_model} does not exist, choices are {MODEL_NAMES}") + raise ValueError(f"Job Model `{task_model}` does not exist, choices are {MODEL_NAMES}") model = apps.get_model(app_label="scheduler", model_name=task_model) task = model.objects.filter(id=task_id).first() if task is None: From 174c84d70d573d20108700c7d098c0bf6b63cba5 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Fri, 11 Oct 2024 09:01:25 -0400 Subject: [PATCH 07/47] wip --- scheduler/migrations/0019_task.py | 166 ++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 scheduler/migrations/0019_task.py diff --git a/scheduler/migrations/0019_task.py b/scheduler/migrations/0019_task.py new file mode 100644 index 0000000..71b7618 --- /dev/null +++ b/scheduler/migrations/0019_task.py @@ -0,0 +1,166 @@ +# Generated by Django 5.1.1 on 2024-10-08 14:41 + +import scheduler.models.task +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("scheduler", "0018_alter_crontask_queue_alter_repeatabletask_queue_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="Task", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "name", + models.CharField(help_text="Name of the job", max_length=128, unique=True, verbose_name="name"), + ), + ( + "task_type", + models.CharField( + choices=[ + ("CronTask", "Cron Task"), + ("RepeatableTask", "Repeatable Task"), + ("OnceTask", "Run once"), + ], + default="OnceTask", + max_length=32, + verbose_name="Task type", + ), + ), + ("callable", models.CharField(max_length=2048, verbose_name="callable")), + ( + "enabled", + models.BooleanField( + default=True, + help_text="Should job be scheduled? This field is useful to keep past jobs that should no longer be scheduled", + verbose_name="enabled", + ), + ), + ( + "queue", + models.CharField( + choices=scheduler.models.task.get_queue_choices, + help_text="Queue name", + max_length=255, + verbose_name="queue", + ), + ), + ( + "job_id", + models.CharField( + blank=True, + editable=False, + help_text="Current job_id on queue", + max_length=128, + null=True, + verbose_name="job id", + ), + ), + ( + "at_front", + models.BooleanField( + blank=True, + default=False, + help_text="When queuing the job, add it in the front of the queue", + null=True, + verbose_name="At front", + ), + ), + ( + "timeout", + models.IntegerField( + blank=True, + help_text="Timeout specifies the maximum runtime, in seconds, for the job before it'll be considered 'lost'. Blank uses the default timeout.", + null=True, + verbose_name="timeout", + ), + ), + ( + "result_ttl", + models.IntegerField( + blank=True, + help_text="The TTL value (in seconds) of the job result.
\n -1: Result never expires, you should delete jobs manually.
\n 0: Result gets deleted immediately.
\n >0: Result expires after n seconds.", + null=True, + verbose_name="result ttl", + ), + ), + ( + "failed_runs", + models.PositiveIntegerField( + default=0, help_text="Number of times the task has failed", verbose_name="failed runs" + ), + ), + ( + "successful_runs", + models.PositiveIntegerField( + default=0, help_text="Number of times the task has succeeded", verbose_name="successful runs" + ), + ), + ( + "last_successful_run", + models.DateTimeField( + blank=True, + help_text="Last time the task has succeeded", + null=True, + verbose_name="last successful run", + ), + ), + ( + "last_failed_run", + models.DateTimeField( + blank=True, help_text="Last time the task has failed", null=True, verbose_name="last failed run" + ), + ), + ( + "interval", + models.PositiveIntegerField( + blank=True, help_text="Interval for repeatable task", null=True, verbose_name="interval" + ), + ), + ( + "interval_unit", + models.CharField( + blank=True, + choices=[ + ("seconds", "seconds"), + ("minutes", "minutes"), + ("hours", "hours"), + ("days", "days"), + ("weeks", "weeks"), + ], + default="hours", + max_length=12, + null=True, + verbose_name="interval unit", + ), + ), + ( + "repeat", + models.PositiveIntegerField( + blank=True, + help_text="Number of times to run the job. Leaving this blank means it will run forever.", + null=True, + verbose_name="repeat", + ), + ), + ("scheduled_time", models.DateTimeField(verbose_name="scheduled time")), + ( + "cron_string", + models.CharField( + blank=True, + help_text='Define the schedule in a crontab like syntax.\n Times are in UTC. Use crontab.guru to create a cron string.', + max_length=64, + null=True, + verbose_name="cron string", + ), + ), + ], + ), + ] From 2cb65435074c4a79e516c12ad0a5113b521a7298 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Fri, 11 Oct 2024 09:44:24 -0400 Subject: [PATCH 08/47] wip --- scheduler/management/commands/import.py | 2 ++ scheduler/models/scheduled_task.py | 1 + scheduler/models/task.py | 1 + 3 files changed, 4 insertions(+) diff --git a/scheduler/management/commands/import.py b/scheduler/management/commands/import.py index 2e574cb..d1f0fce 100644 --- a/scheduler/management/commands/import.py +++ b/scheduler/management/commands/import.py @@ -18,6 +18,7 @@ def job_model_str(model_str: str) -> str: return model_str[:-3] + "Task" return model_str + def get_task_type(model_str: str) -> TaskType: model_str = job_model_str(model_str) if model_str not in MODEL_NAMES: @@ -29,6 +30,7 @@ def get_task_type(model_str: str) -> TaskType: elif model_str == "ScheduledTask": return TaskType.ONCE + def create_task_from_dict(task_dict: Dict[str, Any], update): existing_job = Task.objects.filter(name=task_dict["name"]).first() task_type = get_task_type(task_dict["model"]) diff --git a/scheduler/models/scheduled_task.py b/scheduler/models/scheduled_task.py index 7b8c658..a150f06 100644 --- a/scheduler/models/scheduled_task.py +++ b/scheduler/models/scheduled_task.py @@ -406,6 +406,7 @@ class Meta: class RepeatableTask(RepeatableMixin, ScheduledTimeMixin, BaseTask): task_type = "RepeatableTask" + class TimeUnits(models.TextChoices): SECONDS = "seconds", _("seconds") MINUTES = "minutes", _("minutes") diff --git a/scheduler/models/task.py b/scheduler/models/task.py index e70b916..148d8f9 100644 --- a/scheduler/models/task.py +++ b/scheduler/models/task.py @@ -69,6 +69,7 @@ class TaskType(models.TextChoices): REPEATABLE = "RepeatableTask", _("Repeatable Task") ONCE = "OnceTask", _("Run once") + class Task(models.Model): class TimeUnits(models.TextChoices): SECONDS = "seconds", _("seconds") From 8d2bc63f0eef5a2c80cb6d8f8b72e55383a641ad Mon Sep 17 00:00:00 2001 From: Daniel M Date: Fri, 11 Oct 2024 11:29:47 -0400 Subject: [PATCH 09/47] wip --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a9cca6d..980054a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ classifiers = [ 'Framework :: Django :: 5.1', ] homepage = "https://github.com/django-commons/django-tasks-scheduler" -documentation = "https://django-tasks-scheduler.readthedocs.io/en/latest/" +documentation = "https://django-tasks-scheduler.readthedocs.io/" [tool.poetry.urls] "Bug Tracker" = "https://github.com/django-commons/django-tasks-scheduler/issues" From cf85c2f873eed7807e33a6c24540523d79752230 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 15 Oct 2024 10:53:43 -0400 Subject: [PATCH 10/47] fix:tests --- scheduler/tests/test_mgmt_cmds.py | 39 ++++++++++++++++--------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/scheduler/tests/test_mgmt_cmds.py b/scheduler/tests/test_mgmt_cmds.py index 46b66f3..b1ba047 100644 --- a/scheduler/tests/test_mgmt_cmds.py +++ b/scheduler/tests/test_mgmt_cmds.py @@ -233,26 +233,28 @@ def test_import__should_schedule_job(self): call_command("import", filename=self.tmpfile.name) # assert self.assertEqual(1, Task.objects.filter(task_type=TaskType.ONCE).count()) + self.assertEqual(1, Task.objects.filter(task_type=TaskType.REPEATABLE).count()) db_job = Task.objects.filter(task_type=TaskType.ONCE).first() attrs = ["name", "queue", "callable", "enabled", "timeout"] for attr in attrs: self.assertEqual(getattr(jobs[0], attr), getattr(db_job, attr)) def test_import__should_schedule_job_yaml(self): - jobs = list() - jobs.append(task_factory(ScheduledTask, enabled=True, instance_only=True)) - jobs.append(task_factory(RepeatableTask, enabled=True, instance_only=True)) - res = yaml.dump([j.to_dict() for j in jobs], default_flow_style=False) + tasks = list() + tasks.append(task_factory(ScheduledTask, enabled=True, instance_only=True)) + tasks.append(task_factory(RepeatableTask, enabled=True, instance_only=True)) + res = yaml.dump([j.to_dict() for j in tasks], default_flow_style=False) self.tmpfile.write(res) self.tmpfile.flush() # act call_command("import", filename=self.tmpfile.name, format="yaml") # assert self.assertEqual(1, Task.objects.filter(task_type=TaskType.ONCE).count()) - db_job = Task.objects.filter(task_type=TaskType.ONCE).objects.first() + self.assertEqual(1, Task.objects.filter(task_type=TaskType.REPEATABLE).count()) + db_job = Task.objects.filter(task_type=TaskType.ONCE).first() attrs = ["name", "queue", "callable", "enabled", "timeout"] for attr in attrs: - self.assertEqual(getattr(jobs[0], attr), getattr(db_job, attr)) + self.assertEqual(getattr(tasks[0], attr), getattr(db_job, attr)) def test_import__should_schedule_job_yaml_without_yaml_lib(self): jobs = list() @@ -295,10 +297,10 @@ def test_import__should_schedule_job_reset(self): self.assertEqual(getattr(jobs[1], attr), getattr(db_job, attr)) def test_import__should_schedule_job_update_existing(self): - jobs = list() - task_factory(ScheduledTask, enabled=True) - jobs.append(task_factory(ScheduledTask, enabled=True)) - res = json.dumps([j.to_dict() for j in jobs]) + tasks = list() + tasks.append(task_factory(ScheduledTask, enabled=True)) + tasks.append(task_factory(ScheduledTask, enabled=True)) + res = json.dumps([j.to_dict() for j in tasks]) self.tmpfile.write(res) self.tmpfile.flush() # act @@ -309,17 +311,16 @@ def test_import__should_schedule_job_update_existing(self): ) # assert self.assertEqual(2, Task.objects.filter(task_type=TaskType.ONCE).count()) - db_job = Task.objects.filter(task_type=TaskType.ONCE).get(name=jobs[0].name) - self.assertNotEqual(jobs[0].id, db_job.id) + db_job = Task.objects.filter(task_type=TaskType.ONCE).get(name=tasks[0].name) attrs = ["name", "queue", "callable", "enabled", "timeout"] for attr in attrs: - self.assertEqual(getattr(jobs[0], attr), getattr(db_job, attr)) + self.assertEqual(getattr(tasks[0], attr), getattr(db_job, attr)) def test_import__should_schedule_job_without_update_existing(self): - jobs = list() - task_factory(ScheduledTask, enabled=True) - jobs.append(task_factory(ScheduledTask, enabled=True)) - res = json.dumps([j.to_dict() for j in jobs]) + tasks = list() + tasks.append(task_factory(ScheduledTask, enabled=True)) + tasks.append(task_factory(ScheduledTask, enabled=True)) + res = json.dumps([j.to_dict() for j in tasks]) self.tmpfile.write(res) self.tmpfile.flush() # act @@ -329,7 +330,7 @@ def test_import__should_schedule_job_without_update_existing(self): ) # assert self.assertEqual(2, Task.objects.filter(task_type=TaskType.ONCE).count()) - db_job = Task.objects.get(name=jobs[0].name) + db_job = Task.objects.get(name=tasks[0].name) attrs = ["id", "name", "queue", "callable", "enabled", "timeout"] for attr in attrs: - self.assertEqual(getattr(jobs[0], attr), getattr(db_job, attr)) + self.assertEqual(getattr(tasks[0], attr), getattr(db_job, attr)) From e8c8764d7a2a826c10a247701f3273a75df64ad0 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 15 Oct 2024 10:57:11 -0400 Subject: [PATCH 11/47] wip --- scheduler/migrations/0019_task.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scheduler/migrations/0019_task.py b/scheduler/migrations/0019_task.py index 71b7618..f8a892e 100644 --- a/scheduler/migrations/0019_task.py +++ b/scheduler/migrations/0019_task.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.1 on 2024-10-08 14:41 +# Generated by Django 5.1.2 on 2024-10-15 14:56 import scheduler.models.task from django.db import migrations, models @@ -66,10 +66,8 @@ class Migration(migrations.Migration): ( "at_front", models.BooleanField( - blank=True, default=False, help_text="When queuing the job, add it in the front of the queue", - null=True, verbose_name="At front", ), ), From 32ac983ab1539c9dd7a83978f38ca01b845448dd Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 15 Oct 2024 12:40:07 -0400 Subject: [PATCH 12/47] wip --- scheduler/admin/task_admin.py | 21 +++++++++++++---- scheduler/static/admin/js/select-fields.js | 27 ++++++++++++++++++++++ 2 files changed, 43 insertions(+), 5 deletions(-) create mode 100644 scheduler/static/admin/js/select-fields.js diff --git a/scheduler/admin/task_admin.py b/scheduler/admin/task_admin.py index 95221d3..2971c47 100644 --- a/scheduler/admin/task_admin.py +++ b/scheduler/admin/task_admin.py @@ -12,9 +12,7 @@ class HiddenMixin(object): class Media: - js = [ - "admin/js/jquery.init.js", - ] + js = ("admin/js/jquery.init.js",) class JobArgInline(HiddenMixin, GenericStackedInline): @@ -33,6 +31,9 @@ class JobKwargInline(HiddenMixin, GenericStackedInline): class TaskAdmin(admin.ModelAdmin): """TaskAdmin admin view for all task models.""" + class Media: + js = ("admin/js/jquery.init.js", "admin/js/select-fields.js",) + save_on_top = True change_form_template = "admin/scheduler/change_form.html" actions = [ @@ -69,7 +70,7 @@ class TaskAdmin(admin.ModelAdmin): "failed_runs", "last_failed_run", ) - radio_fields = {"task_type": admin.HORIZONTAL} + # radio_fields = {"task_type": admin.HORIZONTAL} fieldsets = ( ( None, @@ -79,10 +80,20 @@ class TaskAdmin(admin.ModelAdmin): "callable", "task_type", ("enabled", "timeout", "result_ttl"), - ("scheduled_time", "cron_string", "interval", "interval_unit", "repeat"), ) ), ), + ( + None, + dict(fields=("scheduled_time",), classes=("tasktype-OnceTask",)), + ), + ( + None, + dict(fields=("cron_string",), classes=("tasktype-CronTask",)), + ), ( + None, + dict(fields=("interval", "interval_unit", "repeat"), classes=("tasktype-RepeatableTask",)), + ), (_("RQ Settings"), dict(fields=(("queue", "at_front"), "job_id"))), ( _("Previous runs info"), diff --git a/scheduler/static/admin/js/select-fields.js b/scheduler/static/admin/js/select-fields.js new file mode 100644 index 0000000..6122549 --- /dev/null +++ b/scheduler/static/admin/js/select-fields.js @@ -0,0 +1,27 @@ +(function ($) { + $(function () { + const tasktypes = { + "CronTask": $(".tasktype-CronTask"), + "RepeatableTask": $(".tasktype-RepeatableTask"), + "OnceTask": $(".tasktype-OnceTask"), + }; + var taskTypeField = $('#id_task_type'); + + function toggleVerified(value) { + console.log(value); + for (const [k, v] of Object.entries(tasktypes)) { + if (k === value) { + v.show(); + } else { + v.hide(); + } + } + } + + toggleVerified(taskTypeField.val()); + + taskTypeField.change(function () { + toggleVerified($(this).val()); + }); + }); +})(django.jQuery); \ No newline at end of file From 54bdf644da88ced358ff3bd1a057fc25e643068d Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 15 Oct 2024 18:46:24 -0400 Subject: [PATCH 13/47] wip --- scheduler/admin/task_admin.py | 8 +- scheduler/migrations/0019_task.py | 4 +- scheduler/models/task.py | 9 +- scheduler/tests/test_cron_task.py | 12 +- scheduler/tests/test_internals.py | 6 +- scheduler/tests/test_old_models.py | 98 ++--- scheduler/tests/test_old_task_model.py | 551 +++++++++++++++++++++++++ scheduler/tests/test_task_model.py | 147 +++---- scheduler/tests/testtools.py | 47 ++- scheduler/tools.py | 20 +- 10 files changed, 742 insertions(+), 160 deletions(-) create mode 100644 scheduler/tests/test_old_task_model.py diff --git a/scheduler/admin/task_admin.py b/scheduler/admin/task_admin.py index 2971c47..199017c 100644 --- a/scheduler/admin/task_admin.py +++ b/scheduler/admin/task_admin.py @@ -32,7 +32,10 @@ class TaskAdmin(admin.ModelAdmin): """TaskAdmin admin view for all task models.""" class Media: - js = ("admin/js/jquery.init.js", "admin/js/select-fields.js",) + js = ( + "admin/js/jquery.init.js", + "admin/js/select-fields.js", + ) save_on_top = True change_form_template = "admin/scheduler/change_form.html" @@ -90,7 +93,8 @@ class Media: ( None, dict(fields=("cron_string",), classes=("tasktype-CronTask",)), - ), ( + ), + ( None, dict(fields=("interval", "interval_unit", "repeat"), classes=("tasktype-RepeatableTask",)), ), diff --git a/scheduler/migrations/0019_task.py b/scheduler/migrations/0019_task.py index f8a892e..a328c7c 100644 --- a/scheduler/migrations/0019_task.py +++ b/scheduler/migrations/0019_task.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.2 on 2024-10-15 14:56 +# Generated by Django 5.1.2 on 2024-10-15 22:39 import scheduler.models.task from django.db import migrations, models @@ -148,7 +148,7 @@ class Migration(migrations.Migration): verbose_name="repeat", ), ), - ("scheduled_time", models.DateTimeField(verbose_name="scheduled time")), + ("scheduled_time", models.DateTimeField(blank=True, null=True, verbose_name="scheduled time")), ( "cron_string", models.CharField( diff --git a/scheduler/models/task.py b/scheduler/models/task.py index 148d8f9..8c40ed6 100644 --- a/scheduler/models/task.py +++ b/scheduler/models/task.py @@ -24,6 +24,7 @@ from scheduler.rq_classes import DjangoQueue from scheduler.settings import QUEUES from scheduler.settings import logger +from scheduler.tools import TaskType SCHEDULER_INTERVAL = settings.SCHEDULER_CONFIG.SCHEDULER_INTERVAL @@ -64,12 +65,6 @@ def get_queue_choices(): return [(queue, queue) for queue in QUEUES.keys()] -class TaskType(models.TextChoices): - CRON = "CronTask", _("Cron Task") - REPEATABLE = "RepeatableTask", _("Repeatable Task") - ONCE = "OnceTask", _("Run once") - - class Task(models.Model): class TimeUnits(models.TextChoices): SECONDS = "seconds", _("seconds") @@ -164,7 +159,7 @@ class TimeUnits(models.TextChoices): null=True, help_text=_("Number of times to run the job. Leaving this blank means it will run forever."), ) - scheduled_time = models.DateTimeField(_("scheduled time")) + scheduled_time = models.DateTimeField(_("scheduled time"), blank=True, null=True) cron_string = models.CharField( _("cron string"), max_length=64, diff --git a/scheduler/tests/test_cron_task.py b/scheduler/tests/test_cron_task.py index 7d3bd4c..81df24d 100644 --- a/scheduler/tests/test_cron_task.py +++ b/scheduler/tests/test_cron_task.py @@ -4,7 +4,7 @@ from scheduler.models import CronTask from scheduler.tools import create_worker from .test_old_models import BaseTestCases -from .testtools import task_factory +from .testtools import old_task_factory from ..queues import get_queue @@ -12,14 +12,14 @@ class TestCronTask(BaseTestCases.TestBaseTask): TaskModelClass = CronTask def test_clean(self): - task = task_factory(CronTask) + task = old_task_factory(CronTask) task.cron_string = "* * * * *" task.queue = list(settings.QUEUES)[0] task.callable = "scheduler.tests.jobs.test_job" self.assertIsNone(task.clean()) def test_clean_cron_string_invalid(self): - task = task_factory(CronTask) + task = old_task_factory(CronTask) task.cron_string = "not-a-cron-string" task.queue = list(settings.QUEUES)[0] task.callable = "scheduler.tests.jobs.test_job" @@ -27,7 +27,7 @@ def test_clean_cron_string_invalid(self): task.clean_cron_string() def test_check_rescheduled_after_execution(self): - task = task_factory( + task = old_task_factory( CronTask, ) queue = task.rqueue @@ -43,7 +43,7 @@ def test_check_rescheduled_after_execution(self): self.assertNotEqual(task.job_id, first_run_id) def test_check_rescheduled_after_failed_execution(self): - task = task_factory( + task = old_task_factory( CronTask, callable_name="scheduler.tests.jobs.scheduler.tests.jobs.test_job", ) @@ -63,7 +63,7 @@ def test_cron_task_enqueuing_jobs(self): queue = get_queue() prev_queued = len(queue.scheduled_job_registry) prev_finished = len(queue.finished_job_registry) - task = task_factory(CronTask, callable_name="scheduler.tests.jobs.enqueue_jobs") + task = old_task_factory(CronTask, callable_name="scheduler.tests.jobs.enqueue_jobs") self.assertEqual(prev_queued + 1, len(queue.scheduled_job_registry)) first_run_id = task.job_id entry = queue.fetch_job(first_run_id) diff --git a/scheduler/tests/test_internals.py b/scheduler/tests/test_internals.py index f3759c3..f916a48 100644 --- a/scheduler/tests/test_internals.py +++ b/scheduler/tests/test_internals.py @@ -2,15 +2,15 @@ from django.utils import timezone -from scheduler.models import ScheduledTask +from scheduler.models.task import TaskType from scheduler.tests.testtools import SchedulerBaseCase, task_factory from scheduler.tools import get_scheduled_task class TestInternals(SchedulerBaseCase): def test_get_scheduled_job(self): - task = task_factory(ScheduledTask, scheduled_time=timezone.now() - timedelta(hours=1)) - self.assertEqual(task, get_scheduled_task(task.task_type, task.id)) + task = task_factory(TaskType.ONCE, scheduled_time=timezone.now() - timedelta(hours=1)) + self.assertEqual(task, get_scheduled_task(TaskType.ONCE, task.id)) with self.assertRaises(ValueError): get_scheduled_task(task.task_type, task.id + 1) with self.assertRaises(ValueError): diff --git a/scheduler/tests/test_old_models.py b/scheduler/tests/test_old_models.py index a20e0ea..8e04b45 100644 --- a/scheduler/tests/test_old_models.py +++ b/scheduler/tests/test_old_models.py @@ -13,7 +13,7 @@ from scheduler.tools import run_task, create_worker from . import jobs from .testtools import ( - task_factory, + old_task_factory, taskarg_factory, _get_job_from_scheduled_registry, SchedulerBaseCase, @@ -41,36 +41,36 @@ class TestBaseTask(SchedulerBaseCase): TaskModelClass = BaseTask def test_callable_func(self): - task = task_factory(self.TaskModelClass) + task = old_task_factory(self.TaskModelClass) task.callable = "scheduler.tests.jobs.test_job" func = task.callable_func() self.assertEqual(jobs.test_job, func) def test_callable_func_not_callable(self): - task = task_factory(self.TaskModelClass) + task = old_task_factory(self.TaskModelClass) task.callable = "scheduler.tests.jobs.test_non_callable" with self.assertRaises(TypeError): task.callable_func() def test_clean_callable(self): - task = task_factory(self.TaskModelClass) + task = old_task_factory(self.TaskModelClass) task.callable = "scheduler.tests.jobs.test_job" self.assertIsNone(task.clean_callable()) def test_clean_callable_invalid(self): - task = task_factory(self.TaskModelClass) + task = old_task_factory(self.TaskModelClass) task.callable = "scheduler.tests.jobs.test_non_callable" with self.assertRaises(ValidationError): task.clean_callable() def test_clean_queue(self): for queue in settings.QUEUES.keys(): - task = task_factory(self.TaskModelClass) + task = old_task_factory(self.TaskModelClass) task.queue = queue self.assertIsNone(task.clean_queue()) def test_clean_queue_invalid(self): - task = task_factory(self.TaskModelClass) + task = old_task_factory(self.TaskModelClass) task.queue = "xxxxxx" task.callable = "scheduler.tests.jobs.test_job" with self.assertRaises(ValidationError): @@ -78,88 +78,88 @@ def test_clean_queue_invalid(self): # next 2 check the above are included in job.clean() function def test_clean_base(self): - task = task_factory(self.TaskModelClass) + task = old_task_factory(self.TaskModelClass) task.queue = list(settings.QUEUES)[0] task.callable = "scheduler.tests.jobs.test_job" self.assertIsNone(task.clean()) def test_clean_invalid_callable(self): - task = task_factory(self.TaskModelClass) + task = old_task_factory(self.TaskModelClass) task.queue = list(settings.QUEUES)[0] task.callable = "scheduler.tests.jobs.test_non_callable" with self.assertRaises(ValidationError): task.clean() def test_clean_invalid_queue(self): - task = task_factory(self.TaskModelClass) + task = old_task_factory(self.TaskModelClass) task.queue = "xxxxxx" task.callable = "scheduler.tests.jobs.test_job" with self.assertRaises(ValidationError): task.clean() def test_is_schedulable_already_scheduled(self): - task = task_factory( + task = old_task_factory( self.TaskModelClass, ) task.schedule() self.assertTrue(task.is_scheduled()) def test_is_schedulable_disabled(self): - task = task_factory(self.TaskModelClass) + task = old_task_factory(self.TaskModelClass) task.enabled = False self.assertFalse(task.enabled) def test_schedule(self): - task = task_factory( + task = old_task_factory( self.TaskModelClass, ) self.assertTrue(task.is_scheduled()) self.assertIsNotNone(task.job_id) def test_unschedulable(self): - task = task_factory(self.TaskModelClass, enabled=False) + task = old_task_factory(self.TaskModelClass, enabled=False) self.assertFalse(task.is_scheduled()) self.assertIsNone(task.job_id) def test_unschedule(self): - task = task_factory( + task = old_task_factory( self.TaskModelClass, ) self.assertTrue(task.unschedule()) self.assertIsNone(task.job_id) def test_unschedule_not_scheduled(self): - task = task_factory(self.TaskModelClass, enabled=False) + task = old_task_factory(self.TaskModelClass, enabled=False) self.assertTrue(task.unschedule()) self.assertIsNone(task.job_id) def test_save_enabled(self): - task = task_factory( + task = old_task_factory( self.TaskModelClass, ) self.assertIsNotNone(task.job_id) def test_save_disabled(self): - task = task_factory(self.TaskModelClass, enabled=False) + task = old_task_factory(self.TaskModelClass, enabled=False) task.save() self.assertIsNone(task.job_id) def test_save_and_schedule(self): - task = task_factory( + task = old_task_factory( self.TaskModelClass, ) self.assertIsNotNone(task.job_id) self.assertTrue(task.is_scheduled()) def test_schedule2(self): - task = task_factory(self.TaskModelClass) + task = old_task_factory(self.TaskModelClass) task.queue = list(settings.QUEUES)[0] task.enabled = False task.scheduled_time = timezone.now() + timedelta(minutes=1) self.assertFalse(task.schedule()) def test_delete_and_unschedule(self): - task = task_factory( + task = old_task_factory( self.TaskModelClass, ) self.assertIsNotNone(task.job_id) @@ -169,16 +169,16 @@ def test_delete_and_unschedule(self): def test_job_create(self): prev_count = self.TaskModelClass.objects.count() - task_factory(self.TaskModelClass) + old_task_factory(self.TaskModelClass) self.assertEqual(self.TaskModelClass.objects.count(), prev_count + 1) def test_str(self): name = "test" - task = task_factory(self.TaskModelClass, name=name) + task = old_task_factory(self.TaskModelClass, name=name) self.assertEqual(f"{self.TaskModelClass.__name__}[{name}={task.callable}()]", str(task)) def test_callable_passthrough(self): - task = task_factory(self.TaskModelClass) + task = old_task_factory(self.TaskModelClass) entry = _get_job_from_scheduled_registry(task) self.assertEqual(entry.func, run_task) job_model, job_id = entry.args @@ -186,30 +186,30 @@ def test_callable_passthrough(self): self.assertEqual(job_id, task.id) def test_timeout_passthrough(self): - task = task_factory(self.TaskModelClass, timeout=500) + task = old_task_factory(self.TaskModelClass, timeout=500) entry = _get_job_from_scheduled_registry(task) self.assertEqual(entry.timeout, 500) def test_at_front_passthrough(self): - task = task_factory(self.TaskModelClass, at_front=True) + task = old_task_factory(self.TaskModelClass, at_front=True) queue = task.rqueue jobs_to_schedule = queue.scheduled_job_registry.get_job_ids() self.assertIn(task.job_id, jobs_to_schedule) def test_callable_result(self): - task = task_factory( + task = old_task_factory( self.TaskModelClass, ) entry = _get_job_from_scheduled_registry(task) self.assertEqual(entry.perform(), 2) def test_callable_empty_args_and_kwargs(self): - task = task_factory(self.TaskModelClass, callable="scheduler.tests.jobs.test_args_kwargs") + task = old_task_factory(self.TaskModelClass, callable="scheduler.tests.jobs.test_args_kwargs") entry = _get_job_from_scheduled_registry(task) self.assertEqual(entry.perform(), "test_args_kwargs()") def test_delete_args(self): - task = task_factory( + task = old_task_factory( self.TaskModelClass, ) arg = taskarg_factory(TaskArg, val="one", content_object=task) @@ -218,7 +218,7 @@ def test_delete_args(self): self.assertEqual(0, task.callable_args.count()) def test_delete_kwargs(self): - task = task_factory( + task = old_task_factory( self.TaskModelClass, ) kwarg = taskarg_factory(TaskKwarg, key="key1", arg_type="str", val="one", content_object=task) @@ -227,7 +227,7 @@ def test_delete_kwargs(self): self.assertEqual(0, task.callable_kwargs.count()) def test_parse_args(self): - task = task_factory( + task = old_task_factory( self.TaskModelClass, ) date = timezone.now() @@ -239,7 +239,7 @@ def test_parse_args(self): self.assertEqual(task.parse_args(), ["one", 2, True, False, date]) def test_parse_kwargs(self): - job = task_factory( + job = old_task_factory( self.TaskModelClass, ) date = timezone.now() @@ -251,7 +251,7 @@ def test_parse_kwargs(self): self.assertEqual(kwargs, dict(key1="one", key2=2, key3=True, key4=date)) def test_callable_args_and_kwargs(self): - task = task_factory(self.TaskModelClass, callable="scheduler.tests.jobs.test_args_kwargs") + task = old_task_factory(self.TaskModelClass, callable="scheduler.tests.jobs.test_args_kwargs") date = timezone.now() taskarg_factory(TaskArg, arg_type="str", val="one", content_object=task) taskarg_factory(TaskKwarg, key="key1", arg_type="int", val=2, content_object=task) @@ -262,7 +262,7 @@ def test_callable_args_and_kwargs(self): self.assertEqual(entry.perform(), "test_args_kwargs('one', key1=2, key2={}, key3=False)".format(date)) def test_function_string(self): - task = task_factory( + task = old_task_factory( self.TaskModelClass, ) date = timezone.now() @@ -283,7 +283,7 @@ def test_function_string(self): def test_admin_list_view(self): # arrange self.client.login(username="admin", password="admin") - job = task_factory( + job = old_task_factory( self.TaskModelClass, ) model = job._meta.model.__name__.lower() @@ -296,7 +296,7 @@ def test_admin_list_view(self): def test_admin_list_view_delete_model(self): # arrange self.client.login(username="admin", password="admin") - task = task_factory( + task = old_task_factory( self.TaskModelClass, ) model = task._meta.model.__name__.lower() @@ -317,7 +317,7 @@ def test_admin_list_view_delete_model(self): def test_admin_run_job_now_enqueues_job_at(self): # arrange self.client.login(username="admin", password="admin") - task = task_factory( + task = old_task_factory( self.TaskModelClass, ) model = task._meta.model.__name__.lower() @@ -341,7 +341,7 @@ def test_admin_run_job_now_enqueues_job_at(self): def test_admin_change_view(self): # arrange self.client.login(username="admin", password="admin") - task = task_factory( + task = old_task_factory( self.TaskModelClass, ) model = task._meta.model.__name__.lower() @@ -359,7 +359,7 @@ def test_admin_change_view(self): def test_admin_change_view__bad_redis_connection(self): # arrange self.client.login(username="admin", password="admin") - task = task_factory(self.TaskModelClass, queue="test2", instance_only=True) + task = old_task_factory(self.TaskModelClass, queue="test2", instance_only=True) task.save(schedule_job=False) model = task._meta.model.__name__.lower() url = reverse( @@ -376,7 +376,7 @@ def test_admin_change_view__bad_redis_connection(self): def test_admin_enqueue_job_now(self): # arrange self.client.login(username="admin", password="admin") - task = task_factory(self.TaskModelClass) + task = old_task_factory(self.TaskModelClass) self.assertIsNotNone(task.job_id) self.assertTrue(task.is_scheduled()) data = { @@ -415,7 +415,7 @@ def test_admin_enqueue_job_now(self): def test_admin_enable_job(self): # arrange self.client.login(username="admin", password="admin") - task = task_factory(self.TaskModelClass, enabled=False) + task = old_task_factory(self.TaskModelClass, enabled=False) self.assertIsNone(task.job_id) self.assertFalse(task.is_scheduled()) data = { @@ -438,7 +438,7 @@ def test_admin_enable_job(self): def test_admin_disable_job(self): # arrange self.client.login(username="admin", password="admin") - task = task_factory(self.TaskModelClass, enabled=True) + task = old_task_factory(self.TaskModelClass, enabled=True) task.save() data = { "action": "disable_selected", @@ -462,7 +462,7 @@ def test_admin_single_delete(self): # arrange self.client.login(username="admin", password="admin") prev_count = self.TaskModelClass.objects.count() - task = task_factory( + task = old_task_factory( self.TaskModelClass, ) self.assertIsNotNone(task.job_id) @@ -488,7 +488,7 @@ def test_admin_single_delete(self): def test_admin_delete_selected(self): # arrange self.client.login(username="admin", password="admin") - task = task_factory(self.TaskModelClass, enabled=True) + task = old_task_factory(self.TaskModelClass, enabled=True) task.save() queue = get_queue(task.queue) scheduled_jobs = queue.scheduled_job_registry.get_job_ids() @@ -519,20 +519,20 @@ class TestSchedulableJob(TestBaseTask): @freeze_time("2016-12-25") @override_settings(USE_TZ=False) def test_schedule_time_no_tz(self): - task = task_factory(self.TaskModelClass) + task = old_task_factory(self.TaskModelClass) task.scheduled_time = datetime(2016, 12, 25, 8, 0, 0, tzinfo=None) self.assertEqual("2016-12-25T08:00:00", task._schedule_time().isoformat()) @freeze_time("2016-12-25") @override_settings(USE_TZ=True) def test_schedule_time_with_tz(self): - task = task_factory(self.TaskModelClass) + task = old_task_factory(self.TaskModelClass) est = zoneinfo.ZoneInfo("US/Eastern") task.scheduled_time = datetime(2016, 12, 25, 8, 0, 0, tzinfo=est) self.assertEqual("2016-12-25T13:00:00+00:00", task._schedule_time().isoformat()) def test_result_ttl_passthrough(self): - job = task_factory(self.TaskModelClass, result_ttl=500) + job = old_task_factory(self.TaskModelClass, result_ttl=500) entry = _get_job_from_scheduled_registry(job) self.assertEqual(entry.result_ttl, 500) @@ -541,11 +541,11 @@ class TestScheduledJob(BaseTestCases.TestSchedulableJob): TaskModelClass = ScheduledTask def test_clean(self): - job = task_factory(self.TaskModelClass) + job = old_task_factory(self.TaskModelClass) job.queue = list(settings.QUEUES)[0] job.callable = "scheduler.tests.jobs.test_job" self.assertIsNone(job.clean()) def test_unschedulable_old_job(self): - job = task_factory(self.TaskModelClass, scheduled_time=timezone.now() - timedelta(hours=1)) + job = old_task_factory(self.TaskModelClass, scheduled_time=timezone.now() - timedelta(hours=1)) self.assertFalse(job.is_scheduled()) diff --git a/scheduler/tests/test_old_task_model.py b/scheduler/tests/test_old_task_model.py new file mode 100644 index 0000000..8e04b45 --- /dev/null +++ b/scheduler/tests/test_old_task_model.py @@ -0,0 +1,551 @@ +import zoneinfo +from datetime import datetime, timedelta + +from django.contrib.messages import get_messages +from django.core.exceptions import ValidationError +from django.test import override_settings +from django.urls import reverse +from django.utils import timezone +from freezegun import freeze_time + +from scheduler import settings +from scheduler.models import BaseTask, TaskArg, TaskKwarg, ScheduledTask +from scheduler.tools import run_task, create_worker +from . import jobs +from .testtools import ( + old_task_factory, + taskarg_factory, + _get_job_from_scheduled_registry, + SchedulerBaseCase, + _get_executions, +) +from ..queues import get_queue + + +def assert_response_has_msg(response, message): + messages = [m.message for m in get_messages(response.wsgi_request)] + assert message in messages, f'expected "{message}" in {messages}' + + +def assert_has_execution_with_status(task, status): + job_list = _get_executions(task) + job_list = [(j.id, j.get_status()) for j in job_list] + for job in job_list: + if job[1] == status: + return + raise AssertionError(f"{task} does not have an execution with status {status}: {job_list}") + + +class BaseTestCases: + class TestBaseTask(SchedulerBaseCase): + TaskModelClass = BaseTask + + def test_callable_func(self): + task = old_task_factory(self.TaskModelClass) + task.callable = "scheduler.tests.jobs.test_job" + func = task.callable_func() + self.assertEqual(jobs.test_job, func) + + def test_callable_func_not_callable(self): + task = old_task_factory(self.TaskModelClass) + task.callable = "scheduler.tests.jobs.test_non_callable" + with self.assertRaises(TypeError): + task.callable_func() + + def test_clean_callable(self): + task = old_task_factory(self.TaskModelClass) + task.callable = "scheduler.tests.jobs.test_job" + self.assertIsNone(task.clean_callable()) + + def test_clean_callable_invalid(self): + task = old_task_factory(self.TaskModelClass) + task.callable = "scheduler.tests.jobs.test_non_callable" + with self.assertRaises(ValidationError): + task.clean_callable() + + def test_clean_queue(self): + for queue in settings.QUEUES.keys(): + task = old_task_factory(self.TaskModelClass) + task.queue = queue + self.assertIsNone(task.clean_queue()) + + def test_clean_queue_invalid(self): + task = old_task_factory(self.TaskModelClass) + task.queue = "xxxxxx" + task.callable = "scheduler.tests.jobs.test_job" + with self.assertRaises(ValidationError): + task.clean() + + # next 2 check the above are included in job.clean() function + def test_clean_base(self): + task = old_task_factory(self.TaskModelClass) + task.queue = list(settings.QUEUES)[0] + task.callable = "scheduler.tests.jobs.test_job" + self.assertIsNone(task.clean()) + + def test_clean_invalid_callable(self): + task = old_task_factory(self.TaskModelClass) + task.queue = list(settings.QUEUES)[0] + task.callable = "scheduler.tests.jobs.test_non_callable" + with self.assertRaises(ValidationError): + task.clean() + + def test_clean_invalid_queue(self): + task = old_task_factory(self.TaskModelClass) + task.queue = "xxxxxx" + task.callable = "scheduler.tests.jobs.test_job" + with self.assertRaises(ValidationError): + task.clean() + + def test_is_schedulable_already_scheduled(self): + task = old_task_factory( + self.TaskModelClass, + ) + task.schedule() + self.assertTrue(task.is_scheduled()) + + def test_is_schedulable_disabled(self): + task = old_task_factory(self.TaskModelClass) + task.enabled = False + self.assertFalse(task.enabled) + + def test_schedule(self): + task = old_task_factory( + self.TaskModelClass, + ) + self.assertTrue(task.is_scheduled()) + self.assertIsNotNone(task.job_id) + + def test_unschedulable(self): + task = old_task_factory(self.TaskModelClass, enabled=False) + self.assertFalse(task.is_scheduled()) + self.assertIsNone(task.job_id) + + def test_unschedule(self): + task = old_task_factory( + self.TaskModelClass, + ) + self.assertTrue(task.unschedule()) + self.assertIsNone(task.job_id) + + def test_unschedule_not_scheduled(self): + task = old_task_factory(self.TaskModelClass, enabled=False) + self.assertTrue(task.unschedule()) + self.assertIsNone(task.job_id) + + def test_save_enabled(self): + task = old_task_factory( + self.TaskModelClass, + ) + self.assertIsNotNone(task.job_id) + + def test_save_disabled(self): + task = old_task_factory(self.TaskModelClass, enabled=False) + task.save() + self.assertIsNone(task.job_id) + + def test_save_and_schedule(self): + task = old_task_factory( + self.TaskModelClass, + ) + self.assertIsNotNone(task.job_id) + self.assertTrue(task.is_scheduled()) + + def test_schedule2(self): + task = old_task_factory(self.TaskModelClass) + task.queue = list(settings.QUEUES)[0] + task.enabled = False + task.scheduled_time = timezone.now() + timedelta(minutes=1) + self.assertFalse(task.schedule()) + + def test_delete_and_unschedule(self): + task = old_task_factory( + self.TaskModelClass, + ) + self.assertIsNotNone(task.job_id) + self.assertTrue(task.is_scheduled()) + task.delete() + self.assertFalse(task.is_scheduled()) + + def test_job_create(self): + prev_count = self.TaskModelClass.objects.count() + old_task_factory(self.TaskModelClass) + self.assertEqual(self.TaskModelClass.objects.count(), prev_count + 1) + + def test_str(self): + name = "test" + task = old_task_factory(self.TaskModelClass, name=name) + self.assertEqual(f"{self.TaskModelClass.__name__}[{name}={task.callable}()]", str(task)) + + def test_callable_passthrough(self): + task = old_task_factory(self.TaskModelClass) + entry = _get_job_from_scheduled_registry(task) + self.assertEqual(entry.func, run_task) + job_model, job_id = entry.args + self.assertEqual(job_model, self.TaskModelClass.__name__) + self.assertEqual(job_id, task.id) + + def test_timeout_passthrough(self): + task = old_task_factory(self.TaskModelClass, timeout=500) + entry = _get_job_from_scheduled_registry(task) + self.assertEqual(entry.timeout, 500) + + def test_at_front_passthrough(self): + task = old_task_factory(self.TaskModelClass, at_front=True) + queue = task.rqueue + jobs_to_schedule = queue.scheduled_job_registry.get_job_ids() + self.assertIn(task.job_id, jobs_to_schedule) + + def test_callable_result(self): + task = old_task_factory( + self.TaskModelClass, + ) + entry = _get_job_from_scheduled_registry(task) + self.assertEqual(entry.perform(), 2) + + def test_callable_empty_args_and_kwargs(self): + task = old_task_factory(self.TaskModelClass, callable="scheduler.tests.jobs.test_args_kwargs") + entry = _get_job_from_scheduled_registry(task) + self.assertEqual(entry.perform(), "test_args_kwargs()") + + def test_delete_args(self): + task = old_task_factory( + self.TaskModelClass, + ) + arg = taskarg_factory(TaskArg, val="one", content_object=task) + self.assertEqual(1, task.callable_args.count()) + arg.delete() + self.assertEqual(0, task.callable_args.count()) + + def test_delete_kwargs(self): + task = old_task_factory( + self.TaskModelClass, + ) + kwarg = taskarg_factory(TaskKwarg, key="key1", arg_type="str", val="one", content_object=task) + self.assertEqual(1, task.callable_kwargs.count()) + kwarg.delete() + self.assertEqual(0, task.callable_kwargs.count()) + + def test_parse_args(self): + task = old_task_factory( + self.TaskModelClass, + ) + date = timezone.now() + taskarg_factory(TaskArg, val="one", content_object=task) + taskarg_factory(TaskArg, arg_type="int", val=2, content_object=task) + taskarg_factory(TaskArg, arg_type="bool", val=True, content_object=task) + taskarg_factory(TaskArg, arg_type="bool", val=False, content_object=task) + taskarg_factory(TaskArg, arg_type="datetime", val=date, content_object=task) + self.assertEqual(task.parse_args(), ["one", 2, True, False, date]) + + def test_parse_kwargs(self): + job = old_task_factory( + self.TaskModelClass, + ) + date = timezone.now() + taskarg_factory(TaskKwarg, key="key1", arg_type="str", val="one", content_object=job) + taskarg_factory(TaskKwarg, key="key2", arg_type="int", val=2, content_object=job) + taskarg_factory(TaskKwarg, key="key3", arg_type="bool", val=True, content_object=job) + taskarg_factory(TaskKwarg, key="key4", arg_type="datetime", val=date, content_object=job) + kwargs = job.parse_kwargs() + self.assertEqual(kwargs, dict(key1="one", key2=2, key3=True, key4=date)) + + def test_callable_args_and_kwargs(self): + task = old_task_factory(self.TaskModelClass, callable="scheduler.tests.jobs.test_args_kwargs") + date = timezone.now() + taskarg_factory(TaskArg, arg_type="str", val="one", content_object=task) + taskarg_factory(TaskKwarg, key="key1", arg_type="int", val=2, content_object=task) + taskarg_factory(TaskKwarg, key="key2", arg_type="datetime", val=date, content_object=task) + taskarg_factory(TaskKwarg, key="key3", arg_type="bool", val=False, content_object=task) + task.save() + entry = _get_job_from_scheduled_registry(task) + self.assertEqual(entry.perform(), "test_args_kwargs('one', key1=2, key2={}, key3=False)".format(date)) + + def test_function_string(self): + task = old_task_factory( + self.TaskModelClass, + ) + date = timezone.now() + taskarg_factory(TaskArg, arg_type="str", val="one", content_object=task) + taskarg_factory(TaskArg, arg_type="int", val="1", content_object=task) + taskarg_factory(TaskArg, arg_type="datetime", val=date, content_object=task) + taskarg_factory(TaskArg, arg_type="bool", val=True, content_object=task) + taskarg_factory(TaskKwarg, key="key1", arg_type="str", val="one", content_object=task) + taskarg_factory(TaskKwarg, key="key2", arg_type="int", val=2, content_object=task) + taskarg_factory(TaskKwarg, key="key3", arg_type="datetime", val=date, content_object=task) + taskarg_factory(TaskKwarg, key="key4", arg_type="bool", val=False, content_object=task) + self.assertEqual( + task.function_string(), + f"scheduler.tests.jobs.test_job('one', 1, {repr(date)}, True, " + f"key1='one', key2=2, key3={repr(date)}, key4=False)", + ) + + def test_admin_list_view(self): + # arrange + self.client.login(username="admin", password="admin") + job = old_task_factory( + self.TaskModelClass, + ) + model = job._meta.model.__name__.lower() + url = reverse(f"admin:scheduler_{model}_changelist") + # act + res = self.client.get(url) + # assert + self.assertEqual(200, res.status_code) + + def test_admin_list_view_delete_model(self): + # arrange + self.client.login(username="admin", password="admin") + task = old_task_factory( + self.TaskModelClass, + ) + model = task._meta.model.__name__.lower() + url = reverse(f"admin:scheduler_{model}_changelist") + # act + res = self.client.post( + url, + data={ + "action": "delete_model", + "_selected_action": [ + task.pk, + ], + }, + ) + # assert + self.assertEqual(302, res.status_code) + + def test_admin_run_job_now_enqueues_job_at(self): + # arrange + self.client.login(username="admin", password="admin") + task = old_task_factory( + self.TaskModelClass, + ) + model = task._meta.model.__name__.lower() + url = reverse(f"admin:scheduler_{model}_changelist") + # act + res = self.client.post( + url, + data={ + "action": "enqueue_job_now", + "_selected_action": [ + task.pk, + ], + }, + ) + # assert + self.assertEqual(302, res.status_code) + task.refresh_from_db() + queue = get_queue(task.queue) + self.assertIn(task.job_id, queue.get_job_ids()) + + def test_admin_change_view(self): + # arrange + self.client.login(username="admin", password="admin") + task = old_task_factory( + self.TaskModelClass, + ) + model = task._meta.model.__name__.lower() + url = reverse( + f"admin:scheduler_{model}_change", + args=[ + task.pk, + ], + ) + # act + res = self.client.get(url) + # assert + self.assertEqual(200, res.status_code) + + def test_admin_change_view__bad_redis_connection(self): + # arrange + self.client.login(username="admin", password="admin") + task = old_task_factory(self.TaskModelClass, queue="test2", instance_only=True) + task.save(schedule_job=False) + model = task._meta.model.__name__.lower() + url = reverse( + f"admin:scheduler_{model}_change", + args=[ + task.pk, + ], + ) + # act + res = self.client.get(url) + # assert + self.assertEqual(200, res.status_code) + + def test_admin_enqueue_job_now(self): + # arrange + self.client.login(username="admin", password="admin") + task = old_task_factory(self.TaskModelClass) + self.assertIsNotNone(task.job_id) + self.assertTrue(task.is_scheduled()) + data = { + "action": "enqueue_job_now", + "_selected_action": [ + task.id, + ], + } + model = task._meta.model.__name__.lower() + url = reverse(f"admin:scheduler_{model}_changelist") + # act + res = self.client.post(url, data=data, follow=True) + + # assert part 1 + self.assertEqual(200, res.status_code) + entry = _get_job_from_scheduled_registry(task) + task_model, scheduled_task_id = entry.args + self.assertEqual(task_model, task.task_type) + self.assertEqual(scheduled_task_id, task.id) + self.assertEqual("scheduled", entry.get_status()) + assert_has_execution_with_status(task, "queued") + + # act 2 + worker = create_worker( + "default", + fork_job_execution=False, + ) + worker.work(burst=True) + + # assert 2 + entry = _get_job_from_scheduled_registry(task) + self.assertEqual(task_model, task.task_type) + self.assertEqual(scheduled_task_id, task.id) + assert_has_execution_with_status(task, "finished") + + def test_admin_enable_job(self): + # arrange + self.client.login(username="admin", password="admin") + task = old_task_factory(self.TaskModelClass, enabled=False) + self.assertIsNone(task.job_id) + self.assertFalse(task.is_scheduled()) + data = { + "action": "enable_selected", + "_selected_action": [ + task.id, + ], + } + model = task._meta.model.__name__.lower() + url = reverse(f"admin:scheduler_{model}_changelist") + # act + res = self.client.post(url, data=data, follow=True) + # assert + self.assertEqual(200, res.status_code) + task.refresh_from_db() + self.assertTrue(task.enabled) + self.assertTrue(task.is_scheduled()) + assert_response_has_msg(res, "1 job was successfully enabled and scheduled.") + + def test_admin_disable_job(self): + # arrange + self.client.login(username="admin", password="admin") + task = old_task_factory(self.TaskModelClass, enabled=True) + task.save() + data = { + "action": "disable_selected", + "_selected_action": [ + task.id, + ], + } + model = task._meta.model.__name__.lower() + url = reverse(f"admin:scheduler_{model}_changelist") + self.assertTrue(task.is_scheduled()) + # act + res = self.client.post(url, data=data, follow=True) + # assert + self.assertEqual(200, res.status_code) + task.refresh_from_db() + self.assertFalse(task.is_scheduled()) + self.assertFalse(task.enabled) + assert_response_has_msg(res, "1 job was successfully disabled and unscheduled.") + + def test_admin_single_delete(self): + # arrange + self.client.login(username="admin", password="admin") + prev_count = self.TaskModelClass.objects.count() + task = old_task_factory( + self.TaskModelClass, + ) + self.assertIsNotNone(task.job_id) + self.assertTrue(task.is_scheduled()) + prev = len(_get_executions(task)) + model = task._meta.model.__name__.lower() + url = reverse( + f"admin:scheduler_{model}_delete", + args=[ + task.pk, + ], + ) + data = { + "post": "yes", + } + # act + res = self.client.post(url, data=data, follow=True) + # assert + self.assertEqual(200, res.status_code) + self.assertEqual(prev_count, self.TaskModelClass.objects.count()) + self.assertEqual(prev - 1, len(_get_executions(task))) + + def test_admin_delete_selected(self): + # arrange + self.client.login(username="admin", password="admin") + task = old_task_factory(self.TaskModelClass, enabled=True) + task.save() + queue = get_queue(task.queue) + scheduled_jobs = queue.scheduled_job_registry.get_job_ids() + job_id = task.job_id + self.assertIn(job_id, scheduled_jobs) + data = { + "action": "delete_selected", + "_selected_action": [ + task.id, + ], + "post": "yes", + } + model = task._meta.model.__name__.lower() + url = reverse(f"admin:scheduler_{model}_changelist") + # act + res = self.client.post(url, data=data, follow=True) + # assert + self.assertEqual(200, res.status_code) + assert_response_has_msg(res, f"Successfully deleted 1 {self.TaskModelClass._meta.verbose_name}.") + self.assertIsNone(self.TaskModelClass.objects.filter(id=task.id).first()) + scheduled_jobs = queue.scheduled_job_registry.get_job_ids() + self.assertNotIn(job_id, scheduled_jobs) + + class TestSchedulableJob(TestBaseTask): + # Currently ScheduledJob and RepeatableJob + TaskModelClass = ScheduledTask + + @freeze_time("2016-12-25") + @override_settings(USE_TZ=False) + def test_schedule_time_no_tz(self): + task = old_task_factory(self.TaskModelClass) + task.scheduled_time = datetime(2016, 12, 25, 8, 0, 0, tzinfo=None) + self.assertEqual("2016-12-25T08:00:00", task._schedule_time().isoformat()) + + @freeze_time("2016-12-25") + @override_settings(USE_TZ=True) + def test_schedule_time_with_tz(self): + task = old_task_factory(self.TaskModelClass) + est = zoneinfo.ZoneInfo("US/Eastern") + task.scheduled_time = datetime(2016, 12, 25, 8, 0, 0, tzinfo=est) + self.assertEqual("2016-12-25T13:00:00+00:00", task._schedule_time().isoformat()) + + def test_result_ttl_passthrough(self): + job = old_task_factory(self.TaskModelClass, result_ttl=500) + entry = _get_job_from_scheduled_registry(job) + self.assertEqual(entry.result_ttl, 500) + + +class TestScheduledJob(BaseTestCases.TestSchedulableJob): + TaskModelClass = ScheduledTask + + def test_clean(self): + job = old_task_factory(self.TaskModelClass) + job.queue = list(settings.QUEUES)[0] + job.callable = "scheduler.tests.jobs.test_job" + self.assertIsNone(job.clean()) + + def test_unschedulable_old_job(self): + job = old_task_factory(self.TaskModelClass, scheduled_time=timezone.now() - timedelta(hours=1)) + self.assertFalse(job.is_scheduled()) diff --git a/scheduler/tests/test_task_model.py b/scheduler/tests/test_task_model.py index a20e0ea..def647d 100644 --- a/scheduler/tests/test_task_model.py +++ b/scheduler/tests/test_task_model.py @@ -9,7 +9,7 @@ from freezegun import freeze_time from scheduler import settings -from scheduler.models import BaseTask, TaskArg, TaskKwarg, ScheduledTask +from scheduler.models import Task, TaskArg, TaskKwarg from scheduler.tools import run_task, create_worker from . import jobs from .testtools import ( @@ -19,6 +19,7 @@ SchedulerBaseCase, _get_executions, ) +from ..models.task import TaskType from ..queues import get_queue @@ -38,39 +39,39 @@ def assert_has_execution_with_status(task, status): class BaseTestCases: class TestBaseTask(SchedulerBaseCase): - TaskModelClass = BaseTask + task_type = None def test_callable_func(self): - task = task_factory(self.TaskModelClass) + task = task_factory(self.task_type) task.callable = "scheduler.tests.jobs.test_job" func = task.callable_func() self.assertEqual(jobs.test_job, func) def test_callable_func_not_callable(self): - task = task_factory(self.TaskModelClass) + task = task_factory(self.task_type) task.callable = "scheduler.tests.jobs.test_non_callable" with self.assertRaises(TypeError): task.callable_func() def test_clean_callable(self): - task = task_factory(self.TaskModelClass) + task = task_factory(self.task_type) task.callable = "scheduler.tests.jobs.test_job" self.assertIsNone(task.clean_callable()) def test_clean_callable_invalid(self): - task = task_factory(self.TaskModelClass) + task = task_factory(self.task_type) task.callable = "scheduler.tests.jobs.test_non_callable" with self.assertRaises(ValidationError): task.clean_callable() def test_clean_queue(self): for queue in settings.QUEUES.keys(): - task = task_factory(self.TaskModelClass) + task = task_factory(self.task_type) task.queue = queue self.assertIsNone(task.clean_queue()) def test_clean_queue_invalid(self): - task = task_factory(self.TaskModelClass) + task = task_factory(self.task_type) task.queue = "xxxxxx" task.callable = "scheduler.tests.jobs.test_job" with self.assertRaises(ValidationError): @@ -78,158 +79,140 @@ def test_clean_queue_invalid(self): # next 2 check the above are included in job.clean() function def test_clean_base(self): - task = task_factory(self.TaskModelClass) + task = task_factory(self.task_type) task.queue = list(settings.QUEUES)[0] task.callable = "scheduler.tests.jobs.test_job" self.assertIsNone(task.clean()) def test_clean_invalid_callable(self): - task = task_factory(self.TaskModelClass) + task = task_factory(self.task_type) task.queue = list(settings.QUEUES)[0] task.callable = "scheduler.tests.jobs.test_non_callable" with self.assertRaises(ValidationError): task.clean() def test_clean_invalid_queue(self): - task = task_factory(self.TaskModelClass) + task = task_factory(self.task_type) task.queue = "xxxxxx" task.callable = "scheduler.tests.jobs.test_job" with self.assertRaises(ValidationError): task.clean() def test_is_schedulable_already_scheduled(self): - task = task_factory( - self.TaskModelClass, - ) + task = task_factory(self.task_type) task.schedule() self.assertTrue(task.is_scheduled()) def test_is_schedulable_disabled(self): - task = task_factory(self.TaskModelClass) + task = task_factory(self.task_type) task.enabled = False self.assertFalse(task.enabled) def test_schedule(self): task = task_factory( - self.TaskModelClass, + self.task_type, ) self.assertTrue(task.is_scheduled()) self.assertIsNotNone(task.job_id) def test_unschedulable(self): - task = task_factory(self.TaskModelClass, enabled=False) + task = task_factory(self.task_type, enabled=False) self.assertFalse(task.is_scheduled()) self.assertIsNone(task.job_id) def test_unschedule(self): - task = task_factory( - self.TaskModelClass, - ) + task = task_factory(self.task_type) self.assertTrue(task.unschedule()) self.assertIsNone(task.job_id) def test_unschedule_not_scheduled(self): - task = task_factory(self.TaskModelClass, enabled=False) + task = task_factory(self.task_type, enabled=False) self.assertTrue(task.unschedule()) self.assertIsNone(task.job_id) def test_save_enabled(self): - task = task_factory( - self.TaskModelClass, - ) + task = task_factory(self.task_type) self.assertIsNotNone(task.job_id) def test_save_disabled(self): - task = task_factory(self.TaskModelClass, enabled=False) + task = task_factory(self.task_type, enabled=False) task.save() self.assertIsNone(task.job_id) def test_save_and_schedule(self): - task = task_factory( - self.TaskModelClass, - ) + task = task_factory(self.task_type) self.assertIsNotNone(task.job_id) self.assertTrue(task.is_scheduled()) def test_schedule2(self): - task = task_factory(self.TaskModelClass) + task = task_factory(self.task_type) task.queue = list(settings.QUEUES)[0] task.enabled = False task.scheduled_time = timezone.now() + timedelta(minutes=1) self.assertFalse(task.schedule()) def test_delete_and_unschedule(self): - task = task_factory( - self.TaskModelClass, - ) + task = task_factory(self.task_type) self.assertIsNotNone(task.job_id) self.assertTrue(task.is_scheduled()) task.delete() self.assertFalse(task.is_scheduled()) def test_job_create(self): - prev_count = self.TaskModelClass.objects.count() - task_factory(self.TaskModelClass) - self.assertEqual(self.TaskModelClass.objects.count(), prev_count + 1) + prev_count = Task.objects.filter(task_type=self.task_type).count() + task_factory(self.task_type) + self.assertEqual(Task.objects.filter(task_type=self.task_type).count(), prev_count + 1) def test_str(self): name = "test" - task = task_factory(self.TaskModelClass, name=name) - self.assertEqual(f"{self.TaskModelClass.__name__}[{name}={task.callable}()]", str(task)) + task = task_factory(self.task_type, name=name) + self.assertEqual(f"{self.task_type.value}[{name}={task.callable}()]", str(task)) def test_callable_passthrough(self): - task = task_factory(self.TaskModelClass) + task = task_factory(self.task_type) entry = _get_job_from_scheduled_registry(task) self.assertEqual(entry.func, run_task) job_model, job_id = entry.args - self.assertEqual(job_model, self.TaskModelClass.__name__) + self.assertEqual(job_model, self.task_type.value) self.assertEqual(job_id, task.id) def test_timeout_passthrough(self): - task = task_factory(self.TaskModelClass, timeout=500) + task = task_factory(self.task_type, timeout=500) entry = _get_job_from_scheduled_registry(task) self.assertEqual(entry.timeout, 500) def test_at_front_passthrough(self): - task = task_factory(self.TaskModelClass, at_front=True) + task = task_factory(self.task_type, at_front=True) queue = task.rqueue jobs_to_schedule = queue.scheduled_job_registry.get_job_ids() self.assertIn(task.job_id, jobs_to_schedule) def test_callable_result(self): - task = task_factory( - self.TaskModelClass, - ) + task = task_factory(self.task_type) entry = _get_job_from_scheduled_registry(task) self.assertEqual(entry.perform(), 2) def test_callable_empty_args_and_kwargs(self): - task = task_factory(self.TaskModelClass, callable="scheduler.tests.jobs.test_args_kwargs") + task = task_factory(self.task_type, callable="scheduler.tests.jobs.test_args_kwargs") entry = _get_job_from_scheduled_registry(task) self.assertEqual(entry.perform(), "test_args_kwargs()") def test_delete_args(self): - task = task_factory( - self.TaskModelClass, - ) + task = task_factory(self.task_type) arg = taskarg_factory(TaskArg, val="one", content_object=task) self.assertEqual(1, task.callable_args.count()) arg.delete() self.assertEqual(0, task.callable_args.count()) def test_delete_kwargs(self): - task = task_factory( - self.TaskModelClass, - ) + task = task_factory(self.task_type) kwarg = taskarg_factory(TaskKwarg, key="key1", arg_type="str", val="one", content_object=task) self.assertEqual(1, task.callable_kwargs.count()) kwarg.delete() self.assertEqual(0, task.callable_kwargs.count()) def test_parse_args(self): - task = task_factory( - self.TaskModelClass, - ) + task = task_factory(self.task_type) date = timezone.now() taskarg_factory(TaskArg, val="one", content_object=task) taskarg_factory(TaskArg, arg_type="int", val=2, content_object=task) @@ -239,9 +222,7 @@ def test_parse_args(self): self.assertEqual(task.parse_args(), ["one", 2, True, False, date]) def test_parse_kwargs(self): - job = task_factory( - self.TaskModelClass, - ) + job = task_factory(self.task_type) date = timezone.now() taskarg_factory(TaskKwarg, key="key1", arg_type="str", val="one", content_object=job) taskarg_factory(TaskKwarg, key="key2", arg_type="int", val=2, content_object=job) @@ -251,7 +232,7 @@ def test_parse_kwargs(self): self.assertEqual(kwargs, dict(key1="one", key2=2, key3=True, key4=date)) def test_callable_args_and_kwargs(self): - task = task_factory(self.TaskModelClass, callable="scheduler.tests.jobs.test_args_kwargs") + task = task_factory(self.task_type, callable="scheduler.tests.jobs.test_args_kwargs") date = timezone.now() taskarg_factory(TaskArg, arg_type="str", val="one", content_object=task) taskarg_factory(TaskKwarg, key="key1", arg_type="int", val=2, content_object=task) @@ -262,9 +243,7 @@ def test_callable_args_and_kwargs(self): self.assertEqual(entry.perform(), "test_args_kwargs('one', key1=2, key2={}, key3=False)".format(date)) def test_function_string(self): - task = task_factory( - self.TaskModelClass, - ) + task = task_factory(self.task_type) date = timezone.now() taskarg_factory(TaskArg, arg_type="str", val="one", content_object=task) taskarg_factory(TaskArg, arg_type="int", val="1", content_object=task) @@ -283,9 +262,7 @@ def test_function_string(self): def test_admin_list_view(self): # arrange self.client.login(username="admin", password="admin") - job = task_factory( - self.TaskModelClass, - ) + job = task_factory(self.task_type) model = job._meta.model.__name__.lower() url = reverse(f"admin:scheduler_{model}_changelist") # act @@ -297,7 +274,7 @@ def test_admin_list_view_delete_model(self): # arrange self.client.login(username="admin", password="admin") task = task_factory( - self.TaskModelClass, + self.task_type, ) model = task._meta.model.__name__.lower() url = reverse(f"admin:scheduler_{model}_changelist") @@ -317,9 +294,7 @@ def test_admin_list_view_delete_model(self): def test_admin_run_job_now_enqueues_job_at(self): # arrange self.client.login(username="admin", password="admin") - task = task_factory( - self.TaskModelClass, - ) + task = task_factory(self.task_type) model = task._meta.model.__name__.lower() url = reverse(f"admin:scheduler_{model}_changelist") # act @@ -342,7 +317,7 @@ def test_admin_change_view(self): # arrange self.client.login(username="admin", password="admin") task = task_factory( - self.TaskModelClass, + self.task_type, ) model = task._meta.model.__name__.lower() url = reverse( @@ -359,7 +334,7 @@ def test_admin_change_view(self): def test_admin_change_view__bad_redis_connection(self): # arrange self.client.login(username="admin", password="admin") - task = task_factory(self.TaskModelClass, queue="test2", instance_only=True) + task = task_factory(self.task_type, queue="test2", instance_only=True) task.save(schedule_job=False) model = task._meta.model.__name__.lower() url = reverse( @@ -376,7 +351,7 @@ def test_admin_change_view__bad_redis_connection(self): def test_admin_enqueue_job_now(self): # arrange self.client.login(username="admin", password="admin") - task = task_factory(self.TaskModelClass) + task = task_factory(self.task_type) self.assertIsNotNone(task.job_id) self.assertTrue(task.is_scheduled()) data = { @@ -415,7 +390,7 @@ def test_admin_enqueue_job_now(self): def test_admin_enable_job(self): # arrange self.client.login(username="admin", password="admin") - task = task_factory(self.TaskModelClass, enabled=False) + task = task_factory(self.task_type, enabled=False) self.assertIsNone(task.job_id) self.assertFalse(task.is_scheduled()) data = { @@ -438,7 +413,7 @@ def test_admin_enable_job(self): def test_admin_disable_job(self): # arrange self.client.login(username="admin", password="admin") - task = task_factory(self.TaskModelClass, enabled=True) + task = task_factory(self.task_type, enabled=True) task.save() data = { "action": "disable_selected", @@ -461,9 +436,9 @@ def test_admin_disable_job(self): def test_admin_single_delete(self): # arrange self.client.login(username="admin", password="admin") - prev_count = self.TaskModelClass.objects.count() + prev_count = Task.objects.filter(task_type=self.task_type).count() task = task_factory( - self.TaskModelClass, + self.task_type, ) self.assertIsNotNone(task.job_id) self.assertTrue(task.is_scheduled()) @@ -482,13 +457,13 @@ def test_admin_single_delete(self): res = self.client.post(url, data=data, follow=True) # assert self.assertEqual(200, res.status_code) - self.assertEqual(prev_count, self.TaskModelClass.objects.count()) + self.assertEqual(prev_count, Task.objects.filter(task_type=self.task_type).count()) self.assertEqual(prev - 1, len(_get_executions(task))) def test_admin_delete_selected(self): # arrange self.client.login(username="admin", password="admin") - task = task_factory(self.TaskModelClass, enabled=True) + task = task_factory(self.task_type, enabled=True) task.save() queue = get_queue(task.queue) scheduled_jobs = queue.scheduled_job_registry.get_job_ids() @@ -507,45 +482,45 @@ def test_admin_delete_selected(self): res = self.client.post(url, data=data, follow=True) # assert self.assertEqual(200, res.status_code) - assert_response_has_msg(res, f"Successfully deleted 1 {self.TaskModelClass._meta.verbose_name}.") - self.assertIsNone(self.TaskModelClass.objects.filter(id=task.id).first()) + assert_response_has_msg(res, f"Successfully deleted 1 task.") + self.assertIsNone(Task.objects.filter(task_type=self.task_type).filter(id=task.id).first()) scheduled_jobs = queue.scheduled_job_registry.get_job_ids() self.assertNotIn(job_id, scheduled_jobs) class TestSchedulableJob(TestBaseTask): # Currently ScheduledJob and RepeatableJob - TaskModelClass = ScheduledTask + task_type = TaskType.ONCE @freeze_time("2016-12-25") @override_settings(USE_TZ=False) def test_schedule_time_no_tz(self): - task = task_factory(self.TaskModelClass) + task = task_factory(self.task_type) task.scheduled_time = datetime(2016, 12, 25, 8, 0, 0, tzinfo=None) self.assertEqual("2016-12-25T08:00:00", task._schedule_time().isoformat()) @freeze_time("2016-12-25") @override_settings(USE_TZ=True) def test_schedule_time_with_tz(self): - task = task_factory(self.TaskModelClass) + task = task_factory(self.task_type) est = zoneinfo.ZoneInfo("US/Eastern") task.scheduled_time = datetime(2016, 12, 25, 8, 0, 0, tzinfo=est) self.assertEqual("2016-12-25T13:00:00+00:00", task._schedule_time().isoformat()) def test_result_ttl_passthrough(self): - job = task_factory(self.TaskModelClass, result_ttl=500) + job = task_factory(self.task_type, result_ttl=500) entry = _get_job_from_scheduled_registry(job) self.assertEqual(entry.result_ttl, 500) class TestScheduledJob(BaseTestCases.TestSchedulableJob): - TaskModelClass = ScheduledTask + task_type = TaskType.ONCE def test_clean(self): - job = task_factory(self.TaskModelClass) + job = task_factory(self.task_type) job.queue = list(settings.QUEUES)[0] job.callable = "scheduler.tests.jobs.test_job" self.assertIsNone(job.clean()) def test_unschedulable_old_job(self): - job = task_factory(self.TaskModelClass, scheduled_time=timezone.now() - timedelta(hours=1)) + job = task_factory(self.task_type, scheduled_time=timezone.now() - timedelta(hours=1)) self.assertFalse(job.is_scheduled()) diff --git a/scheduler/tests/testtools.py b/scheduler/tests/testtools.py index 0238a25..634ce05 100644 --- a/scheduler/tests/testtools.py +++ b/scheduler/tests/testtools.py @@ -8,6 +8,7 @@ from scheduler import settings from scheduler.models import CronTask, TaskKwarg, RepeatableTask, ScheduledTask, BaseTask +from scheduler.models.task import TaskType, Task from scheduler.queues import get_queue @@ -26,7 +27,49 @@ def sequence_gen(): seq = sequence_gen() -def task_factory(cls, callable_name: str = "scheduler.tests.jobs.test_job", instance_only=False, **kwargs): +def task_factory( + task_type: TaskType, callable_name: str = "scheduler.tests.jobs.test_job", instance_only=False, **kwargs +): + values = dict( + name="Scheduled Job %d" % next(seq), + job_id=None, + queue=list(settings.QUEUES.keys())[0], + callable=callable_name, + enabled=True, + timeout=None, + ) + if task_type == TaskType.ONCE: + values.update( + dict( + result_ttl=None, + scheduled_time=timezone.now() + timedelta(days=1), + ) + ) + elif task_type == TaskType.REPEATABLE: + values.update( + dict( + result_ttl=None, + interval=1, + interval_unit="hours", + repeat=None, + scheduled_time=timezone.now() + timedelta(days=1), + ) + ) + elif task_type == CronTask: + values.update( + dict( + cron_string="0 0 * * *", + ) + ) + values.update(kwargs) + if instance_only: + instance = Task(task_type=task_type, **values) + else: + instance = Task.objects.create(task_type=task_type, **values) + return instance + + +def old_task_factory(cls, callable_name: str = "scheduler.tests.jobs.test_job", instance_only=False, **kwargs): values = dict( name="Scheduled Job %d" % next(seq), job_id=None, @@ -69,7 +112,7 @@ def task_factory(cls, callable_name: str = "scheduler.tests.jobs.test_job", inst def taskarg_factory(cls, **kwargs): content_object = kwargs.pop("content_object", None) if content_object is None: - content_object = task_factory(ScheduledTask) + content_object = old_task_factory(ScheduledTask) values = dict( arg_type="str", val="", diff --git a/scheduler/tools.py b/scheduler/tools.py index cac5316..714cd9f 100644 --- a/scheduler/tools.py +++ b/scheduler/tools.py @@ -1,17 +1,25 @@ import importlib import os -from typing import List, Any, Callable, Optional +from typing import List, Any, Callable, Optional, Union import croniter from django.apps import apps +from django.db import models from django.utils import timezone from django.utils.module_loading import import_string +from django.utils.translation import gettext_lazy as _ from scheduler.queues import get_queues, logger, get_queue from scheduler.rq_classes import DjangoWorker, MODEL_NAMES, JobExecution from scheduler.settings import SCHEDULER_CONFIG, Broker +class TaskType(models.TextChoices): + CRON = "CronTask", _("Cron Task") + REPEATABLE = "RepeatableTask", _("Repeatable Task") + ONCE = "OnceTask", _("Run once") + + def callable_func(callable_str: str) -> Callable: path = callable_str.split(".") module = importlib.import_module(".".join(path[:-1])) @@ -31,8 +39,14 @@ def get_next_cron_time(cron_string: Optional[str]) -> Optional[timezone.datetime return next_itr -def get_scheduled_task(task_model: str, task_id: int) -> "BaseTask": # noqa: F821 - if task_model not in MODEL_NAMES: +def get_scheduled_task(task_model: Union[TaskType, str], task_id: int) -> "BaseTask": # noqa: F821 + if isinstance(task_model, TaskType): + model = apps.get_model(app_label="scheduler", model_name="Task") + task = model.objects.filter(task_type=task_model, id=task_id).first() + if task is None: + raise ValueError(f"Job {task_model}:{task_id} does not exit") + return task + if isinstance(task_model, str) and task_model not in MODEL_NAMES: raise ValueError(f"Job Model `{task_model}` does not exist, choices are {MODEL_NAMES}") model = apps.get_model(app_label="scheduler", model_name=task_model) task = model.objects.filter(id=task_id).first() From 708e7e1fc3bca8dcb5ee1c52b39af377b6e74908 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Sat, 16 Nov 2024 10:13:03 -0500 Subject: [PATCH 14/47] update documentation and version --- poetry.lock | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5979b58..136ea0a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1623,17 +1623,17 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "valkey" -version = "6.0.2" +version = "6.0.1" description = "Python client for Valkey forked from redis-py" -optional = true +optional = false python-versions = ">=3.8" files = [ - {file = "valkey-6.0.2-py3-none-any.whl", hash = "sha256:dbbdd65439ee0dc5689502c54f1899504cc7268e85cb7fe8935f062178ff5805"}, - {file = "valkey-6.0.2.tar.gz", hash = "sha256:dc2e91512b82d1da0b91ab0cdbd8c97c0c0250281728cb32f9398760df9caeae"}, + {file = "valkey-6.0.1-py3-none-any.whl", hash = "sha256:6702bf323e88e50ef0be37aad697bcc6334edd40cc66f01259265dd410fa22dc"}, + {file = "valkey-6.0.1.tar.gz", hash = "sha256:58f4628dc038ab5aa04eea6e75557309c9412a8c45e81ad42d53e42b9a36e7dc"}, ] [package.dependencies] -async-timeout = {version = ">=4.0.3", markers = "python_version < \"3.11.3\""} +async-timeout = {version = ">=4.0.3", markers = "python_version < \"3.11\""} [package.extras] libvalkey = ["libvalkey (>=4.0.0)"] @@ -1752,10 +1752,9 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", type = ["pytest-mypy"] [extras] -valkey = ["valkey"] yaml = ["pyyaml"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "d2f15d1a3c26092b506e3da3a2539fe9ca83c8e68c7949c64f71f8af3401b3cb" +content-hash = "641d4bec80938f6c858d36d4d4cce1226021cd0a99263de0f9d3e9a240ae4512" From aeb4b80b997f54b5a4de11b474d36c13b0f6e95a Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 19 Nov 2024 15:18:23 -0500 Subject: [PATCH 15/47] fix:export --- scheduler/models/task.py | 4 +-- scheduler/rq_classes.py | 2 +- scheduler/tests/test_mgmt_cmds.py | 46 ++++++++++++------------- scheduler/tests/test_repeatable_task.py | 3 +- 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/scheduler/models/task.py b/scheduler/models/task.py index 8c40ed6..1ee5870 100644 --- a/scheduler/models/task.py +++ b/scheduler/models/task.py @@ -1,6 +1,6 @@ import math import uuid -from datetime import timedelta +from datetime import timedelta, datetime from typing import Dict import croniter @@ -311,7 +311,7 @@ def unschedule(self) -> bool: self.save(schedule_job=False) return True - def _schedule_time(self): + def _schedule_time(self) -> datetime: if self.task_type == TaskType.CRON: self.scheduled_time = tools.get_next_cron_time(self.cron_string) elif self.task_type == TaskType.REPEATABLE: diff --git a/scheduler/rq_classes.py b/scheduler/rq_classes.py index 4fce7ea..8c15e48 100644 --- a/scheduler/rq_classes.py +++ b/scheduler/rq_classes.py @@ -24,7 +24,7 @@ from scheduler import settings from scheduler.broker_types import PipelineType, ConnectionType -MODEL_NAMES = ["ScheduledTask", "RepeatableTask", "CronTask"] +MODEL_NAMES = ["ScheduledTask", "RepeatableTask", "CronTask", "Task"] rq_job_decorator = job ExecutionStatus = JobStatus diff --git a/scheduler/tests/test_mgmt_cmds.py b/scheduler/tests/test_mgmt_cmds.py index b1ba047..8c0b215 100644 --- a/scheduler/tests/test_mgmt_cmds.py +++ b/scheduler/tests/test_mgmt_cmds.py @@ -10,7 +10,7 @@ from scheduler.models import ScheduledTask, RepeatableTask, Task from scheduler.queues import get_queue from scheduler.tests.jobs import failing_job, test_job -from scheduler.tests.testtools import task_factory +from scheduler.tests.testtools import task_factory, old_task_factory from . import test_settings # noqa from .test_views import BaseTestCase from ..models.task import TaskType @@ -167,8 +167,8 @@ def tearDown(self) -> None: def test_export__should_export_job(self): jobs = list() - jobs.append(task_factory(ScheduledTask, enabled=True)) - jobs.append(task_factory(RepeatableTask, enabled=True)) + jobs.append(task_factory(TaskType.ONCE, enabled=True)) + jobs.append(task_factory(TaskType.REPEATABLE, enabled=True)) # act call_command("export", filename=self.tmpfile.name) @@ -180,8 +180,8 @@ def test_export__should_export_job(self): def test_export__should_export_enabled_jobs_only(self): jobs = list() - jobs.append(task_factory(ScheduledTask, enabled=True)) - jobs.append(task_factory(RepeatableTask, enabled=False)) + jobs.append(task_factory(TaskType.ONCE, enabled=True)) + jobs.append(task_factory(TaskType.REPEATABLE, enabled=False)) # act call_command("export", filename=self.tmpfile.name, enabled=True) @@ -192,8 +192,8 @@ def test_export__should_export_enabled_jobs_only(self): def test_export__should_export_job_yaml_without_yaml_lib(self): jobs = list() - jobs.append(task_factory(ScheduledTask, enabled=True)) - jobs.append(task_factory(RepeatableTask, enabled=True)) + jobs.append(task_factory(TaskType.ONCE, enabled=True)) + jobs.append(task_factory(TaskType.REPEATABLE, enabled=True)) # act with mock.patch.dict("sys.modules", {"yaml": None}): @@ -203,8 +203,8 @@ def test_export__should_export_job_yaml_without_yaml_lib(self): def test_export__should_export_job_yaml_green(self): jobs = list() - jobs.append(task_factory(ScheduledTask, enabled=True)) - jobs.append(task_factory(RepeatableTask, enabled=True)) + jobs.append(task_factory(TaskType.ONCE, enabled=True)) + jobs.append(task_factory(TaskType.REPEATABLE, enabled=True)) # act call_command("export", filename=self.tmpfile.name, format="yaml") @@ -224,8 +224,8 @@ def tearDown(self) -> None: def test_import__should_schedule_job(self): jobs = list() - jobs.append(task_factory(ScheduledTask, enabled=True, instance_only=True)) - jobs.append(task_factory(RepeatableTask, enabled=True, instance_only=True)) + jobs.append(task_factory(TaskType.ONCE, enabled=True, instance_only=True)) + jobs.append(task_factory(TaskType.REPEATABLE, enabled=True, instance_only=True)) res = json.dumps([j.to_dict() for j in jobs]) self.tmpfile.write(res) self.tmpfile.flush() @@ -241,8 +241,8 @@ def test_import__should_schedule_job(self): def test_import__should_schedule_job_yaml(self): tasks = list() - tasks.append(task_factory(ScheduledTask, enabled=True, instance_only=True)) - tasks.append(task_factory(RepeatableTask, enabled=True, instance_only=True)) + tasks.append(task_factory(TaskType.ONCE, enabled=True, instance_only=True)) + tasks.append(task_factory(TaskType.REPEATABLE, enabled=True, instance_only=True)) res = yaml.dump([j.to_dict() for j in tasks], default_flow_style=False) self.tmpfile.write(res) self.tmpfile.flush() @@ -258,8 +258,8 @@ def test_import__should_schedule_job_yaml(self): def test_import__should_schedule_job_yaml_without_yaml_lib(self): jobs = list() - jobs.append(task_factory(ScheduledTask, enabled=True, instance_only=True)) - jobs.append(task_factory(RepeatableTask, enabled=True, instance_only=True)) + jobs.append(task_factory(TaskType.ONCE, enabled=True, instance_only=True)) + jobs.append(task_factory(TaskType.REPEATABLE, enabled=True, instance_only=True)) res = yaml.dump([j.to_dict() for j in jobs], default_flow_style=False) self.tmpfile.write(res) self.tmpfile.flush() @@ -271,10 +271,10 @@ def test_import__should_schedule_job_yaml_without_yaml_lib(self): def test_import__should_schedule_job_reset(self): jobs = list() - task_factory(ScheduledTask, enabled=True) - task_factory(ScheduledTask, enabled=True) - jobs.append(task_factory(ScheduledTask, enabled=True)) - jobs.append(task_factory(RepeatableTask, enabled=True, instance_only=True)) + task_factory(TaskType.ONCE, enabled=True) + task_factory(TaskType.ONCE, enabled=True) + jobs.append(task_factory(TaskType.ONCE, enabled=True)) + jobs.append(task_factory(TaskType.REPEATABLE, enabled=True, instance_only=True)) res = json.dumps([j.to_dict() for j in jobs]) self.tmpfile.write(res) self.tmpfile.flush() @@ -298,8 +298,8 @@ def test_import__should_schedule_job_reset(self): def test_import__should_schedule_job_update_existing(self): tasks = list() - tasks.append(task_factory(ScheduledTask, enabled=True)) - tasks.append(task_factory(ScheduledTask, enabled=True)) + tasks.append(task_factory(TaskType.ONCE, enabled=True)) + tasks.append(task_factory(TaskType.ONCE, enabled=True)) res = json.dumps([j.to_dict() for j in tasks]) self.tmpfile.write(res) self.tmpfile.flush() @@ -318,8 +318,8 @@ def test_import__should_schedule_job_update_existing(self): def test_import__should_schedule_job_without_update_existing(self): tasks = list() - tasks.append(task_factory(ScheduledTask, enabled=True)) - tasks.append(task_factory(ScheduledTask, enabled=True)) + tasks.append(task_factory(TaskType.ONCE, enabled=True)) + tasks.append(task_factory(TaskType.ONCE, enabled=True)) res = json.dumps([j.to_dict() for j in tasks]) self.tmpfile.write(res) self.tmpfile.flush() diff --git a/scheduler/tests/test_repeatable_task.py b/scheduler/tests/test_repeatable_task.py index 43507e4..e29291e 100644 --- a/scheduler/tests/test_repeatable_task.py +++ b/scheduler/tests/test_repeatable_task.py @@ -8,10 +8,11 @@ from scheduler.models import RepeatableTask from scheduler.tests.test_old_models import BaseTestCases from .testtools import task_factory, _get_job_from_scheduled_registry +from ..tools import TaskType class TestRepeatableTask(BaseTestCases.TestSchedulableJob): - TaskModelClass = RepeatableTask + TaskModelClass = TaskType.REPEATABLE def test_unschedulable_old_job(self): job = task_factory(self.TaskModelClass, scheduled_time=timezone.now() - timedelta(hours=1), repeat=0) From e67408dd566c6783edcf5875e23853c86364c36a Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 19 Nov 2024 15:22:13 -0500 Subject: [PATCH 16/47] fix:import --- scheduler/management/commands/import.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scheduler/management/commands/import.py b/scheduler/management/commands/import.py index d1f0fce..60967c2 100644 --- a/scheduler/management/commands/import.py +++ b/scheduler/management/commands/import.py @@ -21,14 +21,13 @@ def job_model_str(model_str: str) -> str: def get_task_type(model_str: str) -> TaskType: model_str = job_model_str(model_str) - if model_str not in MODEL_NAMES: - raise ValueError(f"Invalid model {model_str}") if model_str == "CronTask": return TaskType.CRON elif model_str == "RepeatableTask": return TaskType.REPEATABLE - elif model_str == "ScheduledTask": + elif model_str in {"ScheduledTask", "OnceTask"}: return TaskType.ONCE + raise ValueError(f"Invalid model {model_str}") def create_task_from_dict(task_dict: Dict[str, Any], update): From eeb083081b292cb7d2eb4c920dd496ed53df558c Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 19 Nov 2024 15:36:16 -0500 Subject: [PATCH 17/47] fix:import --- scheduler/models/task.py | 5 +- scheduler/tests/test_mgmt_cmds.py | 336 ------------------ .../tests/test_mgmt_commands/__init__.py | 0 .../test_delete_failed_executions.py | 19 + .../tests/test_mgmt_commands/test_export.py | 71 ++++ .../tests/test_mgmt_commands/test_import.py | 134 +++++++ .../tests/test_mgmt_commands/test_rq_stats.py | 11 + .../test_mgmt_commands/test_rq_worker.py | 115 ++++++ .../tests/test_mgmt_commands/test_run_job.py | 19 + scheduler/tests/testtools.py | 3 +- 10 files changed, 374 insertions(+), 339 deletions(-) delete mode 100644 scheduler/tests/test_mgmt_cmds.py create mode 100644 scheduler/tests/test_mgmt_commands/__init__.py create mode 100644 scheduler/tests/test_mgmt_commands/test_delete_failed_executions.py create mode 100644 scheduler/tests/test_mgmt_commands/test_export.py create mode 100644 scheduler/tests/test_mgmt_commands/test_import.py create mode 100644 scheduler/tests/test_mgmt_commands/test_rq_stats.py create mode 100644 scheduler/tests/test_mgmt_commands/test_rq_worker.py create mode 100644 scheduler/tests/test_mgmt_commands/test_run_job.py diff --git a/scheduler/models/task.py b/scheduler/models/task.py index 1ee5870..00928b9 100644 --- a/scheduler/models/task.py +++ b/scheduler/models/task.py @@ -326,8 +326,9 @@ def _schedule_time(self) -> datetime: def to_dict(self) -> Dict: """Export model to dictionary, so it can be saved as external file backup""" + interval_unit = str(self.interval_unit) if self.interval_unit else None res = dict( - model=self.task_type, + model=str(self.task_type), name=self.name, callable=self.callable, callable_args=[ @@ -354,7 +355,7 @@ def to_dict(self) -> Dict: cron_string=getattr(self, "cron_string", None), scheduled_time=self._schedule_time().isoformat(), interval=getattr(self, "interval", None), - interval_unit=getattr(self, "interval_unit", None), + interval_unit=interval_unit, successful_runs=getattr(self, "successful_runs", None), failed_runs=getattr(self, "failed_runs", None), last_successful_run=getattr(self, "last_successful_run", None), diff --git a/scheduler/tests/test_mgmt_cmds.py b/scheduler/tests/test_mgmt_cmds.py deleted file mode 100644 index 8c0b215..0000000 --- a/scheduler/tests/test_mgmt_cmds.py +++ /dev/null @@ -1,336 +0,0 @@ -import json -import os -import tempfile -from unittest import mock - -import yaml -from django.core.management import call_command -from django.test import TestCase - -from scheduler.models import ScheduledTask, RepeatableTask, Task -from scheduler.queues import get_queue -from scheduler.tests.jobs import failing_job, test_job -from scheduler.tests.testtools import task_factory, old_task_factory -from . import test_settings # noqa -from .test_views import BaseTestCase -from ..models.task import TaskType -from ..tools import create_worker - - -class RqworkerTestCase(TestCase): - - def test_rqworker__no_queues_params(self): - queue = get_queue("default") - - # enqueue some jobs that will fail - jobs = [] - job_ids = [] - for _ in range(0, 3): - job = queue.enqueue(failing_job) - jobs.append(job) - job_ids.append(job.id) - - # Create a worker to execute these jobs - call_command("rqworker", fork_job_execution=False, burst=True) - - # check if all jobs are really failed - for job in jobs: - self.assertTrue(job.is_failed) - - def test_rqworker__job_class_param__green(self): - queue = get_queue("default") - - # enqueue some jobs that will fail - jobs = [] - job_ids = [] - for _ in range(0, 3): - job = queue.enqueue(failing_job) - jobs.append(job) - job_ids.append(job.id) - - # Create a worker to execute these jobs - call_command( - "rqworker", "--job-class", "scheduler.rq_classes.JobExecution", fork_job_execution=False, burst=True - ) - - # check if all jobs are really failed - for job in jobs: - self.assertTrue(job.is_failed) - - def test_rqworker__bad_job_class__fail(self): - queue = get_queue("default") - - # enqueue some jobs that will fail - jobs = [] - job_ids = [] - for _ in range(0, 3): - job = queue.enqueue(failing_job) - jobs.append(job) - job_ids.append(job.id) - - # Create a worker to execute these jobs - with self.assertRaises(ImportError): - call_command("rqworker", "--job-class", "rq.badclass", fork_job_execution=False, burst=True) - - def test_rqworker__run_jobs(self): - queue = get_queue("default") - - # enqueue some jobs that will fail - jobs = [] - job_ids = [] - for _ in range(0, 3): - job = queue.enqueue(failing_job) - jobs.append(job) - job_ids.append(job.id) - - # Create a worker to execute these jobs - call_command("rqworker", "default", fork_job_execution=False, burst=True) - - # check if all jobs are really failed - for job in jobs: - self.assertTrue(job.is_failed) - - def test_rqworker__worker_with_two_queues(self): - queue = get_queue("default") - queue2 = get_queue("django_tasks_scheduler_test") - - # enqueue some jobs that will fail - jobs = [] - job_ids = [] - for _ in range(0, 3): - job = queue.enqueue(failing_job) - jobs.append(job) - job_ids.append(job.id) - job = queue2.enqueue(failing_job) - jobs.append(job) - job_ids.append(job.id) - - # Create a worker to execute these jobs - call_command("rqworker", "default", "django_tasks_scheduler_test", fork_job_execution=False, burst=True) - - # check if all jobs are really failed - for job in jobs: - self.assertTrue(job.is_failed) - - def test_rqworker__worker_with_one_queue__does_not_perform_other_queue_job(self): - queue = get_queue("default") - queue2 = get_queue("django_tasks_scheduler_test") - - job = queue.enqueue(failing_job) - other_job = queue2.enqueue(failing_job) - - # Create a worker to execute these jobs - call_command("rqworker", "default", fork_job_execution=False, burst=True) - # assert - self.assertTrue(job.is_failed) - self.assertTrue(other_job.is_queued) - - -class RqstatsTest(TestCase): - def test_rqstats__does_not_fail(self): - call_command("rqstats", "-j") - call_command("rqstats", "-y") - call_command("rqstats") - - -class DeleteFailedExecutionsTest(BaseTestCase): - def test_delete_failed_executions__delete_jobs(self): - queue = get_queue("default") - call_command("delete_failed_executions", queue="default") - queue.enqueue(failing_job) - worker = create_worker("default") - worker.work(burst=True) - self.assertEqual(1, len(queue.failed_job_registry)) - call_command("delete_failed_executions", queue="default") - self.assertEqual(0, len(queue.failed_job_registry)) - - -class RunJobTest(TestCase): - def test_run_job__should_schedule_job(self): - queue = get_queue("default") - queue.empty() - func_name = f"{test_job.__module__}.{test_job.__name__}" - # act - call_command("run_job", func_name, queue="default") - # assert - job_list = queue.get_jobs() - self.assertEqual(1, len(job_list)) - self.assertEqual(func_name + "()", job_list[0].get_call_string()) - - -class ExportTest(TestCase): - def setUp(self) -> None: - self.tmpfile = tempfile.NamedTemporaryFile() - - def tearDown(self) -> None: - os.remove(self.tmpfile.name) - - def test_export__should_export_job(self): - jobs = list() - jobs.append(task_factory(TaskType.ONCE, enabled=True)) - jobs.append(task_factory(TaskType.REPEATABLE, enabled=True)) - - # act - call_command("export", filename=self.tmpfile.name) - # assert - result = json.load(self.tmpfile) - self.assertEqual(len(jobs), len(result)) - self.assertEqual(result[0], jobs[0].to_dict()) - self.assertEqual(result[1], jobs[1].to_dict()) - - def test_export__should_export_enabled_jobs_only(self): - jobs = list() - jobs.append(task_factory(TaskType.ONCE, enabled=True)) - jobs.append(task_factory(TaskType.REPEATABLE, enabled=False)) - - # act - call_command("export", filename=self.tmpfile.name, enabled=True) - # assert - result = json.load(self.tmpfile) - self.assertEqual(len(jobs) - 1, len(result)) - self.assertEqual(result[0], jobs[0].to_dict()) - - def test_export__should_export_job_yaml_without_yaml_lib(self): - jobs = list() - jobs.append(task_factory(TaskType.ONCE, enabled=True)) - jobs.append(task_factory(TaskType.REPEATABLE, enabled=True)) - - # act - with mock.patch.dict("sys.modules", {"yaml": None}): - with self.assertRaises(SystemExit) as cm: - call_command("export", filename=self.tmpfile.name, format="yaml") - self.assertEqual(cm.exception.code, 1) - - def test_export__should_export_job_yaml_green(self): - jobs = list() - jobs.append(task_factory(TaskType.ONCE, enabled=True)) - jobs.append(task_factory(TaskType.REPEATABLE, enabled=True)) - - # act - call_command("export", filename=self.tmpfile.name, format="yaml") - # assert - result = yaml.load(self.tmpfile, yaml.SafeLoader) - self.assertEqual(len(jobs), len(result)) - self.assertEqual(result[0], jobs[0].to_dict()) - self.assertEqual(result[1], jobs[1].to_dict()) - - -class ImportTest(TestCase): - def setUp(self) -> None: - self.tmpfile = tempfile.NamedTemporaryFile(mode="w") - - def tearDown(self) -> None: - os.remove(self.tmpfile.name) - - def test_import__should_schedule_job(self): - jobs = list() - jobs.append(task_factory(TaskType.ONCE, enabled=True, instance_only=True)) - jobs.append(task_factory(TaskType.REPEATABLE, enabled=True, instance_only=True)) - res = json.dumps([j.to_dict() for j in jobs]) - self.tmpfile.write(res) - self.tmpfile.flush() - # act - call_command("import", filename=self.tmpfile.name) - # assert - self.assertEqual(1, Task.objects.filter(task_type=TaskType.ONCE).count()) - self.assertEqual(1, Task.objects.filter(task_type=TaskType.REPEATABLE).count()) - db_job = Task.objects.filter(task_type=TaskType.ONCE).first() - attrs = ["name", "queue", "callable", "enabled", "timeout"] - for attr in attrs: - self.assertEqual(getattr(jobs[0], attr), getattr(db_job, attr)) - - def test_import__should_schedule_job_yaml(self): - tasks = list() - tasks.append(task_factory(TaskType.ONCE, enabled=True, instance_only=True)) - tasks.append(task_factory(TaskType.REPEATABLE, enabled=True, instance_only=True)) - res = yaml.dump([j.to_dict() for j in tasks], default_flow_style=False) - self.tmpfile.write(res) - self.tmpfile.flush() - # act - call_command("import", filename=self.tmpfile.name, format="yaml") - # assert - self.assertEqual(1, Task.objects.filter(task_type=TaskType.ONCE).count()) - self.assertEqual(1, Task.objects.filter(task_type=TaskType.REPEATABLE).count()) - db_job = Task.objects.filter(task_type=TaskType.ONCE).first() - attrs = ["name", "queue", "callable", "enabled", "timeout"] - for attr in attrs: - self.assertEqual(getattr(tasks[0], attr), getattr(db_job, attr)) - - def test_import__should_schedule_job_yaml_without_yaml_lib(self): - jobs = list() - jobs.append(task_factory(TaskType.ONCE, enabled=True, instance_only=True)) - jobs.append(task_factory(TaskType.REPEATABLE, enabled=True, instance_only=True)) - res = yaml.dump([j.to_dict() for j in jobs], default_flow_style=False) - self.tmpfile.write(res) - self.tmpfile.flush() - # act - with mock.patch.dict("sys.modules", {"yaml": None}): - with self.assertRaises(SystemExit) as cm: - call_command("import", filename=self.tmpfile.name, format="yaml") - self.assertEqual(cm.exception.code, 1) - - def test_import__should_schedule_job_reset(self): - jobs = list() - task_factory(TaskType.ONCE, enabled=True) - task_factory(TaskType.ONCE, enabled=True) - jobs.append(task_factory(TaskType.ONCE, enabled=True)) - jobs.append(task_factory(TaskType.REPEATABLE, enabled=True, instance_only=True)) - res = json.dumps([j.to_dict() for j in jobs]) - self.tmpfile.write(res) - self.tmpfile.flush() - # act - call_command( - "import", - filename=self.tmpfile.name, - reset=True, - ) - # assert - self.assertEqual(1, Task.objects.filter(task_type=TaskType.ONCE).count()) - db_job = Task.objects.filter(task_type=TaskType.ONCE).first() - attrs = ["name", "queue", "callable", "enabled", "timeout"] - for attr in attrs: - self.assertEqual(getattr(jobs[0], attr), getattr(db_job, attr)) - self.assertEqual(1, Task.objects.filter(task_type=TaskType.REPEATABLE).count()) - db_job = Task.objects.filter(task_type=TaskType.REPEATABLE).first() - attrs = ["name", "queue", "callable", "enabled", "timeout"] - for attr in attrs: - self.assertEqual(getattr(jobs[1], attr), getattr(db_job, attr)) - - def test_import__should_schedule_job_update_existing(self): - tasks = list() - tasks.append(task_factory(TaskType.ONCE, enabled=True)) - tasks.append(task_factory(TaskType.ONCE, enabled=True)) - res = json.dumps([j.to_dict() for j in tasks]) - self.tmpfile.write(res) - self.tmpfile.flush() - # act - call_command( - "import", - filename=self.tmpfile.name, - update=True, - ) - # assert - self.assertEqual(2, Task.objects.filter(task_type=TaskType.ONCE).count()) - db_job = Task.objects.filter(task_type=TaskType.ONCE).get(name=tasks[0].name) - attrs = ["name", "queue", "callable", "enabled", "timeout"] - for attr in attrs: - self.assertEqual(getattr(tasks[0], attr), getattr(db_job, attr)) - - def test_import__should_schedule_job_without_update_existing(self): - tasks = list() - tasks.append(task_factory(TaskType.ONCE, enabled=True)) - tasks.append(task_factory(TaskType.ONCE, enabled=True)) - res = json.dumps([j.to_dict() for j in tasks]) - self.tmpfile.write(res) - self.tmpfile.flush() - # act - call_command( - "import", - filename=self.tmpfile.name, - ) - # assert - self.assertEqual(2, Task.objects.filter(task_type=TaskType.ONCE).count()) - db_job = Task.objects.get(name=tasks[0].name) - attrs = ["id", "name", "queue", "callable", "enabled", "timeout"] - for attr in attrs: - self.assertEqual(getattr(tasks[0], attr), getattr(db_job, attr)) diff --git a/scheduler/tests/test_mgmt_commands/__init__.py b/scheduler/tests/test_mgmt_commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scheduler/tests/test_mgmt_commands/test_delete_failed_executions.py b/scheduler/tests/test_mgmt_commands/test_delete_failed_executions.py new file mode 100644 index 0000000..74a53df --- /dev/null +++ b/scheduler/tests/test_mgmt_commands/test_delete_failed_executions.py @@ -0,0 +1,19 @@ +from django.core.management import call_command + +from scheduler.queues import get_queue +from scheduler.tests.jobs import failing_job +from scheduler.tests.test_views import BaseTestCase +from scheduler.tools import create_worker +from scheduler.tests import test_settings # noqa + + +class DeleteFailedExecutionsTest(BaseTestCase): + def test_delete_failed_executions__delete_jobs(self): + queue = get_queue("default") + call_command("delete_failed_executions", queue="default") + queue.enqueue(failing_job) + worker = create_worker("default") + worker.work(burst=True) + self.assertEqual(1, len(queue.failed_job_registry)) + call_command("delete_failed_executions", queue="default") + self.assertEqual(0, len(queue.failed_job_registry)) diff --git a/scheduler/tests/test_mgmt_commands/test_export.py b/scheduler/tests/test_mgmt_commands/test_export.py new file mode 100644 index 0000000..b67b60a --- /dev/null +++ b/scheduler/tests/test_mgmt_commands/test_export.py @@ -0,0 +1,71 @@ +import json +import os +import tempfile +from unittest import mock + +import yaml +from django.core.management import call_command +from django.test import TestCase + +from scheduler.tests.testtools import task_factory +from scheduler.tools import TaskType +from scheduler.tests import test_settings # noqa + + +class ExportTest(TestCase): + def setUp(self) -> None: + self.tmpfile = tempfile.NamedTemporaryFile() + + def tearDown(self) -> None: + os.remove(self.tmpfile.name) + + def test_export__should_export_job(self): + jobs = list() + jobs.append(task_factory(TaskType.ONCE, enabled=True)) + jobs.append(task_factory(TaskType.REPEATABLE, enabled=True)) + + # act + call_command("export", filename=self.tmpfile.name) + # assert + result = json.load(self.tmpfile) + self.assertEqual(len(jobs), len(result)) + self.assertEqual(result[0], jobs[0].to_dict()) + self.assertEqual(result[1], jobs[1].to_dict()) + + def test_export__should_export_enabled_jobs_only(self): + jobs = list() + jobs.append(task_factory(TaskType.ONCE, enabled=True)) + jobs.append(task_factory(TaskType.REPEATABLE, enabled=False)) + + # act + call_command("export", filename=self.tmpfile.name, enabled=True) + # assert + result = json.load(self.tmpfile) + self.assertEqual(len(jobs) - 1, len(result)) + self.assertEqual(result[0], jobs[0].to_dict()) + + def test_export__should_export_job_yaml_without_yaml_lib(self): + jobs = list() + jobs.append(task_factory(TaskType.ONCE, enabled=True)) + jobs.append(task_factory(TaskType.REPEATABLE, enabled=True)) + + # act + with mock.patch.dict("sys.modules", {"yaml": None}): + with self.assertRaises(SystemExit) as cm: + call_command("export", filename=self.tmpfile.name, format="yaml") + self.assertEqual(cm.exception.code, 1) + + def test_export__should_export_job_yaml_green(self): + jobs = list() + jobs.append(task_factory(TaskType.ONCE, enabled=True)) + jobs.append(task_factory(TaskType.REPEATABLE, enabled=True)) + jobs.append(task_factory(TaskType.CRON, enabled=True)) + + # act + call_command("export", filename=self.tmpfile.name, format="yaml") + # assert + result = yaml.load(self.tmpfile, yaml.SafeLoader) + self.assertEqual(len(jobs), len(result)) + self.assertEqual(result[0], jobs[0].to_dict()) + self.assertEqual(result[1], jobs[1].to_dict()) + self.assertEqual(result[2], jobs[2].to_dict()) diff --git a/scheduler/tests/test_mgmt_commands/test_import.py b/scheduler/tests/test_mgmt_commands/test_import.py new file mode 100644 index 0000000..c856e28 --- /dev/null +++ b/scheduler/tests/test_mgmt_commands/test_import.py @@ -0,0 +1,134 @@ +import json +import os +import tempfile +from unittest import mock + +import yaml +from django.core.management import call_command +from django.test import TestCase + +from scheduler.models import Task +from scheduler.tests.testtools import task_factory +from scheduler.tools import TaskType +from scheduler.tests import test_settings # noqa + + +class ImportTest(TestCase): + def setUp(self) -> None: + self.tmpfile = tempfile.NamedTemporaryFile(mode="w") + + def tearDown(self) -> None: + os.remove(self.tmpfile.name) + + def test_import__should_schedule_job(self): + jobs = list() + jobs.append(task_factory(TaskType.ONCE, enabled=True, instance_only=True)) + jobs.append(task_factory(TaskType.REPEATABLE, enabled=True, instance_only=True)) + res = json.dumps([j.to_dict() for j in jobs]) + self.tmpfile.write(res) + self.tmpfile.flush() + # act + call_command("import", filename=self.tmpfile.name) + # assert + self.assertEqual(1, Task.objects.filter(task_type=TaskType.ONCE).count()) + self.assertEqual(1, Task.objects.filter(task_type=TaskType.REPEATABLE).count()) + db_job = Task.objects.filter(task_type=TaskType.ONCE).first() + attrs = ["name", "queue", "callable", "enabled", "timeout"] + for attr in attrs: + self.assertEqual(getattr(jobs[0], attr), getattr(db_job, attr)) + + def test_import__should_schedule_job_yaml(self): + tasks = list() + tasks.append(task_factory(TaskType.ONCE, enabled=True, instance_only=True)) + tasks.append(task_factory(TaskType.REPEATABLE, enabled=True, instance_only=True)) + res = yaml.dump([j.to_dict() for j in tasks], default_flow_style=False) + self.tmpfile.write(res) + self.tmpfile.flush() + # act + call_command("import", filename=self.tmpfile.name, format="yaml") + # assert + self.assertEqual(1, Task.objects.filter(task_type=TaskType.ONCE).count()) + self.assertEqual(1, Task.objects.filter(task_type=TaskType.REPEATABLE).count()) + db_job = Task.objects.filter(task_type=TaskType.ONCE).first() + attrs = ["name", "queue", "callable", "enabled", "timeout"] + for attr in attrs: + self.assertEqual(getattr(tasks[0], attr), getattr(db_job, attr)) + + def test_import__should_schedule_job_yaml_without_yaml_lib(self): + jobs = list() + jobs.append(task_factory(TaskType.ONCE, enabled=True, instance_only=True)) + jobs.append(task_factory(TaskType.REPEATABLE, enabled=True, instance_only=True)) + res = yaml.dump([j.to_dict() for j in jobs], default_flow_style=False) + self.tmpfile.write(res) + self.tmpfile.flush() + # act + with mock.patch.dict("sys.modules", {"yaml": None}): + with self.assertRaises(SystemExit) as cm: + call_command("import", filename=self.tmpfile.name, format="yaml") + self.assertEqual(cm.exception.code, 1) + + def test_import__should_schedule_job_reset(self): + jobs = list() + task_factory(TaskType.ONCE, enabled=True) + task_factory(TaskType.ONCE, enabled=True) + jobs.append(task_factory(TaskType.ONCE, enabled=True)) + jobs.append(task_factory(TaskType.REPEATABLE, enabled=True, instance_only=True)) + res = json.dumps([j.to_dict() for j in jobs]) + self.tmpfile.write(res) + self.tmpfile.flush() + # act + call_command( + "import", + filename=self.tmpfile.name, + reset=True, + ) + # assert + self.assertEqual(1, Task.objects.filter(task_type=TaskType.ONCE).count()) + db_job = Task.objects.filter(task_type=TaskType.ONCE).first() + attrs = ["name", "queue", "callable", "enabled", "timeout"] + for attr in attrs: + self.assertEqual(getattr(jobs[0], attr), getattr(db_job, attr)) + self.assertEqual(1, Task.objects.filter(task_type=TaskType.REPEATABLE).count()) + db_job = Task.objects.filter(task_type=TaskType.REPEATABLE).first() + attrs = ["name", "queue", "callable", "enabled", "timeout"] + for attr in attrs: + self.assertEqual(getattr(jobs[1], attr), getattr(db_job, attr)) + + def test_import__should_schedule_job_update_existing(self): + tasks = list() + tasks.append(task_factory(TaskType.ONCE, enabled=True)) + tasks.append(task_factory(TaskType.ONCE, enabled=True)) + res = json.dumps([j.to_dict() for j in tasks]) + self.tmpfile.write(res) + self.tmpfile.flush() + # act + call_command( + "import", + filename=self.tmpfile.name, + update=True, + ) + # assert + self.assertEqual(2, Task.objects.filter(task_type=TaskType.ONCE).count()) + db_job = Task.objects.filter(task_type=TaskType.ONCE).get(name=tasks[0].name) + attrs = ["name", "queue", "callable", "enabled", "timeout"] + for attr in attrs: + self.assertEqual(getattr(tasks[0], attr), getattr(db_job, attr)) + + def test_import__should_schedule_job_without_update_existing(self): + tasks = list() + tasks.append(task_factory(TaskType.ONCE, enabled=True)) + tasks.append(task_factory(TaskType.ONCE, enabled=True)) + res = json.dumps([j.to_dict() for j in tasks]) + self.tmpfile.write(res) + self.tmpfile.flush() + # act + call_command( + "import", + filename=self.tmpfile.name, + ) + # assert + self.assertEqual(2, Task.objects.filter(task_type=TaskType.ONCE).count()) + db_job = Task.objects.get(name=tasks[0].name) + attrs = ["id", "name", "queue", "callable", "enabled", "timeout"] + for attr in attrs: + self.assertEqual(getattr(tasks[0], attr), getattr(db_job, attr)) diff --git a/scheduler/tests/test_mgmt_commands/test_rq_stats.py b/scheduler/tests/test_mgmt_commands/test_rq_stats.py new file mode 100644 index 0000000..0daf641 --- /dev/null +++ b/scheduler/tests/test_mgmt_commands/test_rq_stats.py @@ -0,0 +1,11 @@ +from django.core.management import call_command +from django.test import TestCase + +from scheduler.tests import test_settings # noqa + + +class RqstatsTest(TestCase): + def test_rqstats__does_not_fail(self): + call_command("rqstats", "-j") + call_command("rqstats", "-y") + call_command("rqstats") diff --git a/scheduler/tests/test_mgmt_commands/test_rq_worker.py b/scheduler/tests/test_mgmt_commands/test_rq_worker.py new file mode 100644 index 0000000..c4e4e49 --- /dev/null +++ b/scheduler/tests/test_mgmt_commands/test_rq_worker.py @@ -0,0 +1,115 @@ +from django.core.management import call_command +from django.test import TestCase + +from scheduler.queues import get_queue +from scheduler.tests.jobs import failing_job +from scheduler.tests import test_settings # noqa + + +class RqworkerTestCase(TestCase): + + def test_rqworker__no_queues_params(self): + queue = get_queue("default") + + # enqueue some jobs that will fail + jobs = [] + job_ids = [] + for _ in range(0, 3): + job = queue.enqueue(failing_job) + jobs.append(job) + job_ids.append(job.id) + + # Create a worker to execute these jobs + call_command("rqworker", fork_job_execution=False, burst=True) + + # check if all jobs are really failed + for job in jobs: + self.assertTrue(job.is_failed) + + def test_rqworker__job_class_param__green(self): + queue = get_queue("default") + + # enqueue some jobs that will fail + jobs = [] + job_ids = [] + for _ in range(0, 3): + job = queue.enqueue(failing_job) + jobs.append(job) + job_ids.append(job.id) + + # Create a worker to execute these jobs + call_command( + "rqworker", "--job-class", "scheduler.rq_classes.JobExecution", fork_job_execution=False, burst=True + ) + + # check if all jobs are really failed + for job in jobs: + self.assertTrue(job.is_failed) + + def test_rqworker__bad_job_class__fail(self): + queue = get_queue("default") + + # enqueue some jobs that will fail + jobs = [] + job_ids = [] + for _ in range(0, 3): + job = queue.enqueue(failing_job) + jobs.append(job) + job_ids.append(job.id) + + # Create a worker to execute these jobs + with self.assertRaises(ImportError): + call_command("rqworker", "--job-class", "rq.badclass", fork_job_execution=False, burst=True) + + def test_rqworker__run_jobs(self): + queue = get_queue("default") + + # enqueue some jobs that will fail + jobs = [] + job_ids = [] + for _ in range(0, 3): + job = queue.enqueue(failing_job) + jobs.append(job) + job_ids.append(job.id) + + # Create a worker to execute these jobs + call_command("rqworker", "default", fork_job_execution=False, burst=True) + + # check if all jobs are really failed + for job in jobs: + self.assertTrue(job.is_failed) + + def test_rqworker__worker_with_two_queues(self): + queue = get_queue("default") + queue2 = get_queue("django_tasks_scheduler_test") + + # enqueue some jobs that will fail + jobs = [] + job_ids = [] + for _ in range(0, 3): + job = queue.enqueue(failing_job) + jobs.append(job) + job_ids.append(job.id) + job = queue2.enqueue(failing_job) + jobs.append(job) + job_ids.append(job.id) + + # Create a worker to execute these jobs + call_command("rqworker", "default", "django_tasks_scheduler_test", fork_job_execution=False, burst=True) + + # check if all jobs are really failed + for job in jobs: + self.assertTrue(job.is_failed) + + def test_rqworker__worker_with_one_queue__does_not_perform_other_queue_job(self): + queue = get_queue("default") + queue2 = get_queue("django_tasks_scheduler_test") + + job = queue.enqueue(failing_job) + other_job = queue2.enqueue(failing_job) + + # Create a worker to execute these jobs + call_command("rqworker", "default", fork_job_execution=False, burst=True) + # assert + self.assertTrue(job.is_failed) + self.assertTrue(other_job.is_queued) diff --git a/scheduler/tests/test_mgmt_commands/test_run_job.py b/scheduler/tests/test_mgmt_commands/test_run_job.py new file mode 100644 index 0000000..4efe24d --- /dev/null +++ b/scheduler/tests/test_mgmt_commands/test_run_job.py @@ -0,0 +1,19 @@ +from django.core.management import call_command +from django.test import TestCase + +from scheduler.queues import get_queue +from scheduler.tests.jobs import test_job +from scheduler.tests import test_settings # noqa + + +class RunJobTest(TestCase): + def test_run_job__should_schedule_job(self): + queue = get_queue("default") + queue.empty() + func_name = f"{test_job.__module__}.{test_job.__name__}" + # act + call_command("run_job", func_name, queue="default") + # assert + job_list = queue.get_jobs() + self.assertEqual(1, len(job_list)) + self.assertEqual(func_name + "()", job_list[0].get_call_string()) diff --git a/scheduler/tests/testtools.py b/scheduler/tests/testtools.py index 634ce05..0347139 100644 --- a/scheduler/tests/testtools.py +++ b/scheduler/tests/testtools.py @@ -55,7 +55,7 @@ def task_factory( scheduled_time=timezone.now() + timedelta(days=1), ) ) - elif task_type == CronTask: + elif task_type == TaskType.CRON: values.update( dict( cron_string="0 0 * * *", @@ -69,6 +69,7 @@ def task_factory( return instance +# TODO remove def old_task_factory(cls, callable_name: str = "scheduler.tests.jobs.test_job", instance_only=False, **kwargs): values = dict( name="Scheduled Job %d" % next(seq), From c901d19616d6b0364aa4b4452dd50ab399acce7b Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 19 Nov 2024 15:38:51 -0500 Subject: [PATCH 18/47] wip --- scheduler/tests/test_old_models.py | 8 +++----- scheduler/tests/test_views.py | 13 ++++++------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/scheduler/tests/test_old_models.py b/scheduler/tests/test_old_models.py index 8e04b45..347da88 100644 --- a/scheduler/tests/test_old_models.py +++ b/scheduler/tests/test_old_models.py @@ -341,9 +341,7 @@ def test_admin_run_job_now_enqueues_job_at(self): def test_admin_change_view(self): # arrange self.client.login(username="admin", password="admin") - task = old_task_factory( - self.TaskModelClass, - ) + task = old_task_factory(self.TaskModelClass) model = task._meta.model.__name__.lower() url = reverse( f"admin:scheduler_{model}_change", @@ -512,7 +510,7 @@ def test_admin_delete_selected(self): scheduled_jobs = queue.scheduled_job_registry.get_job_ids() self.assertNotIn(job_id, scheduled_jobs) - class TestSchedulableJob(TestBaseTask): + class TestSchedulableTask(TestBaseTask): # Currently ScheduledJob and RepeatableJob TaskModelClass = ScheduledTask @@ -537,7 +535,7 @@ def test_result_ttl_passthrough(self): self.assertEqual(entry.result_ttl, 500) -class TestScheduledJob(BaseTestCases.TestSchedulableJob): +class TestScheduledTask(BaseTestCases.TestSchedulableTask): TaskModelClass = ScheduledTask def test_clean(self): diff --git a/scheduler/tests/test_views.py b/scheduler/tests/test_views.py index afae1d0..930cb9e 100644 --- a/scheduler/tests/test_views.py +++ b/scheduler/tests/test_views.py @@ -8,12 +8,11 @@ from django.urls import reverse from scheduler.queues import get_queue -from scheduler.tools import create_worker -from . import test_settings # noqa -from .jobs import failing_job, long_job, test_job -from .testtools import assert_message_in_response, task_factory, _get_job_from_scheduled_registry -from ..models import ScheduledTask -from ..rq_classes import JobExecution, ExecutionStatus +from scheduler.rq_classes import JobExecution, ExecutionStatus +from scheduler.tests import test_settings # noqa +from scheduler.tests.jobs import failing_job, long_job, test_job +from scheduler.tests.testtools import assert_message_in_response, task_factory, _get_job_from_scheduled_registry +from scheduler.tools import create_worker, TaskType class BaseTestCase(TestCase): @@ -358,7 +357,7 @@ def test_job_details(self): def test_scheduled_job_details(self): """Job data is displayed properly""" - scheduled_job = task_factory(ScheduledTask, enabled=True) + scheduled_job = task_factory(TaskType.ONCE, enabled=True) job = _get_job_from_scheduled_registry(scheduled_job) url = reverse( From 696113280124361c7a917948461778efbad272f3 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 19 Nov 2024 15:50:31 -0500 Subject: [PATCH 19/47] wip --- scheduler/tests/test_old_models/__init__.py | 0 .../{ => test_old_models}/test_cron_task.py | 6 +- .../{ => test_old_models}/test_old_models.py | 13 +- .../test_old_repeatable_task.py | 207 ++++++++++++++++++ .../test_old_task_model.py | 17 +- scheduler/tests/test_task_types/__init__.py | 0 .../tests/test_task_types/test_cron_task.py | 78 +++++++ .../test_repeatable_task.py | 56 ++--- .../{ => test_task_types}/test_task_model.py | 20 +- 9 files changed, 334 insertions(+), 63 deletions(-) create mode 100644 scheduler/tests/test_old_models/__init__.py rename scheduler/tests/{ => test_old_models}/test_cron_task.py (94%) rename scheduler/tests/{ => test_old_models}/test_old_models.py (99%) create mode 100644 scheduler/tests/test_old_models/test_old_repeatable_task.py rename scheduler/tests/{ => test_old_models}/test_old_task_model.py (98%) create mode 100644 scheduler/tests/test_task_types/__init__.py create mode 100644 scheduler/tests/test_task_types/test_cron_task.py rename scheduler/tests/{ => test_task_types}/test_repeatable_task.py (76%) rename scheduler/tests/{ => test_task_types}/test_task_model.py (98%) diff --git a/scheduler/tests/test_old_models/__init__.py b/scheduler/tests/test_old_models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scheduler/tests/test_cron_task.py b/scheduler/tests/test_old_models/test_cron_task.py similarity index 94% rename from scheduler/tests/test_cron_task.py rename to scheduler/tests/test_old_models/test_cron_task.py index 81df24d..33743ec 100644 --- a/scheduler/tests/test_cron_task.py +++ b/scheduler/tests/test_old_models/test_cron_task.py @@ -3,9 +3,9 @@ from scheduler import settings from scheduler.models import CronTask from scheduler.tools import create_worker -from .test_old_models import BaseTestCases -from .testtools import old_task_factory -from ..queues import get_queue +from scheduler.tests.test_old_models.test_old_models import BaseTestCases +from scheduler.tests.testtools import old_task_factory +from scheduler.queues import get_queue class TestCronTask(BaseTestCases.TestBaseTask): diff --git a/scheduler/tests/test_old_models.py b/scheduler/tests/test_old_models/test_old_models.py similarity index 99% rename from scheduler/tests/test_old_models.py rename to scheduler/tests/test_old_models/test_old_models.py index 347da88..38696bd 100644 --- a/scheduler/tests/test_old_models.py +++ b/scheduler/tests/test_old_models/test_old_models.py @@ -10,16 +10,11 @@ from scheduler import settings from scheduler.models import BaseTask, TaskArg, TaskKwarg, ScheduledTask +from scheduler.queues import get_queue +from scheduler.tests import jobs +from scheduler.tests.testtools import ( + old_task_factory, taskarg_factory, _get_job_from_scheduled_registry, SchedulerBaseCase, _get_executions) from scheduler.tools import run_task, create_worker -from . import jobs -from .testtools import ( - old_task_factory, - taskarg_factory, - _get_job_from_scheduled_registry, - SchedulerBaseCase, - _get_executions, -) -from ..queues import get_queue def assert_response_has_msg(response, message): diff --git a/scheduler/tests/test_old_models/test_old_repeatable_task.py b/scheduler/tests/test_old_models/test_old_repeatable_task.py new file mode 100644 index 0000000..5e4c4b8 --- /dev/null +++ b/scheduler/tests/test_old_models/test_old_repeatable_task.py @@ -0,0 +1,207 @@ +from datetime import timedelta + +from django.core.exceptions import ValidationError +from django.test import override_settings +from django.utils import timezone + +from scheduler import settings +from scheduler.models import RepeatableTask +from scheduler.tests.test_old_models.test_old_task_model import BaseTestCases + +from scheduler.tests.testtools import old_task_factory, _get_job_from_scheduled_registry + + +class TestRepeatableTask(BaseTestCases.TestSchedulableTask): + TaskModelClass = RepeatableTask + + def test_unschedulable_old_job(self): + job = old_task_factory(self.TaskModelClass, scheduled_time=timezone.now() - timedelta(hours=1), repeat=0) + self.assertFalse(job.is_scheduled()) + + def test_schedulable_old_job_repeat_none(self): + # If repeat is None, the job should be scheduled + job = old_task_factory(self.TaskModelClass, scheduled_time=timezone.now() - timedelta(hours=1), repeat=None) + self.assertTrue(job.is_scheduled()) + + def test_clean(self): + job = old_task_factory(self.TaskModelClass) + job.queue = list(settings.QUEUES)[0] + job.callable = "scheduler.tests.jobs.test_job" + job.interval = 1 + job.result_ttl = -1 + self.assertIsNone(job.clean()) + + def test_clean_seconds(self): + job = old_task_factory(self.TaskModelClass) + job.queue = list(settings.QUEUES)[0] + job.callable = "scheduler.tests.jobs.test_job" + job.interval = 60 + job.result_ttl = -1 + job.interval_unit = "seconds" + self.assertIsNone(job.clean()) + + @override_settings( + SCHEDULER_CONFIG={ + "SCHEDULER_INTERVAL": 10, + } + ) + def test_clean_too_frequent(self): + job = old_task_factory(self.TaskModelClass) + job.queue = list(settings.QUEUES)[0] + job.callable = "scheduler.tests.jobs.test_job" + job.interval = 2 # Smaller than 10 + job.result_ttl = -1 + job.interval_unit = "seconds" + with self.assertRaises(ValidationError): + job.clean_interval_unit() + + def test_clean_not_multiple(self): + job = old_task_factory(self.TaskModelClass) + job.queue = list(settings.QUEUES)[0] + job.callable = "scheduler.tests.jobs.test_job" + job.interval = 121 + job.interval_unit = "seconds" + with self.assertRaises(ValidationError): + job.clean_interval_unit() + + def test_clean_short_result_ttl(self): + job = old_task_factory(self.TaskModelClass) + job.queue = list(settings.QUEUES)[0] + job.callable = "scheduler.tests.jobs.test_job" + job.interval = 1 + job.repeat = 1 + job.result_ttl = 3599 + job.interval_unit = "hours" + job.repeat = 42 + with self.assertRaises(ValidationError): + job.clean_result_ttl() + + def test_clean_indefinite_result_ttl(self): + job = old_task_factory(self.TaskModelClass) + job.queue = list(settings.QUEUES)[0] + job.callable = "scheduler.tests.jobs.test_job" + job.interval = 1 + job.result_ttl = -1 + job.interval_unit = "hours" + job.clean_result_ttl() + + def test_clean_undefined_result_ttl(self): + job = old_task_factory(self.TaskModelClass) + job.queue = list(settings.QUEUES)[0] + job.callable = "scheduler.tests.jobs.test_job" + job.interval = 1 + job.interval_unit = "hours" + job.clean_result_ttl() + + def test_interval_seconds_weeks(self): + job = old_task_factory(self.TaskModelClass, interval=2, interval_unit="weeks") + self.assertEqual(1209600.0, job.interval_seconds()) + + def test_interval_seconds_days(self): + job = old_task_factory(self.TaskModelClass, interval=2, interval_unit="days") + self.assertEqual(172800.0, job.interval_seconds()) + + def test_interval_seconds_hours(self): + job = old_task_factory(self.TaskModelClass, interval=2, interval_unit="hours") + self.assertEqual(7200.0, job.interval_seconds()) + + def test_interval_seconds_minutes(self): + job = old_task_factory(self.TaskModelClass, interval=15, interval_unit="minutes") + self.assertEqual(900.0, job.interval_seconds()) + + def test_interval_seconds_seconds(self): + job = RepeatableTask(interval=15, interval_unit="seconds") + self.assertEqual(15.0, job.interval_seconds()) + + def test_interval_display(self): + job = old_task_factory(self.TaskModelClass, interval=15, interval_unit="minutes") + self.assertEqual(job.interval_display(), "15 minutes") + + def test_result_interval(self): + job = old_task_factory( + self.TaskModelClass, + ) + entry = _get_job_from_scheduled_registry(job) + self.assertEqual(entry.meta["interval"], 3600) + + def test_repeat(self): + job = old_task_factory(self.TaskModelClass, repeat=10) + entry = _get_job_from_scheduled_registry(job) + self.assertEqual(entry.meta["repeat"], 10) + + def test_repeat_old_job_exhausted(self): + base_time = timezone.now() + job = old_task_factory(self.TaskModelClass, scheduled_time=base_time - timedelta(hours=10), repeat=10) + self.assertEqual(job.is_scheduled(), False) + + def test_repeat_old_job_last_iter(self): + base_time = timezone.now() + job = old_task_factory(self.TaskModelClass, scheduled_time=base_time - timedelta(hours=9, minutes=30), repeat=10) + self.assertEqual(job.repeat, 0) + self.assertEqual(job.is_scheduled(), True) + + def test_repeat_old_job_remaining(self): + base_time = timezone.now() + job = old_task_factory(self.TaskModelClass, scheduled_time=base_time - timedelta(minutes=30), repeat=5) + self.assertEqual(job.repeat, 4) + self.assertEqual(job.scheduled_time, base_time + timedelta(minutes=30)) + self.assertEqual(job.is_scheduled(), True) + + def test_repeat_none_interval_2_min(self): + base_time = timezone.now() + job = old_task_factory(self.TaskModelClass, scheduled_time=base_time - timedelta(minutes=29), repeat=None) + job.interval = 120 + job.interval_unit = "seconds" + job.schedule() + self.assertTrue(job.scheduled_time > base_time) + self.assertTrue(job.is_scheduled()) + + def test_check_rescheduled_after_execution(self): + task = old_task_factory(self.TaskModelClass, scheduled_time=timezone.now() + timedelta(seconds=1), repeat=10) + queue = task.rqueue + first_run_id = task.job_id + entry = queue.fetch_job(first_run_id) + queue.run_sync(entry) + task.refresh_from_db() + self.assertEqual(task.failed_runs, 0) + self.assertIsNone(task.last_failed_run) + self.assertEqual(task.successful_runs, 1) + self.assertIsNotNone(task.last_successful_run) + self.assertTrue(task.is_scheduled()) + self.assertNotEqual(task.job_id, first_run_id) + + def test_check_rescheduled_after_execution_failed_job(self): + task = old_task_factory( + self.TaskModelClass, + callable_name="scheduler.tests.jobs.failing_job", + scheduled_time=timezone.now() + timedelta(seconds=1), + repeat=10, + ) + queue = task.rqueue + first_run_id = task.job_id + entry = queue.fetch_job(first_run_id) + queue.run_sync(entry) + task.refresh_from_db() + self.assertEqual(task.failed_runs, 1) + self.assertIsNotNone(task.last_failed_run) + self.assertEqual(task.successful_runs, 0) + self.assertIsNone(task.last_successful_run) + self.assertTrue(task.is_scheduled()) + self.assertNotEqual(task.job_id, first_run_id) + + def test_check_not_rescheduled_after_last_repeat(self): + task = old_task_factory( + self.TaskModelClass, + scheduled_time=timezone.now() + timedelta(seconds=1), + repeat=1, + ) + queue = task.rqueue + first_run_id = task.job_id + entry = queue.fetch_job(first_run_id) + queue.run_sync(entry) + task.refresh_from_db() + self.assertEqual(task.failed_runs, 0) + self.assertIsNone(task.last_failed_run) + self.assertEqual(task.successful_runs, 1) + self.assertIsNotNone(task.last_successful_run) + self.assertNotEqual(task.job_id, first_run_id) diff --git a/scheduler/tests/test_old_task_model.py b/scheduler/tests/test_old_models/test_old_task_model.py similarity index 98% rename from scheduler/tests/test_old_task_model.py rename to scheduler/tests/test_old_models/test_old_task_model.py index 8e04b45..274401f 100644 --- a/scheduler/tests/test_old_task_model.py +++ b/scheduler/tests/test_old_models/test_old_task_model.py @@ -11,15 +11,10 @@ from scheduler import settings from scheduler.models import BaseTask, TaskArg, TaskKwarg, ScheduledTask from scheduler.tools import run_task, create_worker -from . import jobs -from .testtools import ( - old_task_factory, - taskarg_factory, - _get_job_from_scheduled_registry, - SchedulerBaseCase, - _get_executions, -) -from ..queues import get_queue +from scheduler.tests import jobs +from scheduler.tests.testtools import old_task_factory, taskarg_factory, _get_job_from_scheduled_registry, \ + SchedulerBaseCase, _get_executions +from scheduler.queues import get_queue def assert_response_has_msg(response, message): @@ -512,7 +507,7 @@ def test_admin_delete_selected(self): scheduled_jobs = queue.scheduled_job_registry.get_job_ids() self.assertNotIn(job_id, scheduled_jobs) - class TestSchedulableJob(TestBaseTask): + class TestSchedulableTask(TestBaseTask): # Currently ScheduledJob and RepeatableJob TaskModelClass = ScheduledTask @@ -537,7 +532,7 @@ def test_result_ttl_passthrough(self): self.assertEqual(entry.result_ttl, 500) -class TestScheduledJob(BaseTestCases.TestSchedulableJob): +class TestScheduledJob(BaseTestCases.TestSchedulableTask): TaskModelClass = ScheduledTask def test_clean(self): diff --git a/scheduler/tests/test_task_types/__init__.py b/scheduler/tests/test_task_types/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scheduler/tests/test_task_types/test_cron_task.py b/scheduler/tests/test_task_types/test_cron_task.py new file mode 100644 index 0000000..a7d2a7a --- /dev/null +++ b/scheduler/tests/test_task_types/test_cron_task.py @@ -0,0 +1,78 @@ +from django.core.exceptions import ValidationError + +from scheduler import settings +from scheduler.queues import get_queue +from scheduler.tests.test_task_types.test_task_model import BaseTestCases +from scheduler.tests.testtools import task_factory +from scheduler.tools import create_worker, TaskType + + +class TestCronTask(BaseTestCases.TestBaseTask): + task_type = TaskType.CRON + + def test_clean(self): + task = task_factory(self.task_type) + task.cron_string = "* * * * *" + task.queue = list(settings.QUEUES)[0] + task.callable = "scheduler.tests.jobs.test_job" + self.assertIsNone(task.clean()) + + def test_clean_cron_string_invalid(self): + task = task_factory(self.task_type) + task.cron_string = "not-a-cron-string" + task.queue = list(settings.QUEUES)[0] + task.callable = "scheduler.tests.jobs.test_job" + with self.assertRaises(ValidationError): + task.clean_cron_string() + + def test_check_rescheduled_after_execution(self): + task = task_factory(self.task_type) + queue = task.rqueue + first_run_id = task.job_id + entry = queue.fetch_job(first_run_id) + queue.run_sync(entry) + task.refresh_from_db() + self.assertEqual(task.failed_runs, 0) + self.assertIsNone(task.last_failed_run) + self.assertEqual(task.successful_runs, 1) + self.assertIsNotNone(task.last_successful_run) + self.assertTrue(task.is_scheduled()) + self.assertNotEqual(task.job_id, first_run_id) + + def test_check_rescheduled_after_failed_execution(self): + task = task_factory( + self.task_type, + callable_name="scheduler.tests.jobs.scheduler.tests.jobs.test_job", + ) + queue = task.rqueue + first_run_id = task.job_id + entry = queue.fetch_job(first_run_id) + queue.run_sync(entry) + task.refresh_from_db() + self.assertEqual(task.failed_runs, 1) + self.assertIsNotNone(task.last_failed_run) + self.assertEqual(task.successful_runs, 0) + self.assertIsNone(task.last_successful_run) + self.assertTrue(task.is_scheduled()) + self.assertNotEqual(task.job_id, first_run_id) + + def test_cron_task_enqueuing_jobs(self): + queue = get_queue() + prev_queued = len(queue.scheduled_job_registry) + prev_finished = len(queue.finished_job_registry) + task = task_factory(self.task_type, callable_name="scheduler.tests.jobs.enqueue_jobs") + self.assertEqual(prev_queued + 1, len(queue.scheduled_job_registry)) + first_run_id = task.job_id + entry = queue.fetch_job(first_run_id) + queue.run_sync(entry) + self.assertEqual(20, len(queue)) + self.assertEqual(prev_finished + 1, len(queue.finished_job_registry)) + worker = create_worker( + "default", + fork_job_execution=False, + ) + worker.work(burst=True) + self.assertEqual(prev_finished + 21, len(queue.finished_job_registry)) + worker.refresh() + self.assertEqual(20, worker.successful_job_count) + self.assertEqual(0, worker.failed_job_count) diff --git a/scheduler/tests/test_repeatable_task.py b/scheduler/tests/test_task_types/test_repeatable_task.py similarity index 76% rename from scheduler/tests/test_repeatable_task.py rename to scheduler/tests/test_task_types/test_repeatable_task.py index e29291e..2836f07 100644 --- a/scheduler/tests/test_repeatable_task.py +++ b/scheduler/tests/test_task_types/test_repeatable_task.py @@ -6,25 +6,25 @@ from scheduler import settings from scheduler.models import RepeatableTask -from scheduler.tests.test_old_models import BaseTestCases -from .testtools import task_factory, _get_job_from_scheduled_registry -from ..tools import TaskType +from scheduler.tests.testtools import task_factory, _get_job_from_scheduled_registry +from scheduler.tools import TaskType +from scheduler.tests.test_task_types.test_task_model import BaseTestCases -class TestRepeatableTask(BaseTestCases.TestSchedulableJob): - TaskModelClass = TaskType.REPEATABLE +class TestRepeatableTask(BaseTestCases.TestSchedulableTask): + task_type = TaskType.REPEATABLE def test_unschedulable_old_job(self): - job = task_factory(self.TaskModelClass, scheduled_time=timezone.now() - timedelta(hours=1), repeat=0) + job = task_factory(self.task_type, scheduled_time=timezone.now() - timedelta(hours=1), repeat=0) self.assertFalse(job.is_scheduled()) def test_schedulable_old_job_repeat_none(self): # If repeat is None, the job should be scheduled - job = task_factory(self.TaskModelClass, scheduled_time=timezone.now() - timedelta(hours=1), repeat=None) + job = task_factory(self.task_type, scheduled_time=timezone.now() - timedelta(hours=1), repeat=None) self.assertTrue(job.is_scheduled()) def test_clean(self): - job = task_factory(self.TaskModelClass) + job = task_factory(self.task_type) job.queue = list(settings.QUEUES)[0] job.callable = "scheduler.tests.jobs.test_job" job.interval = 1 @@ -32,7 +32,7 @@ def test_clean(self): self.assertIsNone(job.clean()) def test_clean_seconds(self): - job = task_factory(self.TaskModelClass) + job = task_factory(self.task_type) job.queue = list(settings.QUEUES)[0] job.callable = "scheduler.tests.jobs.test_job" job.interval = 60 @@ -46,7 +46,7 @@ def test_clean_seconds(self): } ) def test_clean_too_frequent(self): - job = task_factory(self.TaskModelClass) + job = task_factory(self.task_type) job.queue = list(settings.QUEUES)[0] job.callable = "scheduler.tests.jobs.test_job" job.interval = 2 # Smaller than 10 @@ -56,7 +56,7 @@ def test_clean_too_frequent(self): job.clean_interval_unit() def test_clean_not_multiple(self): - job = task_factory(self.TaskModelClass) + job = task_factory(self.task_type) job.queue = list(settings.QUEUES)[0] job.callable = "scheduler.tests.jobs.test_job" job.interval = 121 @@ -65,7 +65,7 @@ def test_clean_not_multiple(self): job.clean_interval_unit() def test_clean_short_result_ttl(self): - job = task_factory(self.TaskModelClass) + job = task_factory(self.task_type) job.queue = list(settings.QUEUES)[0] job.callable = "scheduler.tests.jobs.test_job" job.interval = 1 @@ -77,7 +77,7 @@ def test_clean_short_result_ttl(self): job.clean_result_ttl() def test_clean_indefinite_result_ttl(self): - job = task_factory(self.TaskModelClass) + job = task_factory(self.task_type) job.queue = list(settings.QUEUES)[0] job.callable = "scheduler.tests.jobs.test_job" job.interval = 1 @@ -86,7 +86,7 @@ def test_clean_indefinite_result_ttl(self): job.clean_result_ttl() def test_clean_undefined_result_ttl(self): - job = task_factory(self.TaskModelClass) + job = task_factory(self.task_type) job.queue = list(settings.QUEUES)[0] job.callable = "scheduler.tests.jobs.test_job" job.interval = 1 @@ -94,19 +94,19 @@ def test_clean_undefined_result_ttl(self): job.clean_result_ttl() def test_interval_seconds_weeks(self): - job = task_factory(self.TaskModelClass, interval=2, interval_unit="weeks") + job = task_factory(self.task_type, interval=2, interval_unit="weeks") self.assertEqual(1209600.0, job.interval_seconds()) def test_interval_seconds_days(self): - job = task_factory(self.TaskModelClass, interval=2, interval_unit="days") + job = task_factory(self.task_type, interval=2, interval_unit="days") self.assertEqual(172800.0, job.interval_seconds()) def test_interval_seconds_hours(self): - job = task_factory(self.TaskModelClass, interval=2, interval_unit="hours") + job = task_factory(self.task_type, interval=2, interval_unit="hours") self.assertEqual(7200.0, job.interval_seconds()) def test_interval_seconds_minutes(self): - job = task_factory(self.TaskModelClass, interval=15, interval_unit="minutes") + job = task_factory(self.task_type, interval=15, interval_unit="minutes") self.assertEqual(900.0, job.interval_seconds()) def test_interval_seconds_seconds(self): @@ -114,42 +114,42 @@ def test_interval_seconds_seconds(self): self.assertEqual(15.0, job.interval_seconds()) def test_interval_display(self): - job = task_factory(self.TaskModelClass, interval=15, interval_unit="minutes") + job = task_factory(self.task_type, interval=15, interval_unit="minutes") self.assertEqual(job.interval_display(), "15 minutes") def test_result_interval(self): job = task_factory( - self.TaskModelClass, + self.task_type, ) entry = _get_job_from_scheduled_registry(job) self.assertEqual(entry.meta["interval"], 3600) def test_repeat(self): - job = task_factory(self.TaskModelClass, repeat=10) + job = task_factory(self.task_type, repeat=10) entry = _get_job_from_scheduled_registry(job) self.assertEqual(entry.meta["repeat"], 10) def test_repeat_old_job_exhausted(self): base_time = timezone.now() - job = task_factory(self.TaskModelClass, scheduled_time=base_time - timedelta(hours=10), repeat=10) + job = task_factory(self.task_type, scheduled_time=base_time - timedelta(hours=10), repeat=10) self.assertEqual(job.is_scheduled(), False) def test_repeat_old_job_last_iter(self): base_time = timezone.now() - job = task_factory(self.TaskModelClass, scheduled_time=base_time - timedelta(hours=9, minutes=30), repeat=10) + job = task_factory(self.task_type, scheduled_time=base_time - timedelta(hours=9, minutes=30), repeat=10) self.assertEqual(job.repeat, 0) self.assertEqual(job.is_scheduled(), True) def test_repeat_old_job_remaining(self): base_time = timezone.now() - job = task_factory(self.TaskModelClass, scheduled_time=base_time - timedelta(minutes=30), repeat=5) + job = task_factory(self.task_type, scheduled_time=base_time - timedelta(minutes=30), repeat=5) self.assertEqual(job.repeat, 4) self.assertEqual(job.scheduled_time, base_time + timedelta(minutes=30)) self.assertEqual(job.is_scheduled(), True) def test_repeat_none_interval_2_min(self): base_time = timezone.now() - job = task_factory(self.TaskModelClass, scheduled_time=base_time - timedelta(minutes=29), repeat=None) + job = task_factory(self.task_type, scheduled_time=base_time - timedelta(minutes=29), repeat=None) job.interval = 120 job.interval_unit = "seconds" job.schedule() @@ -157,7 +157,7 @@ def test_repeat_none_interval_2_min(self): self.assertTrue(job.is_scheduled()) def test_check_rescheduled_after_execution(self): - task = task_factory(self.TaskModelClass, scheduled_time=timezone.now() + timedelta(seconds=1), repeat=10) + task = task_factory(self.task_type, scheduled_time=timezone.now() + timedelta(seconds=1), repeat=10) queue = task.rqueue first_run_id = task.job_id entry = queue.fetch_job(first_run_id) @@ -172,7 +172,7 @@ def test_check_rescheduled_after_execution(self): def test_check_rescheduled_after_execution_failed_job(self): task = task_factory( - self.TaskModelClass, + self.task_type, callable_name="scheduler.tests.jobs.failing_job", scheduled_time=timezone.now() + timedelta(seconds=1), repeat=10, @@ -191,7 +191,7 @@ def test_check_rescheduled_after_execution_failed_job(self): def test_check_not_rescheduled_after_last_repeat(self): task = task_factory( - self.TaskModelClass, + self.task_type, scheduled_time=timezone.now() + timedelta(seconds=1), repeat=1, ) diff --git a/scheduler/tests/test_task_model.py b/scheduler/tests/test_task_types/test_task_model.py similarity index 98% rename from scheduler/tests/test_task_model.py rename to scheduler/tests/test_task_types/test_task_model.py index def647d..c8727fa 100644 --- a/scheduler/tests/test_task_model.py +++ b/scheduler/tests/test_task_types/test_task_model.py @@ -10,17 +10,13 @@ from scheduler import settings from scheduler.models import Task, TaskArg, TaskKwarg +from scheduler.models.task import TaskType +from scheduler.queues import get_queue +from scheduler.tests import jobs +from scheduler.tests.testtools import ( + task_factory, taskarg_factory, _get_job_from_scheduled_registry, + SchedulerBaseCase, _get_executions, ) from scheduler.tools import run_task, create_worker -from . import jobs -from .testtools import ( - task_factory, - taskarg_factory, - _get_job_from_scheduled_registry, - SchedulerBaseCase, - _get_executions, -) -from ..models.task import TaskType -from ..queues import get_queue def assert_response_has_msg(response, message): @@ -487,7 +483,7 @@ def test_admin_delete_selected(self): scheduled_jobs = queue.scheduled_job_registry.get_job_ids() self.assertNotIn(job_id, scheduled_jobs) - class TestSchedulableJob(TestBaseTask): + class TestSchedulableTask(TestBaseTask): # Currently ScheduledJob and RepeatableJob task_type = TaskType.ONCE @@ -512,7 +508,7 @@ def test_result_ttl_passthrough(self): self.assertEqual(entry.result_ttl, 500) -class TestScheduledJob(BaseTestCases.TestSchedulableJob): +class TestScheduledTask(BaseTestCases.TestSchedulableTask): task_type = TaskType.ONCE def test_clean(self): From be603faf1a4d0204352f13cf27263d0b564d54d5 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 19 Nov 2024 15:52:27 -0500 Subject: [PATCH 20/47] wip --- .../tests/test_old_models/test_old_models.py | 18 +++++++++--------- .../test_old_repeatable_task.py | 6 +++--- .../test_old_models/test_old_task_model.py | 18 +++++++++--------- .../test_task_types/test_repeatable_task.py | 6 +++--- .../tests/test_task_types/test_task_model.py | 18 +++++++++--------- scheduler/tests/test_views.py | 4 ++-- scheduler/tests/testtools.py | 2 +- 7 files changed, 36 insertions(+), 36 deletions(-) diff --git a/scheduler/tests/test_old_models/test_old_models.py b/scheduler/tests/test_old_models/test_old_models.py index 38696bd..e8a04a0 100644 --- a/scheduler/tests/test_old_models/test_old_models.py +++ b/scheduler/tests/test_old_models/test_old_models.py @@ -13,7 +13,7 @@ from scheduler.queues import get_queue from scheduler.tests import jobs from scheduler.tests.testtools import ( - old_task_factory, taskarg_factory, _get_job_from_scheduled_registry, SchedulerBaseCase, _get_executions) + old_task_factory, taskarg_factory, _get_task_job_execution_from_registry, SchedulerBaseCase, _get_executions) from scheduler.tools import run_task, create_worker @@ -174,7 +174,7 @@ def test_str(self): def test_callable_passthrough(self): task = old_task_factory(self.TaskModelClass) - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) self.assertEqual(entry.func, run_task) job_model, job_id = entry.args self.assertEqual(job_model, self.TaskModelClass.__name__) @@ -182,7 +182,7 @@ def test_callable_passthrough(self): def test_timeout_passthrough(self): task = old_task_factory(self.TaskModelClass, timeout=500) - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) self.assertEqual(entry.timeout, 500) def test_at_front_passthrough(self): @@ -195,12 +195,12 @@ def test_callable_result(self): task = old_task_factory( self.TaskModelClass, ) - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) self.assertEqual(entry.perform(), 2) def test_callable_empty_args_and_kwargs(self): task = old_task_factory(self.TaskModelClass, callable="scheduler.tests.jobs.test_args_kwargs") - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) self.assertEqual(entry.perform(), "test_args_kwargs()") def test_delete_args(self): @@ -253,7 +253,7 @@ def test_callable_args_and_kwargs(self): taskarg_factory(TaskKwarg, key="key2", arg_type="datetime", val=date, content_object=task) taskarg_factory(TaskKwarg, key="key3", arg_type="bool", val=False, content_object=task) task.save() - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) self.assertEqual(entry.perform(), "test_args_kwargs('one', key1=2, key2={}, key3=False)".format(date)) def test_function_string(self): @@ -385,7 +385,7 @@ def test_admin_enqueue_job_now(self): # assert part 1 self.assertEqual(200, res.status_code) - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) task_model, scheduled_task_id = entry.args self.assertEqual(task_model, task.task_type) self.assertEqual(scheduled_task_id, task.id) @@ -400,7 +400,7 @@ def test_admin_enqueue_job_now(self): worker.work(burst=True) # assert 2 - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) self.assertEqual(task_model, task.task_type) self.assertEqual(scheduled_task_id, task.id) assert_has_execution_with_status(task, "finished") @@ -526,7 +526,7 @@ def test_schedule_time_with_tz(self): def test_result_ttl_passthrough(self): job = old_task_factory(self.TaskModelClass, result_ttl=500) - entry = _get_job_from_scheduled_registry(job) + entry = _get_task_job_execution_from_registry(job) self.assertEqual(entry.result_ttl, 500) diff --git a/scheduler/tests/test_old_models/test_old_repeatable_task.py b/scheduler/tests/test_old_models/test_old_repeatable_task.py index 5e4c4b8..60591cb 100644 --- a/scheduler/tests/test_old_models/test_old_repeatable_task.py +++ b/scheduler/tests/test_old_models/test_old_repeatable_task.py @@ -8,7 +8,7 @@ from scheduler.models import RepeatableTask from scheduler.tests.test_old_models.test_old_task_model import BaseTestCases -from scheduler.tests.testtools import old_task_factory, _get_job_from_scheduled_registry +from scheduler.tests.testtools import old_task_factory, _get_task_job_execution_from_registry class TestRepeatableTask(BaseTestCases.TestSchedulableTask): @@ -121,12 +121,12 @@ def test_result_interval(self): job = old_task_factory( self.TaskModelClass, ) - entry = _get_job_from_scheduled_registry(job) + entry = _get_task_job_execution_from_registry(job) self.assertEqual(entry.meta["interval"], 3600) def test_repeat(self): job = old_task_factory(self.TaskModelClass, repeat=10) - entry = _get_job_from_scheduled_registry(job) + entry = _get_task_job_execution_from_registry(job) self.assertEqual(entry.meta["repeat"], 10) def test_repeat_old_job_exhausted(self): diff --git a/scheduler/tests/test_old_models/test_old_task_model.py b/scheduler/tests/test_old_models/test_old_task_model.py index 274401f..194b0ab 100644 --- a/scheduler/tests/test_old_models/test_old_task_model.py +++ b/scheduler/tests/test_old_models/test_old_task_model.py @@ -12,7 +12,7 @@ from scheduler.models import BaseTask, TaskArg, TaskKwarg, ScheduledTask from scheduler.tools import run_task, create_worker from scheduler.tests import jobs -from scheduler.tests.testtools import old_task_factory, taskarg_factory, _get_job_from_scheduled_registry, \ +from scheduler.tests.testtools import old_task_factory, taskarg_factory, _get_task_job_execution_from_registry, \ SchedulerBaseCase, _get_executions from scheduler.queues import get_queue @@ -174,7 +174,7 @@ def test_str(self): def test_callable_passthrough(self): task = old_task_factory(self.TaskModelClass) - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) self.assertEqual(entry.func, run_task) job_model, job_id = entry.args self.assertEqual(job_model, self.TaskModelClass.__name__) @@ -182,7 +182,7 @@ def test_callable_passthrough(self): def test_timeout_passthrough(self): task = old_task_factory(self.TaskModelClass, timeout=500) - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) self.assertEqual(entry.timeout, 500) def test_at_front_passthrough(self): @@ -195,12 +195,12 @@ def test_callable_result(self): task = old_task_factory( self.TaskModelClass, ) - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) self.assertEqual(entry.perform(), 2) def test_callable_empty_args_and_kwargs(self): task = old_task_factory(self.TaskModelClass, callable="scheduler.tests.jobs.test_args_kwargs") - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) self.assertEqual(entry.perform(), "test_args_kwargs()") def test_delete_args(self): @@ -253,7 +253,7 @@ def test_callable_args_and_kwargs(self): taskarg_factory(TaskKwarg, key="key2", arg_type="datetime", val=date, content_object=task) taskarg_factory(TaskKwarg, key="key3", arg_type="bool", val=False, content_object=task) task.save() - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) self.assertEqual(entry.perform(), "test_args_kwargs('one', key1=2, key2={}, key3=False)".format(date)) def test_function_string(self): @@ -387,7 +387,7 @@ def test_admin_enqueue_job_now(self): # assert part 1 self.assertEqual(200, res.status_code) - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) task_model, scheduled_task_id = entry.args self.assertEqual(task_model, task.task_type) self.assertEqual(scheduled_task_id, task.id) @@ -402,7 +402,7 @@ def test_admin_enqueue_job_now(self): worker.work(burst=True) # assert 2 - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) self.assertEqual(task_model, task.task_type) self.assertEqual(scheduled_task_id, task.id) assert_has_execution_with_status(task, "finished") @@ -528,7 +528,7 @@ def test_schedule_time_with_tz(self): def test_result_ttl_passthrough(self): job = old_task_factory(self.TaskModelClass, result_ttl=500) - entry = _get_job_from_scheduled_registry(job) + entry = _get_task_job_execution_from_registry(job) self.assertEqual(entry.result_ttl, 500) diff --git a/scheduler/tests/test_task_types/test_repeatable_task.py b/scheduler/tests/test_task_types/test_repeatable_task.py index 2836f07..ba73c97 100644 --- a/scheduler/tests/test_task_types/test_repeatable_task.py +++ b/scheduler/tests/test_task_types/test_repeatable_task.py @@ -6,7 +6,7 @@ from scheduler import settings from scheduler.models import RepeatableTask -from scheduler.tests.testtools import task_factory, _get_job_from_scheduled_registry +from scheduler.tests.testtools import task_factory, _get_task_job_execution_from_registry from scheduler.tools import TaskType from scheduler.tests.test_task_types.test_task_model import BaseTestCases @@ -121,12 +121,12 @@ def test_result_interval(self): job = task_factory( self.task_type, ) - entry = _get_job_from_scheduled_registry(job) + entry = _get_task_job_execution_from_registry(job) self.assertEqual(entry.meta["interval"], 3600) def test_repeat(self): job = task_factory(self.task_type, repeat=10) - entry = _get_job_from_scheduled_registry(job) + entry = _get_task_job_execution_from_registry(job) self.assertEqual(entry.meta["repeat"], 10) def test_repeat_old_job_exhausted(self): diff --git a/scheduler/tests/test_task_types/test_task_model.py b/scheduler/tests/test_task_types/test_task_model.py index c8727fa..d81fdaa 100644 --- a/scheduler/tests/test_task_types/test_task_model.py +++ b/scheduler/tests/test_task_types/test_task_model.py @@ -14,7 +14,7 @@ from scheduler.queues import get_queue from scheduler.tests import jobs from scheduler.tests.testtools import ( - task_factory, taskarg_factory, _get_job_from_scheduled_registry, + task_factory, taskarg_factory, _get_task_job_execution_from_registry, SchedulerBaseCase, _get_executions, ) from scheduler.tools import run_task, create_worker @@ -166,7 +166,7 @@ def test_str(self): def test_callable_passthrough(self): task = task_factory(self.task_type) - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) self.assertEqual(entry.func, run_task) job_model, job_id = entry.args self.assertEqual(job_model, self.task_type.value) @@ -174,7 +174,7 @@ def test_callable_passthrough(self): def test_timeout_passthrough(self): task = task_factory(self.task_type, timeout=500) - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) self.assertEqual(entry.timeout, 500) def test_at_front_passthrough(self): @@ -185,12 +185,12 @@ def test_at_front_passthrough(self): def test_callable_result(self): task = task_factory(self.task_type) - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) self.assertEqual(entry.perform(), 2) def test_callable_empty_args_and_kwargs(self): task = task_factory(self.task_type, callable="scheduler.tests.jobs.test_args_kwargs") - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) self.assertEqual(entry.perform(), "test_args_kwargs()") def test_delete_args(self): @@ -235,7 +235,7 @@ def test_callable_args_and_kwargs(self): taskarg_factory(TaskKwarg, key="key2", arg_type="datetime", val=date, content_object=task) taskarg_factory(TaskKwarg, key="key3", arg_type="bool", val=False, content_object=task) task.save() - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) self.assertEqual(entry.perform(), "test_args_kwargs('one', key1=2, key2={}, key3=False)".format(date)) def test_function_string(self): @@ -363,7 +363,7 @@ def test_admin_enqueue_job_now(self): # assert part 1 self.assertEqual(200, res.status_code) - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) task_model, scheduled_task_id = entry.args self.assertEqual(task_model, task.task_type) self.assertEqual(scheduled_task_id, task.id) @@ -378,7 +378,7 @@ def test_admin_enqueue_job_now(self): worker.work(burst=True) # assert 2 - entry = _get_job_from_scheduled_registry(task) + entry = _get_task_job_execution_from_registry(task) self.assertEqual(task_model, task.task_type) self.assertEqual(scheduled_task_id, task.id) assert_has_execution_with_status(task, "finished") @@ -504,7 +504,7 @@ def test_schedule_time_with_tz(self): def test_result_ttl_passthrough(self): job = task_factory(self.task_type, result_ttl=500) - entry = _get_job_from_scheduled_registry(job) + entry = _get_task_job_execution_from_registry(job) self.assertEqual(entry.result_ttl, 500) diff --git a/scheduler/tests/test_views.py b/scheduler/tests/test_views.py index 930cb9e..5d6f227 100644 --- a/scheduler/tests/test_views.py +++ b/scheduler/tests/test_views.py @@ -11,7 +11,7 @@ from scheduler.rq_classes import JobExecution, ExecutionStatus from scheduler.tests import test_settings # noqa from scheduler.tests.jobs import failing_job, long_job, test_job -from scheduler.tests.testtools import assert_message_in_response, task_factory, _get_job_from_scheduled_registry +from scheduler.tests.testtools import assert_message_in_response, task_factory, _get_task_job_execution_from_registry from scheduler.tools import create_worker, TaskType @@ -358,7 +358,7 @@ def test_job_details(self): def test_scheduled_job_details(self): """Job data is displayed properly""" scheduled_job = task_factory(TaskType.ONCE, enabled=True) - job = _get_job_from_scheduled_registry(scheduled_job) + job = _get_task_job_execution_from_registry(scheduled_job) url = reverse( "job_details", diff --git a/scheduler/tests/testtools.py b/scheduler/tests/testtools.py index 0347139..8548486 100644 --- a/scheduler/tests/testtools.py +++ b/scheduler/tests/testtools.py @@ -128,7 +128,7 @@ def taskarg_factory(cls, **kwargs): return instance -def _get_job_from_scheduled_registry(django_task: BaseTask): +def _get_task_job_execution_from_registry(django_task: BaseTask): jobs_to_schedule = django_task.rqueue.scheduled_job_registry.get_job_ids() entry = next(i for i in jobs_to_schedule if i == django_task.job_id) return django_task.rqueue.fetch_job(entry) From 02c7871cb97de05fa36c8cc28b5afe3889dc7f70 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 19 Nov 2024 16:16:38 -0500 Subject: [PATCH 21/47] wip --- scheduler/management/commands/export.py | 11 ++++++-- scheduler/management/commands/import.py | 4 +-- scheduler/models/task.py | 8 ++++-- scheduler/rq_classes.py | 18 +++++++------ .../tests/test_task_types/test_once_task.py | 22 +++++++++++++++ .../tests/test_task_types/test_task_model.py | 13 --------- scheduler/tools.py | 27 +++++++++++++------ 7 files changed, 68 insertions(+), 35 deletions(-) create mode 100644 scheduler/tests/test_task_types/test_once_task.py diff --git a/scheduler/management/commands/export.py b/scheduler/management/commands/export.py index bb2b249..2babc9e 100644 --- a/scheduler/management/commands/export.py +++ b/scheduler/management/commands/export.py @@ -4,7 +4,9 @@ from django.apps import apps from django.core.management.base import BaseCommand -from scheduler.tools import MODEL_NAMES +from scheduler.models import Task +from scheduler.rq_classes import MODEL_NAMES +from scheduler.tools import OLD_MODEL_NAMES class Command(BaseCommand): @@ -43,7 +45,12 @@ def add_arguments(self, parser): def handle(self, *args, **options): file = open(options.get("filename"), "w") if options.get("filename") else sys.stdout res = list() - for model_name in MODEL_NAMES: + jobs = Task.objects.all() + if options.get("enabled"): + jobs = jobs.filter(enabled=True) + for job in jobs: + res.append(job.to_dict()) + for model_name in OLD_MODEL_NAMES: model = apps.get_model(app_label="scheduler", model_name=model_name) jobs = model.objects.all() if options.get("enabled"): diff --git a/scheduler/management/commands/import.py b/scheduler/management/commands/import.py index 60967c2..8b42f67 100644 --- a/scheduler/management/commands/import.py +++ b/scheduler/management/commands/import.py @@ -10,7 +10,7 @@ from scheduler.models import TaskArg, TaskKwarg, Task from scheduler.models.task import TaskType -from scheduler.tools import MODEL_NAMES +from scheduler.tools import OLD_MODEL_NAMES def job_model_str(model_str: str) -> str: @@ -133,7 +133,7 @@ def handle(self, *args, **options): jobs = yaml.load(file, yaml.SafeLoader) if options.get("reset"): - for model_name in MODEL_NAMES: + for model_name in OLD_MODEL_NAMES: model = apps.get_model(app_label="scheduler", model_name=model_name) model.objects.all().delete() diff --git a/scheduler/models/task.py b/scheduler/models/task.py index 00928b9..e854d0b 100644 --- a/scheduler/models/task.py +++ b/scheduler/models/task.py @@ -51,9 +51,13 @@ def success_callback(job, connection, result, *args, **kwargs): model_name = job.meta.get("task_type", None) if model_name is None: return - model = apps.get_model(app_label="scheduler", model_name=model_name) - task = model.objects.filter(job_id=job.id).first() + + task = Task.objects.filter(job_id=job.id).first() if task is None: + model = apps.get_model(app_label="scheduler", model_name=model_name) + task = model.objects.filter(job_id=job.id).first() + if task is None: + logger.warn(f"Could not find task for job {job.id}") return task.job_id = None task.successful_runs += 1 diff --git a/scheduler/rq_classes.py b/scheduler/rq_classes.py index 8c15e48..d4acf50 100644 --- a/scheduler/rq_classes.py +++ b/scheduler/rq_classes.py @@ -24,7 +24,8 @@ from scheduler import settings from scheduler.broker_types import PipelineType, ConnectionType -MODEL_NAMES = ["ScheduledTask", "RepeatableTask", "CronTask", "Task"] +OLD_MODEL_NAMES = ["ScheduledTask", "RepeatableTask", "CronTask"] +MODEL_NAMES = ["OnceTask", "RepeatableTask", "CronTask"] rq_job_decorator = job ExecutionStatus = JobStatus @@ -63,7 +64,8 @@ def is_scheduled_task(self) -> bool: def is_execution_of(self, task: "Task") -> bool: # noqa: F821 return ( - self.meta.get("task_type", None) == task.task_type and self.meta.get("scheduled_task_id", None) == task.id + self.meta.get("task_type", None) == task.task_type and self.meta.get("scheduled_task_id", + None) == task.id ) def stop_execution(self, connection: ConnectionType): @@ -92,11 +94,11 @@ def __str__(self): return f"{self.name}/{','.join(self.queue_names())}" def _start_scheduler( - self, - burst: bool = False, - logging_level: str = "INFO", - date_format: str = "%H:%M:%S", - log_format: str = "%(asctime)s %(message)s", + self, + burst: bool = False, + logging_level: str = "INFO", + date_format: str = "%H:%M:%S", + log_format: str = "%(asctime)s %(message)s", ) -> None: """Starts the scheduler process. This is specifically designed to be run by the worker when running the `work()` method. @@ -262,7 +264,7 @@ def __init__(self, *args, **kwargs) -> None: @staticmethod def reschedule_all_jobs(): - for model_name in MODEL_NAMES: + for model_name in OLD_MODEL_NAMES: model = apps.get_model(app_label="scheduler", model_name=model_name) enabled_jobs = model.objects.filter(enabled=True) unscheduled_jobs = filter(lambda j: j.ready_for_schedule(), enabled_jobs) diff --git a/scheduler/tests/test_task_types/test_once_task.py b/scheduler/tests/test_task_types/test_once_task.py new file mode 100644 index 0000000..f9b686c --- /dev/null +++ b/scheduler/tests/test_task_types/test_once_task.py @@ -0,0 +1,22 @@ +from datetime import timedelta + +from django.utils import timezone + +from scheduler import settings +from scheduler.models.task import TaskType +from scheduler.tests.test_task_types.test_task_model import BaseTestCases +from scheduler.tests.testtools import task_factory + + +class TestScheduledTask(BaseTestCases.TestSchedulableTask): + task_type = TaskType.ONCE + + def test_clean(self): + job = task_factory(self.task_type) + job.queue = list(settings.QUEUES)[0] + job.callable = "scheduler.tests.jobs.test_job" + self.assertIsNone(job.clean()) + + def test_unschedulable_old_job(self): + job = task_factory(self.task_type, scheduled_time=timezone.now() - timedelta(hours=1)) + self.assertFalse(job.is_scheduled()) diff --git a/scheduler/tests/test_task_types/test_task_model.py b/scheduler/tests/test_task_types/test_task_model.py index d81fdaa..5f0a3ad 100644 --- a/scheduler/tests/test_task_types/test_task_model.py +++ b/scheduler/tests/test_task_types/test_task_model.py @@ -507,16 +507,3 @@ def test_result_ttl_passthrough(self): entry = _get_task_job_execution_from_registry(job) self.assertEqual(entry.result_ttl, 500) - -class TestScheduledTask(BaseTestCases.TestSchedulableTask): - task_type = TaskType.ONCE - - def test_clean(self): - job = task_factory(self.task_type) - job.queue = list(settings.QUEUES)[0] - job.callable = "scheduler.tests.jobs.test_job" - self.assertIsNone(job.clean()) - - def test_unschedulable_old_job(self): - job = task_factory(self.task_type, scheduled_time=timezone.now() - timedelta(hours=1)) - self.assertFalse(job.is_scheduled()) diff --git a/scheduler/tools.py b/scheduler/tools.py index 714cd9f..1af3cfe 100644 --- a/scheduler/tools.py +++ b/scheduler/tools.py @@ -10,7 +10,7 @@ from django.utils.translation import gettext_lazy as _ from scheduler.queues import get_queues, logger, get_queue -from scheduler.rq_classes import DjangoWorker, MODEL_NAMES, JobExecution +from scheduler.rq_classes import DjangoWorker, OLD_MODEL_NAMES, JobExecution, MODEL_NAMES from scheduler.settings import SCHEDULER_CONFIG, Broker @@ -39,15 +39,26 @@ def get_next_cron_time(cron_string: Optional[str]) -> Optional[timezone.datetime return next_itr -def get_scheduled_task(task_model: Union[TaskType, str], task_id: int) -> "BaseTask": # noqa: F821 - if isinstance(task_model, TaskType): - model = apps.get_model(app_label="scheduler", model_name="Task") - task = model.objects.filter(task_type=task_model, id=task_id).first() +def get_scheduled_task(task_model: str, task_id: int) -> "BaseTask": # noqa: F821 + if isinstance(task_model, str) and task_model not in OLD_MODEL_NAMES and task_model not in MODEL_NAMES: + raise ValueError(f"Job Model `{task_model}` does not exist, choices are {OLD_MODEL_NAMES}") + + # Try with new model names + model = apps.get_model(app_label="scheduler", model_name="Task") + if task_model == "OnceTask": + task = model.objects.filter(task_type=TaskType.ONCE, id=task_id).first() if task is None: raise ValueError(f"Job {task_model}:{task_id} does not exit") return task - if isinstance(task_model, str) and task_model not in MODEL_NAMES: - raise ValueError(f"Job Model `{task_model}` does not exist, choices are {MODEL_NAMES}") + elif task_model == "RepeatableTask": + task = model.objects.filter(task_type=TaskType.REPEATABLE, id=task_id).first() + if task is not None: + return task + elif task_model == "CronTask": + task = model.objects.filter(task_type=TaskType.CRON, id=task_id).first() + if task is not None: + return task + model = apps.get_model(app_label="scheduler", model_name=task_model) task = model.objects.filter(id=task_id).first() if task is None: @@ -81,7 +92,7 @@ def create_worker(*queue_names, **kwargs) -> DjangoWorker: queues = get_queues(*queue_names) existing_workers = DjangoWorker.all(connection=queues[0].connection) existing_worker_names = set(map(lambda w: w.name, existing_workers)) - kwargs["fork_job_execution"] = SCHEDULER_CONFIG.BROKER != Broker.FAKEREDIS + kwargs.setdefault("fork_job_execution", SCHEDULER_CONFIG.BROKER != Broker.FAKEREDIS) if kwargs.get("name", None) is None: kwargs["name"] = _calc_worker_name(existing_worker_names) From 49c0cda4b267beabbaa9348e6cb49c647ca66def Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 19 Nov 2024 17:38:49 -0500 Subject: [PATCH 22/47] wip --- scheduler/management/commands/export.py | 10 +--- scheduler/management/commands/import.py | 6 ++- scheduler/models/task.py | 8 ++-- scheduler/rq_classes.py | 6 +-- .../tests/test_mgmt_commands/test_export.py | 4 +- scheduler/tools.py | 46 ++++++++----------- 6 files changed, 36 insertions(+), 44 deletions(-) diff --git a/scheduler/management/commands/export.py b/scheduler/management/commands/export.py index 2babc9e..567cca2 100644 --- a/scheduler/management/commands/export.py +++ b/scheduler/management/commands/export.py @@ -5,8 +5,7 @@ from django.core.management.base import BaseCommand from scheduler.models import Task -from scheduler.rq_classes import MODEL_NAMES -from scheduler.tools import OLD_MODEL_NAMES +from scheduler.tools import MODEL_NAMES class Command(BaseCommand): @@ -45,12 +44,7 @@ def add_arguments(self, parser): def handle(self, *args, **options): file = open(options.get("filename"), "w") if options.get("filename") else sys.stdout res = list() - jobs = Task.objects.all() - if options.get("enabled"): - jobs = jobs.filter(enabled=True) - for job in jobs: - res.append(job.to_dict()) - for model_name in OLD_MODEL_NAMES: + for model_name in MODEL_NAMES: model = apps.get_model(app_label="scheduler", model_name=model_name) jobs = model.objects.all() if options.get("enabled"): diff --git a/scheduler/management/commands/import.py b/scheduler/management/commands/import.py index 8b42f67..a89ddc4 100644 --- a/scheduler/management/commands/import.py +++ b/scheduler/management/commands/import.py @@ -10,7 +10,7 @@ from scheduler.models import TaskArg, TaskKwarg, Task from scheduler.models.task import TaskType -from scheduler.tools import OLD_MODEL_NAMES +from scheduler.tools import MODEL_NAMES def job_model_str(model_str: str) -> str: @@ -21,6 +21,8 @@ def job_model_str(model_str: str) -> str: def get_task_type(model_str: str) -> TaskType: model_str = job_model_str(model_str) + if TaskType(model_str): + return TaskType(model_str) if model_str == "CronTask": return TaskType.CRON elif model_str == "RepeatableTask": @@ -133,7 +135,7 @@ def handle(self, *args, **options): jobs = yaml.load(file, yaml.SafeLoader) if options.get("reset"): - for model_name in OLD_MODEL_NAMES: + for model_name in MODEL_NAMES: model = apps.get_model(app_label="scheduler", model_name=model_name) model.objects.all().delete() diff --git a/scheduler/models/task.py b/scheduler/models/task.py index e854d0b..0848dc2 100644 --- a/scheduler/models/task.py +++ b/scheduler/models/task.py @@ -48,13 +48,13 @@ def failure_callback(job, connection, result, *args, **kwargs): def success_callback(job, connection, result, *args, **kwargs): - model_name = job.meta.get("task_type", None) - if model_name is None: + task_type = job.meta.get("task_type", None) + if task_type is None: return task = Task.objects.filter(job_id=job.id).first() if task is None: - model = apps.get_model(app_label="scheduler", model_name=model_name) + model = apps.get_model(app_label="scheduler", model_name=task_type) task = model.objects.filter(job_id=job.id).first() if task is None: logger.warn(f"Could not find task for job {job.id}") @@ -267,7 +267,7 @@ def ready_for_schedule(self) -> bool: if not self.enabled: logger.debug(f"Task {str(self)} disabled, enable task before scheduling") return False - if self.task_type == TaskType.REPEATABLE and self._schedule_time() < timezone.now(): + if self.task_type in {TaskType.REPEATABLE, TaskType.ONCE} and self._schedule_time() < timezone.now(): return False return True diff --git a/scheduler/rq_classes.py b/scheduler/rq_classes.py index d4acf50..8c536b0 100644 --- a/scheduler/rq_classes.py +++ b/scheduler/rq_classes.py @@ -24,8 +24,8 @@ from scheduler import settings from scheduler.broker_types import PipelineType, ConnectionType -OLD_MODEL_NAMES = ["ScheduledTask", "RepeatableTask", "CronTask"] -MODEL_NAMES = ["OnceTask", "RepeatableTask", "CronTask"] +MODEL_NAMES = ["ScheduledTask", "RepeatableTask", "CronTask", "Task"] +TASK_TYPES = ["OnceTaskType", "RepeatableTaskType", "CronTaskType"] rq_job_decorator = job ExecutionStatus = JobStatus @@ -264,7 +264,7 @@ def __init__(self, *args, **kwargs) -> None: @staticmethod def reschedule_all_jobs(): - for model_name in OLD_MODEL_NAMES: + for model_name in MODEL_NAMES: model = apps.get_model(app_label="scheduler", model_name=model_name) enabled_jobs = model.objects.filter(enabled=True) unscheduled_jobs = filter(lambda j: j.ready_for_schedule(), enabled_jobs) diff --git a/scheduler/tests/test_mgmt_commands/test_export.py b/scheduler/tests/test_mgmt_commands/test_export.py index b67b60a..6ad1878 100644 --- a/scheduler/tests/test_mgmt_commands/test_export.py +++ b/scheduler/tests/test_mgmt_commands/test_export.py @@ -7,16 +7,18 @@ from django.core.management import call_command from django.test import TestCase +from scheduler.tests import test_settings # noqa from scheduler.tests.testtools import task_factory from scheduler.tools import TaskType -from scheduler.tests import test_settings # noqa class ExportTest(TestCase): def setUp(self) -> None: + super().setUp() self.tmpfile = tempfile.NamedTemporaryFile() def tearDown(self) -> None: + super().tearDown() os.remove(self.tmpfile.name) def test_export__should_export_job(self): diff --git a/scheduler/tools.py b/scheduler/tools.py index 1af3cfe..e851b9a 100644 --- a/scheduler/tools.py +++ b/scheduler/tools.py @@ -1,6 +1,6 @@ import importlib import os -from typing import List, Any, Callable, Optional, Union +from typing import List, Any, Callable, Optional import croniter from django.apps import apps @@ -10,14 +10,14 @@ from django.utils.translation import gettext_lazy as _ from scheduler.queues import get_queues, logger, get_queue -from scheduler.rq_classes import DjangoWorker, OLD_MODEL_NAMES, JobExecution, MODEL_NAMES +from scheduler.rq_classes import DjangoWorker, JobExecution, TASK_TYPES, MODEL_NAMES from scheduler.settings import SCHEDULER_CONFIG, Broker class TaskType(models.TextChoices): - CRON = "CronTask", _("Cron Task") - REPEATABLE = "RepeatableTask", _("Repeatable Task") - ONCE = "OnceTask", _("Run once") + CRON = "CronTaskType", _("Cron Task") + REPEATABLE = "RepeatableTaskType", _("Repeatable Task") + ONCE = "OnceTaskType", _("Run once") def callable_func(callable_str: str) -> Callable: @@ -39,31 +39,25 @@ def get_next_cron_time(cron_string: Optional[str]) -> Optional[timezone.datetime return next_itr -def get_scheduled_task(task_model: str, task_id: int) -> "BaseTask": # noqa: F821 - if isinstance(task_model, str) and task_model not in OLD_MODEL_NAMES and task_model not in MODEL_NAMES: - raise ValueError(f"Job Model `{task_model}` does not exist, choices are {OLD_MODEL_NAMES}") - +def get_scheduled_task(task_type_str: str, task_id: int) -> "BaseTask": # noqa: F821 # Try with new model names model = apps.get_model(app_label="scheduler", model_name="Task") - if task_model == "OnceTask": - task = model.objects.filter(task_type=TaskType.ONCE, id=task_id).first() + if task_type_str in TASK_TYPES: + try: + task_type = TaskType(task_type_str) + task = model.objects.filter(task_type=TaskType.ONCE, id=task_id).first() + if task is None: + raise ValueError(f"Job {task_type}:{task_id} does not exit") + return task + except ValueError: + raise ValueError(f"Invalid task type {task_type_str}") + elif task_type_str in MODEL_NAMES: + model = apps.get_model(app_label="scheduler", model_name=task_type_str) + task = model.objects.filter(id=task_id).first() if task is None: - raise ValueError(f"Job {task_model}:{task_id} does not exit") + raise ValueError(f"Job {task_type_str}:{task_id} does not exit") return task - elif task_model == "RepeatableTask": - task = model.objects.filter(task_type=TaskType.REPEATABLE, id=task_id).first() - if task is not None: - return task - elif task_model == "CronTask": - task = model.objects.filter(task_type=TaskType.CRON, id=task_id).first() - if task is not None: - return task - - model = apps.get_model(app_label="scheduler", model_name=task_model) - task = model.objects.filter(id=task_id).first() - if task is None: - raise ValueError(f"Job {task_model}:{task_id} does not exit") - return task + raise ValueError(f"Job Model {task_type_str} does not exist, choices are {TASK_TYPES}") def run_task(task_model: str, task_id: int) -> Any: From d1bf375e52dd5bb9ccff700ee9ece740d51dfd0c Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 19 Nov 2024 17:43:19 -0500 Subject: [PATCH 23/47] wip --- scheduler/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scheduler/tools.py b/scheduler/tools.py index e851b9a..c73aae8 100644 --- a/scheduler/tools.py +++ b/scheduler/tools.py @@ -45,7 +45,7 @@ def get_scheduled_task(task_type_str: str, task_id: int) -> "BaseTask": # noqa: if task_type_str in TASK_TYPES: try: task_type = TaskType(task_type_str) - task = model.objects.filter(task_type=TaskType.ONCE, id=task_id).first() + task = model.objects.filter(task_type=task_type, id=task_id).first() if task is None: raise ValueError(f"Job {task_type}:{task_id} does not exit") return task From 004f3a0d8cdf30520ec0b9f50f58038b9616860c Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 19 Nov 2024 17:46:44 -0500 Subject: [PATCH 24/47] wip --- scheduler/management/commands/export.py | 1 - scheduler/tests/test_old_models/test_old_repeatable_task.py | 5 +++-- scheduler/tests/test_task_types/test_task_model.py | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/scheduler/management/commands/export.py b/scheduler/management/commands/export.py index 567cca2..bb2b249 100644 --- a/scheduler/management/commands/export.py +++ b/scheduler/management/commands/export.py @@ -4,7 +4,6 @@ from django.apps import apps from django.core.management.base import BaseCommand -from scheduler.models import Task from scheduler.tools import MODEL_NAMES diff --git a/scheduler/tests/test_old_models/test_old_repeatable_task.py b/scheduler/tests/test_old_models/test_old_repeatable_task.py index 60591cb..6005cd3 100644 --- a/scheduler/tests/test_old_models/test_old_repeatable_task.py +++ b/scheduler/tests/test_old_models/test_old_repeatable_task.py @@ -7,7 +7,6 @@ from scheduler import settings from scheduler.models import RepeatableTask from scheduler.tests.test_old_models.test_old_task_model import BaseTestCases - from scheduler.tests.testtools import old_task_factory, _get_task_job_execution_from_registry @@ -136,7 +135,9 @@ def test_repeat_old_job_exhausted(self): def test_repeat_old_job_last_iter(self): base_time = timezone.now() - job = old_task_factory(self.TaskModelClass, scheduled_time=base_time - timedelta(hours=9, minutes=30), repeat=10) + job = old_task_factory( + self.TaskModelClass, scheduled_time=base_time - timedelta(hours=9, minutes=30), repeat=10, + ) self.assertEqual(job.repeat, 0) self.assertEqual(job.is_scheduled(), True) diff --git a/scheduler/tests/test_task_types/test_task_model.py b/scheduler/tests/test_task_types/test_task_model.py index 5f0a3ad..ee8afa8 100644 --- a/scheduler/tests/test_task_types/test_task_model.py +++ b/scheduler/tests/test_task_types/test_task_model.py @@ -478,7 +478,7 @@ def test_admin_delete_selected(self): res = self.client.post(url, data=data, follow=True) # assert self.assertEqual(200, res.status_code) - assert_response_has_msg(res, f"Successfully deleted 1 task.") + assert_response_has_msg(res, "Successfully deleted 1 task.") self.assertIsNone(Task.objects.filter(task_type=self.task_type).filter(id=task.id).first()) scheduled_jobs = queue.scheduled_job_registry.get_job_ids() self.assertNotIn(job_id, scheduled_jobs) @@ -506,4 +506,3 @@ def test_result_ttl_passthrough(self): job = task_factory(self.task_type, result_ttl=500) entry = _get_task_job_execution_from_registry(job) self.assertEqual(entry.result_ttl, 500) - From 1f79a4556500cb5083b8f986451f3effc0622791 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 19 Nov 2024 17:57:29 -0500 Subject: [PATCH 25/47] wip --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4e2cb1e..b5706b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ croniter = ">=2.0" click = "^8.1" rq = "^1.16" pyyaml = { version = "^6.0", optional = true } -valkey = "6.0.1" +valkey = { version = "^6.0.2", optional = true} [tool.poetry.dev-dependencies] poetry = "^1.8.3" @@ -60,6 +60,7 @@ freezegun = "^1.5" [tool.poetry.extras] yaml = ["pyyaml"] +valkey = ["valkey"] [tool.flake8] max-line-length = 120 From d50cf0b0395a061126ef1d449deb1b1e5a30106a Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 19 Nov 2024 17:58:09 -0500 Subject: [PATCH 26/47] wip --- poetry.lock | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index 136ea0a..5979b58 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1623,17 +1623,17 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "valkey" -version = "6.0.1" +version = "6.0.2" description = "Python client for Valkey forked from redis-py" -optional = false +optional = true python-versions = ">=3.8" files = [ - {file = "valkey-6.0.1-py3-none-any.whl", hash = "sha256:6702bf323e88e50ef0be37aad697bcc6334edd40cc66f01259265dd410fa22dc"}, - {file = "valkey-6.0.1.tar.gz", hash = "sha256:58f4628dc038ab5aa04eea6e75557309c9412a8c45e81ad42d53e42b9a36e7dc"}, + {file = "valkey-6.0.2-py3-none-any.whl", hash = "sha256:dbbdd65439ee0dc5689502c54f1899504cc7268e85cb7fe8935f062178ff5805"}, + {file = "valkey-6.0.2.tar.gz", hash = "sha256:dc2e91512b82d1da0b91ab0cdbd8c97c0c0250281728cb32f9398760df9caeae"}, ] [package.dependencies] -async-timeout = {version = ">=4.0.3", markers = "python_version < \"3.11\""} +async-timeout = {version = ">=4.0.3", markers = "python_version < \"3.11.3\""} [package.extras] libvalkey = ["libvalkey (>=4.0.0)"] @@ -1752,9 +1752,10 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", type = ["pytest-mypy"] [extras] +valkey = ["valkey"] yaml = ["pyyaml"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "641d4bec80938f6c858d36d4d4cce1226021cd0a99263de0f9d3e9a240ae4512" +content-hash = "d2f15d1a3c26092b506e3da3a2539fe9ca83c8e68c7949c64f71f8af3401b3cb" From 1321bad7300b4e064abbe1e473383130896729f2 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 19 Nov 2024 18:00:46 -0500 Subject: [PATCH 27/47] wip --- scheduler/models/task.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scheduler/models/task.py b/scheduler/models/task.py index 0848dc2..11e783d 100644 --- a/scheduler/models/task.py +++ b/scheduler/models/task.py @@ -395,6 +395,8 @@ def delete(self, **kwargs): super(Task, self).delete(**kwargs) def interval_display(self): + if self.interval is None or self.interval_unit is None: + return "" return "{} {}".format(self.interval, self.get_interval_unit_display()) def interval_seconds(self): From 42e17a86bb72753ef8465a6bd72947683c590bf8 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 19 Nov 2024 18:16:25 -0500 Subject: [PATCH 28/47] wip --- scheduler/admin/task_admin.py | 25 +++++++++++++++------- scheduler/models/task.py | 5 ----- scheduler/static/admin/js/select-fields.js | 6 +++--- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/scheduler/admin/task_admin.py b/scheduler/admin/task_admin.py index 199017c..c24e967 100644 --- a/scheduler/admin/task_admin.py +++ b/scheduler/admin/task_admin.py @@ -7,7 +7,7 @@ from scheduler import tools from scheduler.models import TaskArg, TaskKwarg, Task from scheduler.settings import SCHEDULER_CONFIG, logger -from scheduler.tools import get_job_executions_for_task +from scheduler.tools import get_job_executions_for_task, TaskType class HiddenMixin(object): @@ -56,9 +56,7 @@ class Media: "function_string", "is_scheduled", "queue", - "scheduled_time", - "interval_display", - "cron_string", + "task_schedule", "next_run", "successful_runs", "last_successful_run", @@ -88,15 +86,15 @@ class Media: ), ( None, - dict(fields=("scheduled_time",), classes=("tasktype-OnceTask",)), + dict(fields=("scheduled_time",), classes=("tasktype-OnceTaskType",)), ), ( None, - dict(fields=("cron_string",), classes=("tasktype-CronTask",)), + dict(fields=("cron_string",), classes=("tasktype-CronTaskType",)), ), ( None, - dict(fields=("interval", "interval_unit", "repeat"), classes=("tasktype-RepeatableTask",)), + dict(fields=("interval", "interval_unit", "repeat"), classes=("tasktype-RepeatableTaskType",)), ), (_("RQ Settings"), dict(fields=(("queue", "at_front"), "job_id"))), ( @@ -105,8 +103,19 @@ class Media: ), ) + @admin.display(description="Schedule") + def task_schedule(self, o: Task) -> str: + if o.task_type == TaskType.ONCE.value: + return f"Run once: {o.scheduled_time:%Y-%m-%d %H:%M:%S}" + elif o.task_type == TaskType.CRON.value: + return f"Cron: {o.cron_string}" + elif o.task_type == TaskType.REPEATABLE.value: + if o.interval is None or o.interval_unit is None: + return "" + return "Repeatable: {} {}".format(o.interval, o.get_interval_unit_display()) + @admin.display(description="Next run") - def next_run(self, o: Task): + def next_run(self, o: Task) -> str: return tools.get_next_cron_time(o.cron_string) def change_view(self, request, object_id, form_url="", extra_context=None): diff --git a/scheduler/models/task.py b/scheduler/models/task.py index 11e783d..c2a5249 100644 --- a/scheduler/models/task.py +++ b/scheduler/models/task.py @@ -394,11 +394,6 @@ def delete(self, **kwargs): self.unschedule() super(Task, self).delete(**kwargs) - def interval_display(self): - if self.interval is None or self.interval_unit is None: - return "" - return "{} {}".format(self.interval, self.get_interval_unit_display()) - def interval_seconds(self): kwargs = { self.interval_unit: self.interval, diff --git a/scheduler/static/admin/js/select-fields.js b/scheduler/static/admin/js/select-fields.js index 6122549..50ed38f 100644 --- a/scheduler/static/admin/js/select-fields.js +++ b/scheduler/static/admin/js/select-fields.js @@ -1,9 +1,9 @@ (function ($) { $(function () { const tasktypes = { - "CronTask": $(".tasktype-CronTask"), - "RepeatableTask": $(".tasktype-RepeatableTask"), - "OnceTask": $(".tasktype-OnceTask"), + "CronTaskType": $(".tasktype-CronTaskType"), + "RepeatableTaskType": $(".tasktype-RepeatableTaskType"), + "OnceTaskType": $(".tasktype-OnceTaskType"), }; var taskTypeField = $('#id_task_type'); From bb5f063096166824c6e32a5f1202fa7ab850f2c6 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 19 Nov 2024 18:22:22 -0500 Subject: [PATCH 29/47] wip --- scheduler/admin/task_admin.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scheduler/admin/task_admin.py b/scheduler/admin/task_admin.py index c24e967..ff91d73 100644 --- a/scheduler/admin/task_admin.py +++ b/scheduler/admin/task_admin.py @@ -1,10 +1,9 @@ -import redis -import valkey from django.contrib import admin, messages from django.contrib.contenttypes.admin import GenericStackedInline from django.utils.translation import gettext_lazy as _ from scheduler import tools +from scheduler.broker_types import ConnectionErrorTypes from scheduler.models import TaskArg, TaskKwarg, Task from scheduler.settings import SCHEDULER_CONFIG, logger from scheduler.tools import get_job_executions_for_task, TaskType @@ -123,7 +122,7 @@ def change_view(self, request, object_id, form_url="", extra_context=None): obj = self.get_object(request, object_id) try: execution_list = get_job_executions_for_task(obj.queue, obj) - except (redis.ConnectionError, valkey.ConnectionError) as e: + except ConnectionErrorTypes as e: logger.warn(f"Could not get job executions: {e}") execution_list = list() paginator = self.get_paginator(request, execution_list, SCHEDULER_CONFIG.EXECUTIONS_IN_PAGE) From 554fb893927ebe888e2f528e7a6c25f55c4caa33 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 19 Nov 2024 18:36:11 -0500 Subject: [PATCH 30/47] wip --- scheduler/tests/test_task_types/test_repeatable_task.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scheduler/tests/test_task_types/test_repeatable_task.py b/scheduler/tests/test_task_types/test_repeatable_task.py index ba73c97..4dd9aec 100644 --- a/scheduler/tests/test_task_types/test_repeatable_task.py +++ b/scheduler/tests/test_task_types/test_repeatable_task.py @@ -113,10 +113,6 @@ def test_interval_seconds_seconds(self): job = RepeatableTask(interval=15, interval_unit="seconds") self.assertEqual(15.0, job.interval_seconds()) - def test_interval_display(self): - job = task_factory(self.task_type, interval=15, interval_unit="minutes") - self.assertEqual(job.interval_display(), "15 minutes") - def test_result_interval(self): job = task_factory( self.task_type, From 467ba436d077c0a59a7ea4888063b565a257ad11 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 19 Nov 2024 18:36:37 -0500 Subject: [PATCH 31/47] wip --- .../migrations/0020_alter_task_task_type.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 scheduler/migrations/0020_alter_task_task_type.py diff --git a/scheduler/migrations/0020_alter_task_task_type.py b/scheduler/migrations/0020_alter_task_task_type.py new file mode 100644 index 0000000..d55932c --- /dev/null +++ b/scheduler/migrations/0020_alter_task_task_type.py @@ -0,0 +1,27 @@ +# Generated by Django 5.1.3 on 2024-11-19 23:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("scheduler", "0019_task"), + ] + + operations = [ + migrations.AlterField( + model_name="task", + name="task_type", + field=models.CharField( + choices=[ + ("CronTaskType", "Cron Task"), + ("RepeatableTaskType", "Repeatable Task"), + ("OnceTaskType", "Run once"), + ], + default="OnceTaskType", + max_length=32, + verbose_name="Task type", + ), + ), + ] From cfe67c7040a7e40e341ce08da5447625ca024006 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Tue, 19 Nov 2024 18:37:33 -0500 Subject: [PATCH 32/47] wip --- scheduler/migrations/0019_task.py | 10 +++---- .../migrations/0020_alter_task_task_type.py | 27 ------------------- 2 files changed, 5 insertions(+), 32 deletions(-) delete mode 100644 scheduler/migrations/0020_alter_task_task_type.py diff --git a/scheduler/migrations/0019_task.py b/scheduler/migrations/0019_task.py index a328c7c..7d1681f 100644 --- a/scheduler/migrations/0019_task.py +++ b/scheduler/migrations/0019_task.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.2 on 2024-10-15 22:39 +# Generated by Django 5.1.3 on 2024-11-19 23:36 import scheduler.models.task from django.db import migrations, models @@ -25,11 +25,11 @@ class Migration(migrations.Migration): "task_type", models.CharField( choices=[ - ("CronTask", "Cron Task"), - ("RepeatableTask", "Repeatable Task"), - ("OnceTask", "Run once"), + ("CronTaskType", "Cron Task"), + ("RepeatableTaskType", "Repeatable Task"), + ("OnceTaskType", "Run once"), ], - default="OnceTask", + default="OnceTaskType", max_length=32, verbose_name="Task type", ), diff --git a/scheduler/migrations/0020_alter_task_task_type.py b/scheduler/migrations/0020_alter_task_task_type.py deleted file mode 100644 index d55932c..0000000 --- a/scheduler/migrations/0020_alter_task_task_type.py +++ /dev/null @@ -1,27 +0,0 @@ -# Generated by Django 5.1.3 on 2024-11-19 23:36 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("scheduler", "0019_task"), - ] - - operations = [ - migrations.AlterField( - model_name="task", - name="task_type", - field=models.CharField( - choices=[ - ("CronTaskType", "Cron Task"), - ("RepeatableTaskType", "Repeatable Task"), - ("OnceTaskType", "Run once"), - ], - default="OnceTaskType", - max_length=32, - verbose_name="Task type", - ), - ), - ] From 4eeadf27386b56703ec9c6711220817357e9df7e Mon Sep 17 00:00:00 2001 From: Daniel M Date: Wed, 20 Nov 2024 10:32:18 -0500 Subject: [PATCH 33/47] wip --- scheduler/admin/__init__.py | 4 +++- scheduler/admin/{task_models.py => old_task_models.py} | 3 +++ scheduler/admin/task_admin.py | 9 ++------- 3 files changed, 8 insertions(+), 8 deletions(-) rename scheduler/admin/{task_models.py => old_task_models.py} (98%) diff --git a/scheduler/admin/__init__.py b/scheduler/admin/__init__.py index 47abb5b..9419e52 100644 --- a/scheduler/admin/__init__.py +++ b/scheduler/admin/__init__.py @@ -1,3 +1,5 @@ -from .task_models import TaskAdmin as OldTaskAdmin # noqa: F401 from .ephemeral_models import QueueAdmin, WorkerAdmin # noqa: F401 +from .old_task_models import TaskAdmin as OldTaskAdmin # noqa: F401 from .task_admin import TaskAdmin # noqa: F401 + +__all__ = ["OldTaskAdmin", "QueueAdmin", "WorkerAdmin", "TaskAdmin", ] diff --git a/scheduler/admin/task_models.py b/scheduler/admin/old_task_models.py similarity index 98% rename from scheduler/admin/task_models.py rename to scheduler/admin/old_task_models.py index ca3d8db..e5a6cc5 100644 --- a/scheduler/admin/task_models.py +++ b/scheduler/admin/old_task_models.py @@ -1,5 +1,6 @@ from django.contrib import admin, messages from django.contrib.contenttypes.admin import GenericStackedInline +from django.http import HttpRequest from django.utils.translation import gettext_lazy as _ from scheduler import tools @@ -158,6 +159,8 @@ class TaskAdmin(admin.ModelAdmin): }, ), ) + def has_add_permission(self, request: HttpRequest)-> bool: + return False def get_list_display(self, request): if self.model.__name__ not in _LIST_DISPLAY_EXTRA: diff --git a/scheduler/admin/task_admin.py b/scheduler/admin/task_admin.py index ff91d73..b89b7d2 100644 --- a/scheduler/admin/task_admin.py +++ b/scheduler/admin/task_admin.py @@ -9,18 +9,13 @@ from scheduler.tools import get_job_executions_for_task, TaskType -class HiddenMixin(object): - class Media: - js = ("admin/js/jquery.init.js",) - - -class JobArgInline(HiddenMixin, GenericStackedInline): +class JobArgInline(GenericStackedInline): model = TaskArg extra = 0 fieldsets = ((None, dict(fields=("arg_type", "val"))),) -class JobKwargInline(HiddenMixin, GenericStackedInline): +class JobKwargInline(GenericStackedInline): model = TaskKwarg extra = 0 fieldsets = ((None, dict(fields=("key", ("arg_type", "val")))),) From d24742b42eb132531c2da123ffc43d2d036bf567 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Wed, 20 Nov 2024 10:33:51 -0500 Subject: [PATCH 34/47] wip --- scheduler/admin/old_task_models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scheduler/admin/old_task_models.py b/scheduler/admin/old_task_models.py index e5a6cc5..6f42849 100644 --- a/scheduler/admin/old_task_models.py +++ b/scheduler/admin/old_task_models.py @@ -159,7 +159,8 @@ class TaskAdmin(admin.ModelAdmin): }, ), ) - def has_add_permission(self, request: HttpRequest)-> bool: + + def has_add_permission(self, request: HttpRequest) -> bool: return False def get_list_display(self, request): From 2da0bdf2a6976c5da436769e8473be35f67a400b Mon Sep 17 00:00:00 2001 From: Daniel M Date: Wed, 20 Nov 2024 16:15:53 -0500 Subject: [PATCH 35/47] wip --- scheduler/admin/old_task_models.py | 1 + scheduler/management/commands/import.py | 3 ++- ...id_repeatabletask_new_task_id_and_more.py} | 24 ++++++++++++++++++- scheduler/models/__init__.py | 14 +++++++---- scheduler/models/scheduled_task.py | 1 + 5 files changed, 37 insertions(+), 6 deletions(-) rename scheduler/migrations/{0019_task.py => 0019_task_crontask_new_task_id_repeatabletask_new_task_id_and_more.py} (88%) diff --git a/scheduler/admin/old_task_models.py b/scheduler/admin/old_task_models.py index 6f42849..905f752 100644 --- a/scheduler/admin/old_task_models.py +++ b/scheduler/admin/old_task_models.py @@ -134,6 +134,7 @@ class TaskAdmin(admin.ModelAdmin): "function_string", "is_scheduled", "queue", + "new_task_id", ) list_display_links = ("name",) readonly_fields = ("job_id",) diff --git a/scheduler/management/commands/import.py b/scheduler/management/commands/import.py index a89ddc4..bbc7337 100644 --- a/scheduler/management/commands/import.py +++ b/scheduler/management/commands/import.py @@ -52,7 +52,8 @@ def create_task_from_dict(task_dict: Dict[str, Any], update): if not settings.USE_TZ and not timezone.is_naive(target): target = timezone.make_naive(target) kwargs["scheduled_time"] = target - model_fields = set(map(lambda field: field.attname, Task._meta.get_fields())) + model_fields = filter(lambda field:hasattr(field,'attname'),Task._meta.get_fields()) + model_fields = set(map(lambda field: field.attname, model_fields)) keys_to_ignore = list(filter(lambda _k: _k not in model_fields, kwargs.keys())) for k in keys_to_ignore: del kwargs[k] diff --git a/scheduler/migrations/0019_task.py b/scheduler/migrations/0019_task_crontask_new_task_id_repeatabletask_new_task_id_and_more.py similarity index 88% rename from scheduler/migrations/0019_task.py rename to scheduler/migrations/0019_task_crontask_new_task_id_repeatabletask_new_task_id_and_more.py index 7d1681f..bfdbcc1 100644 --- a/scheduler/migrations/0019_task.py +++ b/scheduler/migrations/0019_task_crontask_new_task_id_repeatabletask_new_task_id_and_more.py @@ -1,5 +1,6 @@ -# Generated by Django 5.1.3 on 2024-11-19 23:36 +# Generated by Django 5.1.3 on 2024-11-20 20:32 +import django.db.models.deletion import scheduler.models.task from django.db import migrations, models @@ -161,4 +162,25 @@ class Migration(migrations.Migration): ), ], ), + migrations.AddField( + model_name="crontask", + name="new_task_id", + field=models.ForeignKey( + blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to="scheduler.task" + ), + ), + migrations.AddField( + model_name="repeatabletask", + name="new_task_id", + field=models.ForeignKey( + blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to="scheduler.task" + ), + ), + migrations.AddField( + model_name="scheduledtask", + name="new_task_id", + field=models.ForeignKey( + blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to="scheduler.task" + ), + ), ] diff --git a/scheduler/models/__init__.py b/scheduler/models/__init__.py index 59baf5a..044f774 100644 --- a/scheduler/models/__init__.py +++ b/scheduler/models/__init__.py @@ -1,4 +1,10 @@ -from .args import TaskKwarg, TaskArg, BaseTaskArg # noqa: F401 -from .queue import Queue # noqa: F401 -from .scheduled_task import BaseTask, ScheduledTask, RepeatableTask, CronTask # noqa: F401 -from .task import Task # noqa: F401 +from .args import TaskKwarg, TaskArg +from .scheduled_task import BaseTask, ScheduledTask, RepeatableTask, CronTask +from .queue import Queue +from .task import Task + +__all__ = [ + "TaskKwarg", "TaskArg", + "ScheduledTask", "RepeatableTask", "CronTask", + "Queue", "Task", +] diff --git a/scheduler/models/scheduled_task.py b/scheduler/models/scheduled_task.py index a150f06..20fdd12 100644 --- a/scheduler/models/scheduled_task.py +++ b/scheduler/models/scheduled_task.py @@ -124,6 +124,7 @@ class BaseTask(models.Model): >0: Result expires after n seconds.""" ), ) + new_task_id = models.ForeignKey('Task', on_delete=models.CASCADE, blank=True, null=True) def callable_func(self): """Translate callable string to callable""" From bec98bff3dabe43024ae1e461ed49401545b6a94 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Thu, 21 Nov 2024 08:02:50 -0500 Subject: [PATCH 36/47] docs --- docs/changelog.md | 14 ++++++---- docs/configuration.md | 2 +- docs/index.md | 59 +++++++++++++++++++++++++------------------ docs/installation.md | 2 +- scheduler/queues.py | 2 +- 5 files changed, 46 insertions(+), 33 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index c839514..84e0445 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,13 +1,17 @@ # Changelog -## v2.2.0 🌈 +## v3.0.0 🌈 + +### Breaking Changes + +- Renamed `REDIS_CLIENT_KWARGS` configuration to `CLIENT_KWARGS`. ### 🚀 Features -- Created a new `Task` model representing all kind of scheduled tasks. - - In future versions, `CronTask`, `ScheduledTask` and `RepeatableTask` will be removed. - - `Task` model has a `task_type` field to differentiate between the types of tasks. - - Old tasks in the database will be migrated to the new `Task` model automatically. +- Created a new `Task` model representing all kind of scheduled tasks. + - In future versions, `CronTask`, `ScheduledTask` and `RepeatableTask` will be removed. + - `Task` model has a `task_type` field to differentiate between the types of tasks. + - Old tasks in the database will be migrated to the new `Task` model automatically. ## v2.1.1 🌈 diff --git a/docs/configuration.md b/docs/configuration.md index fb05ec8..da5c5a9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -20,7 +20,7 @@ SCHEDULER_QUEUES = { 'USERNAME': 'some-user', 'PASSWORD': 'some-password', 'DEFAULT_TIMEOUT': 360, - 'REDIS_CLIENT_KWARGS': { # Eventual additional Redis connection arguments + 'CLIENT_KWARGS': { # Eventual additional Redis connection arguments 'ssl_cert_reqs': None, }, 'TOKEN_VALIDATION_METHOD': None, # Method to validate auth-header diff --git a/docs/index.md b/docs/index.md index 24f3275..9c35786 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,8 +1,8 @@ # Django tasks Scheduler -[![Django CI][1]][2] -![badge][3] -[![badge][4]][5] +[![Django CI][badge]][2] +![badge][coverage] +[![badge][pypi-downloads]][pypi] --- @@ -11,6 +11,28 @@ This allows remembering scheduled tasks, their parameters, etc. ## Terminology +### Scheduled Task + +Starting v3.0.0, django-tasks-scheduler is using a single `Task` model with different task types, the task types are: + +- `ONCE` - Run the task once at a scheduled time. +- `REPEATABLE` - Run the task multiple times (limited number of times or infinite times) based on a time interval. +- `CRON` - Run a task indefinitely based on a cron string schedule. + +This enables having one admin view for all scheduled tasks, and having one table in the database to maintain the task +reduces the number of overall queries. +An `Task` instance contains all relevant information about a task to enable the users to schedule using django-admin and +track their status. + +Previously, there were three different models for ScheduledTask. These exist for legacy purposes and are scheduled to +be removed. + +* `Scheduled Task` - Run a task once, on a specific time (can be immediate). +* `Repeatable Task` - Run a task multiple times (limited number of times or infinite times) based on an interval +* `Cron Task` - Run a task multiple times (limited number of times or infinite times) based on a cron string + +Scheduled tasks are scheduled when the django application starts, and after a scheduled task is executed. + ### Queue A queue of messages between processes (main django-app process and worker usually). @@ -34,7 +56,7 @@ This is a subprocess of worker. Once a worker listening to the queue becomes available, the job will be executed -### Scheduled Task Execution +### Scheduled Job Execution A scheduler checking the queue periodically will check whether the time the job should be executed has come, and if so, it will queue it. @@ -42,19 +64,6 @@ it will queue it. * A job is considered scheduled if it is queued to be executed, or scheduled to be executed. * If there is no scheduler, the job will not be queued to run. -### Scheduled Task - -django models storing information about jobs. So it is possible to schedule using -django-admin and track their status. - -There are three types of ScheduledTask. - -* `Scheduled Task` - Run a job once, on a specific time (can be immediate). -* `Repeatable Task` - Run a job multiple times (limited number of times or infinite times) based on an interval -* `Cron Task` - Run a job multiple times (limited number of times or infinite times) based on a cron string - -Scheduled jobs are scheduled when the django application starts, and after a scheduled task is executed. - ## Scheduler sequence diagram ```mermaid @@ -121,24 +130,24 @@ sequenceDiagram ## Reporting issues or Features requests -Please report issues via [GitHub Issues][6] . +Please report issues via [GitHub Issues][issues] . --- ## Acknowledgements -A lot of django-admin views and their tests were adopted from [django-rq][7]. +A lot of django-admin views and their tests were adopted from [django-rq][django-rq]. -[1]:https://github.com/django-commons/django-tasks-scheduler/actions/workflows/test.yml/badge.svg +[badge]:https://github.com/django-commons/django-tasks-scheduler/actions/workflows/test.yml/badge.svg [2]:https://github.com/django-commons/django-tasks-scheduler/actions/workflows/test.yml -[3]:https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/cunla/b756396efb895f0e34558c980f1ca0c7/raw/django-tasks-scheduler-4.json +[coverage]:https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/cunla/b756396efb895f0e34558c980f1ca0c7/raw/django-tasks-scheduler-4.json -[4]:https://img.shields.io/pypi/dm/django-tasks-scheduler +[pypi-downloads]:https://img.shields.io/pypi/dm/django-tasks-scheduler -[5]:https://pypi.org/project/django-tasks-scheduler/ +[pypi]:https://pypi.org/project/django-tasks-scheduler/ -[6]:https://github.com/django-commons/django-tasks-scheduler/issues +[issues]:https://github.com/django-commons/django-tasks-scheduler/issues -[7]:https://github.com/rq/django-rq \ No newline at end of file +[django-rq]:https://github.com/rq/django-rq \ No newline at end of file diff --git a/docs/installation.md b/docs/installation.md index 6a2db1d..13573b0 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -26,7 +26,7 @@ 'USERNAME': 'some-user', 'PASSWORD': 'some-password', 'DEFAULT_TIMEOUT': 360, - 'REDIS_CLIENT_KWARGS': { # Eventual additional Redis connection arguments + 'CLIENT_KWARGS': { # Eventual additional Redis connection arguments 'ssl_cert_reqs': None, }, }, diff --git a/scheduler/queues.py b/scheduler/queues.py index df65ce7..15abbd9 100644 --- a/scheduler/queues.py +++ b/scheduler/queues.py @@ -76,7 +76,7 @@ def _get_broker_connection(config, use_strict_broker=False): password=config.get("PASSWORD"), ssl=config.get("SSL", False), ssl_cert_reqs=config.get("SSL_CERT_REQS", "required"), - **config.get("REDIS_CLIENT_KWARGS", {}), + **config.get("CLIENT_KWARGS", {}), ) From 5bc9e07c567a4d6a34c3c648e82d7d6cc04cdf37 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Fri, 29 Nov 2024 15:16:52 -0500 Subject: [PATCH 37/47] add link to new task for old models admin --- scheduler/admin/old_task_models.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/scheduler/admin/old_task_models.py b/scheduler/admin/old_task_models.py index 905f752..ac4232c 100644 --- a/scheduler/admin/old_task_models.py +++ b/scheduler/admin/old_task_models.py @@ -1,11 +1,15 @@ +from typing import Optional + from django.contrib import admin, messages from django.contrib.contenttypes.admin import GenericStackedInline from django.http import HttpRequest +from django.urls import reverse +from django.utils.html import format_html from django.utils.translation import gettext_lazy as _ from scheduler import tools from scheduler.broker_types import ConnectionErrorTypes -from scheduler.models import CronTask, TaskArg, TaskKwarg, RepeatableTask, ScheduledTask +from scheduler.models import CronTask, TaskArg, TaskKwarg, RepeatableTask, ScheduledTask, BaseTask from scheduler.settings import SCHEDULER_CONFIG, logger from scheduler.tools import get_job_executions_for_task @@ -130,11 +134,11 @@ class TaskAdmin(admin.ModelAdmin): list_display = ( "enabled", "name", + "link_new_task", "job_id", "function_string", "is_scheduled", "queue", - "new_task_id", ) list_display_links = ("name",) readonly_fields = ("job_id",) @@ -161,6 +165,14 @@ class TaskAdmin(admin.ModelAdmin): ), ) + @admin.display(description="New task") + def link_new_task(self, o: BaseTask) -> Optional[str]: + if o.new_task_id is None: + return None + url = reverse(f"admin:scheduler_task_change", args=[o.new_task_id, ]) + html = format_html(f"""{o.new_task_id}""") + return html + def has_add_permission(self, request: HttpRequest) -> bool: return False From e6b9923460091dc0a2e5aeca8f11e5fb57952628 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Fri, 29 Nov 2024 15:19:41 -0500 Subject: [PATCH 38/47] rename --- ..._crontask_queue_alter_repeatabletask_queue_and_more.py | 8 ++++---- scheduler/models/__init__.py | 2 +- .../models/{scheduled_task.py => old_scheduled_task.py} | 0 3 files changed, 5 insertions(+), 5 deletions(-) rename scheduler/models/{scheduled_task.py => old_scheduled_task.py} (100%) diff --git a/scheduler/migrations/0018_alter_crontask_queue_alter_repeatabletask_queue_and_more.py b/scheduler/migrations/0018_alter_crontask_queue_alter_repeatabletask_queue_and_more.py index aeb79c5..9b8b28c 100644 --- a/scheduler/migrations/0018_alter_crontask_queue_alter_repeatabletask_queue_and_more.py +++ b/scheduler/migrations/0018_alter_crontask_queue_alter_repeatabletask_queue_and_more.py @@ -1,6 +1,6 @@ # Generated by Django 5.1b1 on 2024-06-29 14:21 -import scheduler.models.scheduled_task +import scheduler.models.old_scheduled_task from django.db import migrations, models @@ -14,16 +14,16 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='crontask', name='queue', - field=models.CharField(choices=scheduler.models.scheduled_task.get_queue_choices, help_text='Queue name', max_length=255, verbose_name='queue'), + field=models.CharField(choices=scheduler.models.old_scheduled_task.get_queue_choices, help_text='Queue name', max_length=255, verbose_name='queue'), ), migrations.AlterField( model_name='repeatabletask', name='queue', - field=models.CharField(choices=scheduler.models.scheduled_task.get_queue_choices, help_text='Queue name', max_length=255, verbose_name='queue'), + field=models.CharField(choices=scheduler.models.old_scheduled_task.get_queue_choices, help_text='Queue name', max_length=255, verbose_name='queue'), ), migrations.AlterField( model_name='scheduledtask', name='queue', - field=models.CharField(choices=scheduler.models.scheduled_task.get_queue_choices, help_text='Queue name', max_length=255, verbose_name='queue'), + field=models.CharField(choices=scheduler.models.old_scheduled_task.get_queue_choices, help_text='Queue name', max_length=255, verbose_name='queue'), ), ] diff --git a/scheduler/models/__init__.py b/scheduler/models/__init__.py index 044f774..04f4862 100644 --- a/scheduler/models/__init__.py +++ b/scheduler/models/__init__.py @@ -1,5 +1,5 @@ from .args import TaskKwarg, TaskArg -from .scheduled_task import BaseTask, ScheduledTask, RepeatableTask, CronTask +from .old_scheduled_task import BaseTask, ScheduledTask, RepeatableTask, CronTask from .queue import Queue from .task import Task diff --git a/scheduler/models/scheduled_task.py b/scheduler/models/old_scheduled_task.py similarity index 100% rename from scheduler/models/scheduled_task.py rename to scheduler/models/old_scheduled_task.py From b3692815dbb41ac35af8309d8fab1c155e2e165d Mon Sep 17 00:00:00 2001 From: Daniel M Date: Fri, 29 Nov 2024 15:26:42 -0500 Subject: [PATCH 39/47] job => task --- scheduler/management/commands/import.py | 29 +++++++++++++------------ 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/scheduler/management/commands/import.py b/scheduler/management/commands/import.py index bbc7337..bd8ac14 100644 --- a/scheduler/management/commands/import.py +++ b/scheduler/management/commands/import.py @@ -1,5 +1,5 @@ import sys -from typing import Dict, Any +from typing import Dict, Any, Optional import click from django.apps import apps @@ -32,16 +32,16 @@ def get_task_type(model_str: str) -> TaskType: raise ValueError(f"Invalid model {model_str}") -def create_task_from_dict(task_dict: Dict[str, Any], update): - existing_job = Task.objects.filter(name=task_dict["name"]).first() +def create_task_from_dict(task_dict: Dict[str, Any], update: bool) -> Optional[Task]: + existing_task = Task.objects.filter(name=task_dict["name"]).first() task_type = get_task_type(task_dict["model"]) - if existing_job: + if existing_task: if update: - click.echo(f'Found existing job "{existing_job}, removing it to be reinserted"') - existing_job.delete() + click.echo(f'Found existing job "{existing_task}, removing it to be reinserted"') + existing_task.delete() else: - click.echo(f'Found existing job "{existing_job}", skipping') - return + click.echo(f'Found existing job "{existing_task}", skipping') + return None kwargs = dict(task_dict) kwargs["task_type"] = task_type del kwargs["model"] @@ -52,27 +52,28 @@ def create_task_from_dict(task_dict: Dict[str, Any], update): if not settings.USE_TZ and not timezone.is_naive(target): target = timezone.make_naive(target) kwargs["scheduled_time"] = target - model_fields = filter(lambda field:hasattr(field,'attname'),Task._meta.get_fields()) + model_fields = filter(lambda field: hasattr(field, 'attname'), Task._meta.get_fields()) model_fields = set(map(lambda field: field.attname, model_fields)) keys_to_ignore = list(filter(lambda _k: _k not in model_fields, kwargs.keys())) for k in keys_to_ignore: del kwargs[k] - scheduled_job = Task.objects.create(**kwargs) - click.echo(f"Created job {scheduled_job}") - content_type = ContentType.objects.get_for_model(scheduled_job) + task = Task.objects.create(**kwargs) + click.echo(f"Created task {task}") + content_type = ContentType.objects.get_for_model(task) for arg in task_dict["callable_args"]: TaskArg.objects.create( content_type=content_type, - object_id=scheduled_job.id, + object_id=task.id, **arg, ) for arg in task_dict["callable_kwargs"]: TaskKwarg.objects.create( content_type=content_type, - object_id=scheduled_job.id, + object_id=task.id, **arg, ) + return task class Command(BaseCommand): From 0a61af4782ed016e933ce7a1994f480f9a0896c7 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Fri, 29 Nov 2024 15:41:02 -0500 Subject: [PATCH 40/47] job => task --- scheduler/admin/old_task_models.py | 25 ++++-- scheduler/admin/task_admin.py | 13 +-- scheduler/models/migrate_util.py | 87 +++++++++++++++++++ scheduler/models/old_scheduled_task.py | 3 +- .../tests/test_old_models/test_old_models.py | 4 +- .../test_old_models/test_old_task_model.py | 4 +- 6 files changed, 120 insertions(+), 16 deletions(-) create mode 100644 scheduler/models/migrate_util.py diff --git a/scheduler/admin/old_task_models.py b/scheduler/admin/old_task_models.py index ac4232c..9fe1651 100644 --- a/scheduler/admin/old_task_models.py +++ b/scheduler/admin/old_task_models.py @@ -9,7 +9,7 @@ from scheduler import tools from scheduler.broker_types import ConnectionErrorTypes -from scheduler.models import CronTask, TaskArg, TaskKwarg, RepeatableTask, ScheduledTask, BaseTask +from scheduler.models import CronTask, TaskArg, TaskKwarg, RepeatableTask, ScheduledTask, BaseTask, migrate_util from scheduler.settings import SCHEDULER_CONFIG, logger from scheduler.tools import get_job_executions_for_task @@ -113,6 +113,11 @@ class JobKwargInline(HiddenMixin, GenericStackedInline): ) +def get_message_bit(rows_updated: int) -> str: + message_bit = "1 task was" if rows_updated == 1 else f"{rows_updated} tasks were" + return message_bit + + @admin.register(CronTask, ScheduledTask, RepeatableTask) class TaskAdmin(admin.ModelAdmin): """TaskAdmin admin view for all task models. @@ -230,6 +235,16 @@ def delete_model(self, request, obj): obj.unschedule() super(TaskAdmin, self).delete_model(request, obj) + @admin.action(description=_("Migrate to new Task model(s)"), permissions=("change",)) + def migrate(self, request, queryset): + rows_updated = 0 + for obj in queryset.filter(enabled=True).iterator(): + migrate_util.migrate(obj) + rows_updated += 1 + + level = messages.WARNING if not rows_updated else messages.INFO + self.message_user(request, f"{get_message_bit(rows_updated)} successfully migrated to new model.", level=level) + @admin.action(description=_("Disable selected %(verbose_name_plural)s"), permissions=("change",)) def disable_selected(self, request, queryset): rows_updated = 0 @@ -238,10 +253,9 @@ def disable_selected(self, request, queryset): obj.unschedule() rows_updated += 1 - message_bit = "1 job was" if rows_updated == 1 else f"{rows_updated} jobs were" - level = messages.WARNING if not rows_updated else messages.INFO - self.message_user(request, f"{message_bit} successfully disabled and unscheduled.", level=level) + self.message_user(request, f"{get_message_bit(rows_updated)} successfully disabled and unscheduled.", + level=level) @admin.action(description=_("Enable selected %(verbose_name_plural)s"), permissions=("change",)) def enable_selected(self, request, queryset): @@ -251,9 +265,8 @@ def enable_selected(self, request, queryset): obj.save() rows_updated += 1 - message_bit = "1 job was" if rows_updated == 1 else f"{rows_updated} jobs were" level = messages.WARNING if not rows_updated else messages.INFO - self.message_user(request, f"{message_bit} successfully enabled and scheduled.", level=level) + self.message_user(request, f"{get_message_bit(rows_updated)} successfully enabled and scheduled.", level=level) @admin.action(description="Enqueue now", permissions=("change",)) def enqueue_job_now(self, request, queryset): diff --git a/scheduler/admin/task_admin.py b/scheduler/admin/task_admin.py index b89b7d2..e19af2c 100644 --- a/scheduler/admin/task_admin.py +++ b/scheduler/admin/task_admin.py @@ -21,6 +21,11 @@ class JobKwargInline(GenericStackedInline): fieldsets = ((None, dict(fields=("key", ("arg_type", "val")))),) +def get_message_bit(rows_updated: int) -> str: + message_bit = "1 task was" if rows_updated == 1 else f"{rows_updated} tasks were" + return message_bit + + @admin.register(Task) class TaskAdmin(admin.ModelAdmin): """TaskAdmin admin view for all task models.""" @@ -153,10 +158,9 @@ def disable_selected(self, request, queryset): obj.unschedule() rows_updated += 1 - message_bit = "1 job was" if rows_updated == 1 else f"{rows_updated} jobs were" - level = messages.WARNING if not rows_updated else messages.INFO - self.message_user(request, f"{message_bit} successfully disabled and unscheduled.", level=level) + self.message_user(request, f"{get_message_bit(rows_updated)} successfully disabled and unscheduled.", + level=level) @admin.action(description=_("Enable selected %(verbose_name_plural)s"), permissions=("change",)) def enable_selected(self, request, queryset): @@ -166,9 +170,8 @@ def enable_selected(self, request, queryset): obj.save() rows_updated += 1 - message_bit = "1 job was" if rows_updated == 1 else f"{rows_updated} jobs were" level = messages.WARNING if not rows_updated else messages.INFO - self.message_user(request, f"{message_bit} successfully enabled and scheduled.", level=level) + self.message_user(request, f"{get_message_bit(rows_updated)} successfully enabled and scheduled.", level=level) @admin.action(description="Enqueue now", permissions=("change",)) def enqueue_job_now(self, request, queryset): diff --git a/scheduler/models/migrate_util.py b/scheduler/models/migrate_util.py new file mode 100644 index 0000000..53127d4 --- /dev/null +++ b/scheduler/models/migrate_util.py @@ -0,0 +1,87 @@ +from datetime import datetime +from typing import Dict, Any, Optional + +from django.conf import settings +from django.contrib.contenttypes.models import ContentType +from django.utils import timezone + +from scheduler.models.old_scheduled_task import BaseTask +from scheduler.models.task import Task, TaskArg, TaskKwarg +from scheduler.settings import logger +from scheduler.tools import TaskType + + +def job_model_str(model_str: str) -> str: + if model_str.find("Job") == len(model_str) - 3: + return model_str[:-3] + "Task" + return model_str + + +def get_task_type(model_str: str) -> TaskType: + model_str = job_model_str(model_str) + if TaskType(model_str): + return TaskType(model_str) + if model_str == "CronTask": + return TaskType.CRON + elif model_str == "RepeatableTask": + return TaskType.REPEATABLE + elif model_str in {"ScheduledTask", "OnceTask"}: + return TaskType.ONCE + raise ValueError(f"Invalid model {model_str}") + + +def create_task_from_dict(task_dict: Dict[str, Any], recreate: bool) -> Optional[Task]: + if "new_task_id" in task_dict: + existing_task = Task.objects.filter(id=task_dict["new_task_id"]).first() + else: + existing_task = Task.objects.filter(name=task_dict["name"]).first() + task_type = get_task_type(task_dict["model"]) + if existing_task: + if recreate: + logger.info(f'Found existing job "{existing_task}, removing it to be reinserted"') + existing_task.delete() + else: + logger.info(f'Found existing job "{existing_task}", skipping') + return None + kwargs = dict(task_dict) + kwargs["task_type"] = task_type + del kwargs["model"] + del kwargs["callable_args"] + del kwargs["callable_kwargs"] + del kwargs["new_task_id"] + if kwargs.get("scheduled_time", None): + target = datetime.fromisoformat(kwargs["scheduled_time"]) + if not settings.USE_TZ and not timezone.is_naive(target): + target = timezone.make_naive(target) + kwargs["scheduled_time"] = target + model_fields = filter(lambda field: hasattr(field, 'attname'), Task._meta.get_fields()) + model_fields = set(map(lambda field: field.attname, model_fields)) + keys_to_ignore = list(filter(lambda _k: _k not in model_fields, kwargs.keys())) + for k in keys_to_ignore: + del kwargs[k] + task = Task.objects.create(**kwargs) + logger.info(f"Created task {task}") + content_type = ContentType.objects.get_for_model(task) + + for arg in task_dict["callable_args"]: + TaskArg.objects.create( + content_type=content_type, + object_id=task.id, + **arg, + ) + for arg in task_dict["callable_kwargs"]: + TaskKwarg.objects.create( + content_type=content_type, + object_id=task.id, + **arg, + ) + return task + + +def migrate(old: BaseTask) -> Optional[Task]: + old_task_dict = old.to_dict() + new_task = create_task_from_dict(old_task_dict, old_task_dict.get("new_task_id") is not None) + old.new_task_id = new_task.id + old.enabled = False + old.save() + return new_task diff --git a/scheduler/models/old_scheduled_task.py b/scheduler/models/old_scheduled_task.py index 20fdd12..7d7f408 100644 --- a/scheduler/models/old_scheduled_task.py +++ b/scheduler/models/old_scheduled_task.py @@ -299,6 +299,7 @@ def to_dict(self) -> Dict: failed_runs=getattr(self, "failed_runs", 0), last_successful_run=getattr(self, "last_successful_run", None), last_failed_run=getattr(self, "last_failed_run", None), + new_task_id=getattr(self, "new_task_id", None), ) return res @@ -396,7 +397,7 @@ class ScheduledTask(ScheduledTimeMixin, BaseTask): def ready_for_schedule(self) -> bool: return super(ScheduledTask, self).ready_for_schedule() and ( - self.scheduled_time is None or self.scheduled_time >= timezone.now() + self.scheduled_time is None or self.scheduled_time >= timezone.now() ) class Meta: diff --git a/scheduler/tests/test_old_models/test_old_models.py b/scheduler/tests/test_old_models/test_old_models.py index e8a04a0..e76764f 100644 --- a/scheduler/tests/test_old_models/test_old_models.py +++ b/scheduler/tests/test_old_models/test_old_models.py @@ -426,7 +426,7 @@ def test_admin_enable_job(self): task.refresh_from_db() self.assertTrue(task.enabled) self.assertTrue(task.is_scheduled()) - assert_response_has_msg(res, "1 job was successfully enabled and scheduled.") + assert_response_has_msg(res, "1 task was successfully enabled and scheduled.") def test_admin_disable_job(self): # arrange @@ -449,7 +449,7 @@ def test_admin_disable_job(self): task.refresh_from_db() self.assertFalse(task.is_scheduled()) self.assertFalse(task.enabled) - assert_response_has_msg(res, "1 job was successfully disabled and unscheduled.") + assert_response_has_msg(res, "1 task was successfully disabled and unscheduled.") def test_admin_single_delete(self): # arrange diff --git a/scheduler/tests/test_old_models/test_old_task_model.py b/scheduler/tests/test_old_models/test_old_task_model.py index 194b0ab..c8c9afe 100644 --- a/scheduler/tests/test_old_models/test_old_task_model.py +++ b/scheduler/tests/test_old_models/test_old_task_model.py @@ -428,7 +428,7 @@ def test_admin_enable_job(self): task.refresh_from_db() self.assertTrue(task.enabled) self.assertTrue(task.is_scheduled()) - assert_response_has_msg(res, "1 job was successfully enabled and scheduled.") + assert_response_has_msg(res, "1 task was successfully enabled and scheduled.") def test_admin_disable_job(self): # arrange @@ -451,7 +451,7 @@ def test_admin_disable_job(self): task.refresh_from_db() self.assertFalse(task.is_scheduled()) self.assertFalse(task.enabled) - assert_response_has_msg(res, "1 job was successfully disabled and unscheduled.") + assert_response_has_msg(res, "1 task was successfully disabled and unscheduled.") def test_admin_single_delete(self): # arrange From 72c5601ef3097c48a9e2a08510bbdb2f432bde49 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Fri, 29 Nov 2024 15:41:33 -0500 Subject: [PATCH 41/47] job => task --- scheduler/tests/test_task_types/test_task_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scheduler/tests/test_task_types/test_task_model.py b/scheduler/tests/test_task_types/test_task_model.py index ee8afa8..02de659 100644 --- a/scheduler/tests/test_task_types/test_task_model.py +++ b/scheduler/tests/test_task_types/test_task_model.py @@ -404,7 +404,7 @@ def test_admin_enable_job(self): task.refresh_from_db() self.assertTrue(task.enabled) self.assertTrue(task.is_scheduled()) - assert_response_has_msg(res, "1 job was successfully enabled and scheduled.") + assert_response_has_msg(res, "1 task was successfully enabled and scheduled.") def test_admin_disable_job(self): # arrange @@ -427,7 +427,7 @@ def test_admin_disable_job(self): task.refresh_from_db() self.assertFalse(task.is_scheduled()) self.assertFalse(task.enabled) - assert_response_has_msg(res, "1 job was successfully disabled and unscheduled.") + assert_response_has_msg(res, "1 task was successfully disabled and unscheduled.") def test_admin_single_delete(self): # arrange From 81f61f90dce46c3f47711270dda6d89beb6a7d11 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Fri, 29 Nov 2024 15:48:55 -0500 Subject: [PATCH 42/47] job => task --- docs/index.md | 2 ++ scheduler/admin/old_task_models.py | 3 ++- scheduler/management/commands/import.py | 4 +++- scheduler/models/migrate_util.py | 9 ++++++--- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/index.md b/docs/index.md index 9c35786..9a82508 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,6 +9,8 @@ A database backed asynchronous tasks scheduler for django. This allows remembering scheduled tasks, their parameters, etc. +!!! Important + ## Terminology ### Scheduled Task diff --git a/scheduler/admin/old_task_models.py b/scheduler/admin/old_task_models.py index 9fe1651..88c5a30 100644 --- a/scheduler/admin/old_task_models.py +++ b/scheduler/admin/old_task_models.py @@ -127,6 +127,7 @@ class TaskAdmin(admin.ModelAdmin): save_on_top = True change_form_template = "admin/scheduler/change_form.html" actions = [ + "migrate_selected", "disable_selected", "enable_selected", "enqueue_job_now", @@ -236,7 +237,7 @@ def delete_model(self, request, obj): super(TaskAdmin, self).delete_model(request, obj) @admin.action(description=_("Migrate to new Task model(s)"), permissions=("change",)) - def migrate(self, request, queryset): + def migrate_selected(self, request, queryset): rows_updated = 0 for obj in queryset.filter(enabled=True).iterator(): migrate_util.migrate(obj) diff --git a/scheduler/management/commands/import.py b/scheduler/management/commands/import.py index bd8ac14..fb21089 100644 --- a/scheduler/management/commands/import.py +++ b/scheduler/management/commands/import.py @@ -21,8 +21,10 @@ def job_model_str(model_str: str) -> str: def get_task_type(model_str: str) -> TaskType: model_str = job_model_str(model_str) - if TaskType(model_str): + try: return TaskType(model_str) + except ValueError: + pass if model_str == "CronTask": return TaskType.CRON elif model_str == "RepeatableTask": diff --git a/scheduler/models/migrate_util.py b/scheduler/models/migrate_util.py index 53127d4..cee4adb 100644 --- a/scheduler/models/migrate_util.py +++ b/scheduler/models/migrate_util.py @@ -19,8 +19,10 @@ def job_model_str(model_str: str) -> str: def get_task_type(model_str: str) -> TaskType: model_str = job_model_str(model_str) - if TaskType(model_str): + try: return TaskType(model_str) + except ValueError: + pass if model_str == "CronTask": return TaskType.CRON elif model_str == "RepeatableTask": @@ -31,9 +33,10 @@ def get_task_type(model_str: str) -> TaskType: def create_task_from_dict(task_dict: Dict[str, Any], recreate: bool) -> Optional[Task]: + existing_task = None if "new_task_id" in task_dict: existing_task = Task.objects.filter(id=task_dict["new_task_id"]).first() - else: + if existing_task is None: existing_task = Task.objects.filter(name=task_dict["name"]).first() task_type = get_task_type(task_dict["model"]) if existing_task: @@ -42,7 +45,7 @@ def create_task_from_dict(task_dict: Dict[str, Any], recreate: bool) -> Optional existing_task.delete() else: logger.info(f'Found existing job "{existing_task}", skipping') - return None + return existing_task kwargs = dict(task_dict) kwargs["task_type"] = task_type del kwargs["model"] From 763745ec5e68330c9ff346095f72bfc658dbcb0b Mon Sep 17 00:00:00 2001 From: Daniel M Date: Fri, 29 Nov 2024 15:54:26 -0500 Subject: [PATCH 43/47] job => task --- scheduler/admin/old_task_models.py | 6 +++--- scheduler/models/migrate_util.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scheduler/admin/old_task_models.py b/scheduler/admin/old_task_models.py index 88c5a30..412901e 100644 --- a/scheduler/admin/old_task_models.py +++ b/scheduler/admin/old_task_models.py @@ -175,8 +175,8 @@ class TaskAdmin(admin.ModelAdmin): def link_new_task(self, o: BaseTask) -> Optional[str]: if o.new_task_id is None: return None - url = reverse(f"admin:scheduler_task_change", args=[o.new_task_id, ]) - html = format_html(f"""{o.new_task_id}""") + url = reverse(f"admin:scheduler_task_change", args=[o.new_task_id.id, ]) + html = format_html(f"""{o.new_task_id.id}""") return html def has_add_permission(self, request: HttpRequest) -> bool: @@ -239,7 +239,7 @@ def delete_model(self, request, obj): @admin.action(description=_("Migrate to new Task model(s)"), permissions=("change",)) def migrate_selected(self, request, queryset): rows_updated = 0 - for obj in queryset.filter(enabled=True).iterator(): + for obj in queryset.iterator(): migrate_util.migrate(obj) rows_updated += 1 diff --git a/scheduler/models/migrate_util.py b/scheduler/models/migrate_util.py index cee4adb..8bc1c9b 100644 --- a/scheduler/models/migrate_util.py +++ b/scheduler/models/migrate_util.py @@ -84,7 +84,7 @@ def create_task_from_dict(task_dict: Dict[str, Any], recreate: bool) -> Optional def migrate(old: BaseTask) -> Optional[Task]: old_task_dict = old.to_dict() new_task = create_task_from_dict(old_task_dict, old_task_dict.get("new_task_id") is not None) - old.new_task_id = new_task.id + old.new_task_id = new_task old.enabled = False old.save() return new_task From 19a5e7d9b6edf9fef8628b12379111e2312c2f49 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Fri, 29 Nov 2024 16:08:02 -0500 Subject: [PATCH 44/47] update doc --- docs/index.md | 4 ++++ docs/migrate_to_v3.md | 36 ++++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + pyproject.toml | 2 +- 4 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 docs/migrate_to_v3.md diff --git a/docs/index.md b/docs/index.md index 9a82508..d7e70b7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -10,6 +10,10 @@ A database backed asynchronous tasks scheduler for django. This allows remembering scheduled tasks, their parameters, etc. !!! Important + Version 3.0.0 introduced a major design change. Instead of three separate models, there is one new `Task` model. + The goal is to simplify. + Make sure to follow [the migration guide](migrate_to_v3.md) + ## Terminology diff --git a/docs/migrate_to_v3.md b/docs/migrate_to_v3.md new file mode 100644 index 0000000..ed4cf3b --- /dev/null +++ b/docs/migrate_to_v3.md @@ -0,0 +1,36 @@ +Migration from v2 to v3 +======================= + +Version 3.0.0 introduced a major design change. Instead of three separate models, there is one new `Task` model. The +goal is to have one centralized admin view for all your scheduled tasks, regardless of the scheduling type. + +You need to migrate the scheduled tasks using the old models (`ScheduledTask`, `RepeatableTask`, `CronTask`) to the new +model. It can be done using the export/import commands provided. + +After upgrading to django-tasks-scheduler v3.0.0, you will notice you are not able to create new scheduled tasks in the +old models, that is intentional. In the next version of django-tasks-scheduler (v3.1), the old models will be deleted, +so make sure you migrate your old models. + +!!! Note + While we tested different scenarios heavily and left the code for old tasks, we could not account for all different + use cases, therefore, please [open an issue][issues] if you encounter any. + +There are two ways to migrate your existing scheduled tasks: + +# Using the admin views of the old models + +If you go to the admin view of the old models, you will notice there is a new action in the actions drop down menu for +migrating the selected tasks. Use it, and you will also have a link to the new task to compare the migration result. + +Note once you migrate using this method, the old task will be disabled automatically. + +# Export/Import management commands + +Run in your project directory: + +```shell +python manage.py export > scheduled_tasks.json +python manage.py import --filename scheduled_tasks.json +``` + +[issues]: https://github.com/django-commons/django-tasks-scheduler/issues \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index a6b8f0c..de019e9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -101,6 +101,7 @@ theme: nav: - Home: index.md + - Migrate v2 to v3: migrate_to_v3.md - Installation: installation.md - Configuration: configuration.md - Usage: usage.md diff --git a/pyproject.toml b/pyproject.toml index b5706b2..662cf3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "django-tasks-scheduler" packages = [ { include = "scheduler" }, ] -version = "2.2.0" +version = "3.0.0" description = "An async job scheduler for django using redis/valkey brokers" readme = "README.md" keywords = ["redis", "valkey", "django", "background-jobs", "job-queue", "task-queue", "redis-queue", "scheduled-jobs"] From 141206c18ed6aa8c0ebe3ad1e70a9f8db538f956 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Fri, 29 Nov 2024 16:19:11 -0500 Subject: [PATCH 45/47] Add job annotated to job-list --- scheduler/__init__.py | 6 +++++- scheduler/admin/__init__.py | 6 +++--- scheduler/admin/old_task_models.py | 2 +- scheduler/decorators.py | 3 +++ scheduler/models/__init__.py | 2 +- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/scheduler/__init__.py b/scheduler/__init__.py index c6530c8..e7010c5 100644 --- a/scheduler/__init__.py +++ b/scheduler/__init__.py @@ -2,4 +2,8 @@ __version__ = importlib.metadata.version("django-tasks-scheduler") -from .decorators import job # noqa: F401 +from .decorators import job + +__all__ = [ + "job", +] diff --git a/scheduler/admin/__init__.py b/scheduler/admin/__init__.py index 9419e52..ab85f06 100644 --- a/scheduler/admin/__init__.py +++ b/scheduler/admin/__init__.py @@ -1,5 +1,5 @@ -from .ephemeral_models import QueueAdmin, WorkerAdmin # noqa: F401 -from .old_task_models import TaskAdmin as OldTaskAdmin # noqa: F401 -from .task_admin import TaskAdmin # noqa: F401 +from .ephemeral_models import QueueAdmin, WorkerAdmin +from .old_task_models import TaskAdmin as OldTaskAdmin +from .task_admin import TaskAdmin __all__ = ["OldTaskAdmin", "QueueAdmin", "WorkerAdmin", "TaskAdmin", ] diff --git a/scheduler/admin/old_task_models.py b/scheduler/admin/old_task_models.py index 412901e..80ff764 100644 --- a/scheduler/admin/old_task_models.py +++ b/scheduler/admin/old_task_models.py @@ -175,7 +175,7 @@ class TaskAdmin(admin.ModelAdmin): def link_new_task(self, o: BaseTask) -> Optional[str]: if o.new_task_id is None: return None - url = reverse(f"admin:scheduler_task_change", args=[o.new_task_id.id, ]) + url = reverse("admin:scheduler_task_change", args=[o.new_task_id.id, ]) html = format_html(f"""{o.new_task_id.id}""") return html diff --git a/scheduler/decorators.py b/scheduler/decorators.py index 76c1a1c..c8f7e94 100644 --- a/scheduler/decorators.py +++ b/scheduler/decorators.py @@ -2,6 +2,8 @@ from .queues import get_queue, QueueNotFoundError from .rq_classes import rq_job_decorator +JOB_METHODS_LIST = list() + def job(*args, **kwargs): """ @@ -36,5 +38,6 @@ def job(*args, **kwargs): decorator = rq_job_decorator(queue, *args, **kwargs) if func: + JOB_METHODS_LIST.append(f"{func.__module__}.{func.__name__}") return decorator(func) return decorator diff --git a/scheduler/models/__init__.py b/scheduler/models/__init__.py index 04f4862..7dea098 100644 --- a/scheduler/models/__init__.py +++ b/scheduler/models/__init__.py @@ -5,6 +5,6 @@ __all__ = [ "TaskKwarg", "TaskArg", - "ScheduledTask", "RepeatableTask", "CronTask", + "BaseTask", "ScheduledTask", "RepeatableTask", "CronTask", "Queue", "Task", ] From d481ac259211d1a42779f7882396327533c09de2 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Fri, 29 Nov 2024 16:22:52 -0500 Subject: [PATCH 46/47] Add job annotated to job-list --- scheduler/tests/test_job_decorator.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scheduler/tests/test_job_decorator.py b/scheduler/tests/test_job_decorator.py index 7b78554..85a64b9 100644 --- a/scheduler/tests/test_job_decorator.py +++ b/scheduler/tests/test_job_decorator.py @@ -4,6 +4,7 @@ from scheduler import job, settings from . import test_settings # noqa +from ..decorators import JOB_METHODS_LIST from ..queues import get_queue, QueueNotFoundError @@ -34,6 +35,9 @@ class JobDecoratorTest(TestCase): def setUp(self) -> None: get_queue("default").connection.flushall() + def test_all_job_methods_registered(self): + self.assertEqual(1, len(JOB_METHODS_LIST)) + def test_job_decorator_no_params(self): test_job.delay() config = settings.SCHEDULER_CONFIG @@ -71,7 +75,6 @@ def _assert_job_with_func_and_props(self, queue_name, expected_func, expected_re def test_job_decorator_bad_queue(self): with self.assertRaises(QueueNotFoundError): - @job("bad-queue") def test_job_bad_queue(): time.sleep(1) From c00e0c8b4a05716f493ded5009c16047aa2e2824 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Sat, 30 Nov 2024 09:45:05 -0500 Subject: [PATCH 47/47] update deps --- poetry.lock | 248 ++++++++++++++++++++++++++++--------------------- pyproject.toml | 2 +- 2 files changed, 141 insertions(+), 109 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5979b58..9028713 100644 --- a/poetry.lock +++ b/poetry.lock @@ -366,73 +366,73 @@ files = [ [[package]] name = "coverage" -version = "7.6.7" +version = "7.6.8" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.9" files = [ - {file = "coverage-7.6.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:108bb458827765d538abcbf8288599fee07d2743357bdd9b9dad456c287e121e"}, - {file = "coverage-7.6.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c973b2fe4dc445cb865ab369df7521df9c27bf40715c837a113edaa2aa9faf45"}, - {file = "coverage-7.6.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c6b24007c4bcd0b19fac25763a7cac5035c735ae017e9a349b927cfc88f31c1"}, - {file = "coverage-7.6.7-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:acbb8af78f8f91b3b51f58f288c0994ba63c646bc1a8a22ad072e4e7e0a49f1c"}, - {file = "coverage-7.6.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad32a981bcdedb8d2ace03b05e4fd8dace8901eec64a532b00b15217d3677dd2"}, - {file = "coverage-7.6.7-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:34d23e28ccb26236718a3a78ba72744212aa383141961dd6825f6595005c8b06"}, - {file = "coverage-7.6.7-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e25bacb53a8c7325e34d45dddd2f2fbae0dbc230d0e2642e264a64e17322a777"}, - {file = "coverage-7.6.7-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:af05bbba896c4472a29408455fe31b3797b4d8648ed0a2ccac03e074a77e2314"}, - {file = "coverage-7.6.7-cp310-cp310-win32.whl", hash = "sha256:796c9b107d11d2d69e1849b2dfe41730134b526a49d3acb98ca02f4985eeff7a"}, - {file = "coverage-7.6.7-cp310-cp310-win_amd64.whl", hash = "sha256:987a8e3da7da4eed10a20491cf790589a8e5e07656b6dc22d3814c4d88faf163"}, - {file = "coverage-7.6.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7e61b0e77ff4dddebb35a0e8bb5a68bf0f8b872407d8d9f0c726b65dfabe2469"}, - {file = "coverage-7.6.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1a5407a75ca4abc20d6252efeb238377a71ce7bda849c26c7a9bece8680a5d99"}, - {file = "coverage-7.6.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df002e59f2d29e889c37abd0b9ee0d0e6e38c24f5f55d71ff0e09e3412a340ec"}, - {file = "coverage-7.6.7-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:673184b3156cba06154825f25af33baa2671ddae6343f23175764e65a8c4c30b"}, - {file = "coverage-7.6.7-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e69ad502f1a2243f739f5bd60565d14a278be58be4c137d90799f2c263e7049a"}, - {file = "coverage-7.6.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60dcf7605c50ea72a14490d0756daffef77a5be15ed1b9fea468b1c7bda1bc3b"}, - {file = "coverage-7.6.7-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:9c2eb378bebb2c8f65befcb5147877fc1c9fbc640fc0aad3add759b5df79d55d"}, - {file = "coverage-7.6.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3c0317288f032221d35fa4cbc35d9f4923ff0dfd176c79c9b356e8ef8ef2dff4"}, - {file = "coverage-7.6.7-cp311-cp311-win32.whl", hash = "sha256:951aade8297358f3618a6e0660dc74f6b52233c42089d28525749fc8267dccd2"}, - {file = "coverage-7.6.7-cp311-cp311-win_amd64.whl", hash = "sha256:5e444b8e88339a2a67ce07d41faabb1d60d1004820cee5a2c2b54e2d8e429a0f"}, - {file = "coverage-7.6.7-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f07ff574986bc3edb80e2c36391678a271d555f91fd1d332a1e0f4b5ea4b6ea9"}, - {file = "coverage-7.6.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:49ed5ee4109258973630c1f9d099c7e72c5c36605029f3a91fe9982c6076c82b"}, - {file = "coverage-7.6.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3e8796434a8106b3ac025fd15417315d7a58ee3e600ad4dbcfddc3f4b14342c"}, - {file = "coverage-7.6.7-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3b925300484a3294d1c70f6b2b810d6526f2929de954e5b6be2bf8caa1f12c1"}, - {file = "coverage-7.6.7-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c42ec2c522e3ddd683dec5cdce8e62817afb648caedad9da725001fa530d354"}, - {file = "coverage-7.6.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0266b62cbea568bd5e93a4da364d05de422110cbed5056d69339bd5af5685433"}, - {file = "coverage-7.6.7-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e5f2a0f161d126ccc7038f1f3029184dbdf8f018230af17ef6fd6a707a5b881f"}, - {file = "coverage-7.6.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c132b5a22821f9b143f87446805e13580b67c670a548b96da945a8f6b4f2efbb"}, - {file = "coverage-7.6.7-cp312-cp312-win32.whl", hash = "sha256:7c07de0d2a110f02af30883cd7dddbe704887617d5c27cf373362667445a4c76"}, - {file = "coverage-7.6.7-cp312-cp312-win_amd64.whl", hash = "sha256:fd49c01e5057a451c30c9b892948976f5d38f2cbd04dc556a82743ba8e27ed8c"}, - {file = "coverage-7.6.7-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46f21663e358beae6b368429ffadf14ed0a329996248a847a4322fb2e35d64d3"}, - {file = "coverage-7.6.7-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40cca284c7c310d622a1677f105e8507441d1bb7c226f41978ba7c86979609ab"}, - {file = "coverage-7.6.7-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77256ad2345c29fe59ae861aa11cfc74579c88d4e8dbf121cbe46b8e32aec808"}, - {file = "coverage-7.6.7-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87ea64b9fa52bf395272e54020537990a28078478167ade6c61da7ac04dc14bc"}, - {file = "coverage-7.6.7-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d608a7808793e3615e54e9267519351c3ae204a6d85764d8337bd95993581a8"}, - {file = "coverage-7.6.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdd94501d65adc5c24f8a1a0eda110452ba62b3f4aeaba01e021c1ed9cb8f34a"}, - {file = "coverage-7.6.7-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:82c809a62e953867cf57e0548c2b8464207f5f3a6ff0e1e961683e79b89f2c55"}, - {file = "coverage-7.6.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bb684694e99d0b791a43e9fc0fa58efc15ec357ac48d25b619f207c41f2fd384"}, - {file = "coverage-7.6.7-cp313-cp313-win32.whl", hash = "sha256:963e4a08cbb0af6623e61492c0ec4c0ec5c5cf74db5f6564f98248d27ee57d30"}, - {file = "coverage-7.6.7-cp313-cp313-win_amd64.whl", hash = "sha256:14045b8bfd5909196a90da145a37f9d335a5d988a83db34e80f41e965fb7cb42"}, - {file = "coverage-7.6.7-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:f2c7a045eef561e9544359a0bf5784b44e55cefc7261a20e730baa9220c83413"}, - {file = "coverage-7.6.7-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5dd4e4a49d9c72a38d18d641135d2fb0bdf7b726ca60a103836b3d00a1182acd"}, - {file = "coverage-7.6.7-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c95e0fa3d1547cb6f021ab72f5c23402da2358beec0a8e6d19a368bd7b0fb37"}, - {file = "coverage-7.6.7-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f63e21ed474edd23f7501f89b53280014436e383a14b9bd77a648366c81dce7b"}, - {file = "coverage-7.6.7-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ead9b9605c54d15be228687552916c89c9683c215370c4a44f1f217d2adcc34d"}, - {file = "coverage-7.6.7-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:0573f5cbf39114270842d01872952d301027d2d6e2d84013f30966313cadb529"}, - {file = "coverage-7.6.7-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:e2c8e3384c12dfa19fa9a52f23eb091a8fad93b5b81a41b14c17c78e23dd1d8b"}, - {file = "coverage-7.6.7-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:70a56a2ec1869e6e9fa69ef6b76b1a8a7ef709972b9cc473f9ce9d26b5997ce3"}, - {file = "coverage-7.6.7-cp313-cp313t-win32.whl", hash = "sha256:dbba8210f5067398b2c4d96b4e64d8fb943644d5eb70be0d989067c8ca40c0f8"}, - {file = "coverage-7.6.7-cp313-cp313t-win_amd64.whl", hash = "sha256:dfd14bcae0c94004baba5184d1c935ae0d1231b8409eb6c103a5fd75e8ecdc56"}, - {file = "coverage-7.6.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:37a15573f988b67f7348916077c6d8ad43adb75e478d0910957394df397d2874"}, - {file = "coverage-7.6.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b6cce5c76985f81da3769c52203ee94722cd5d5889731cd70d31fee939b74bf0"}, - {file = "coverage-7.6.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ab9763d291a17b527ac6fd11d1a9a9c358280adb320e9c2672a97af346ac2c"}, - {file = "coverage-7.6.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6cf96ceaa275f071f1bea3067f8fd43bec184a25a962c754024c973af871e1b7"}, - {file = "coverage-7.6.7-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aee9cf6b0134d6f932d219ce253ef0e624f4fa588ee64830fcba193269e4daa3"}, - {file = "coverage-7.6.7-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2bc3e45c16564cc72de09e37413262b9f99167803e5e48c6156bccdfb22c8327"}, - {file = "coverage-7.6.7-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:623e6965dcf4e28a3debaa6fcf4b99ee06d27218f46d43befe4db1c70841551c"}, - {file = "coverage-7.6.7-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:850cfd2d6fc26f8346f422920ac204e1d28814e32e3a58c19c91980fa74d8289"}, - {file = "coverage-7.6.7-cp39-cp39-win32.whl", hash = "sha256:c296263093f099da4f51b3dff1eff5d4959b527d4f2f419e16508c5da9e15e8c"}, - {file = "coverage-7.6.7-cp39-cp39-win_amd64.whl", hash = "sha256:90746521206c88bdb305a4bf3342b1b7316ab80f804d40c536fc7d329301ee13"}, - {file = "coverage-7.6.7-pp39.pp310-none-any.whl", hash = "sha256:0ddcb70b3a3a57581b450571b31cb774f23eb9519c2aaa6176d3a84c9fc57671"}, - {file = "coverage-7.6.7.tar.gz", hash = "sha256:d79d4826e41441c9a118ff045e4bccb9fdbdcb1d02413e7ea6eb5c87b5439d24"}, + {file = "coverage-7.6.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b39e6011cd06822eb964d038d5dff5da5d98652b81f5ecd439277b32361a3a50"}, + {file = "coverage-7.6.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:63c19702db10ad79151a059d2d6336fe0c470f2e18d0d4d1a57f7f9713875dcf"}, + {file = "coverage-7.6.8-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3985b9be361d8fb6b2d1adc9924d01dec575a1d7453a14cccd73225cb79243ee"}, + {file = "coverage-7.6.8-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:644ec81edec0f4ad17d51c838a7d01e42811054543b76d4ba2c5d6af741ce2a6"}, + {file = "coverage-7.6.8-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f188a2402f8359cf0c4b1fe89eea40dc13b52e7b4fd4812450da9fcd210181d"}, + {file = "coverage-7.6.8-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e19122296822deafce89a0c5e8685704c067ae65d45e79718c92df7b3ec3d331"}, + {file = "coverage-7.6.8-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:13618bed0c38acc418896005732e565b317aa9e98d855a0e9f211a7ffc2d6638"}, + {file = "coverage-7.6.8-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:193e3bffca48ad74b8c764fb4492dd875038a2f9925530cb094db92bb5e47bed"}, + {file = "coverage-7.6.8-cp310-cp310-win32.whl", hash = "sha256:3988665ee376abce49613701336544041f2117de7b7fbfe91b93d8ff8b151c8e"}, + {file = "coverage-7.6.8-cp310-cp310-win_amd64.whl", hash = "sha256:f56f49b2553d7dd85fd86e029515a221e5c1f8cb3d9c38b470bc38bde7b8445a"}, + {file = "coverage-7.6.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:86cffe9c6dfcfe22e28027069725c7f57f4b868a3f86e81d1c62462764dc46d4"}, + {file = "coverage-7.6.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d82ab6816c3277dc962cfcdc85b1efa0e5f50fb2c449432deaf2398a2928ab94"}, + {file = "coverage-7.6.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:13690e923a3932e4fad4c0ebfb9cb5988e03d9dcb4c5150b5fcbf58fd8bddfc4"}, + {file = "coverage-7.6.8-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4be32da0c3827ac9132bb488d331cb32e8d9638dd41a0557c5569d57cf22c9c1"}, + {file = "coverage-7.6.8-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44e6c85bbdc809383b509d732b06419fb4544dca29ebe18480379633623baafb"}, + {file = "coverage-7.6.8-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:768939f7c4353c0fac2f7c37897e10b1414b571fd85dd9fc49e6a87e37a2e0d8"}, + {file = "coverage-7.6.8-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e44961e36cb13c495806d4cac67640ac2866cb99044e210895b506c26ee63d3a"}, + {file = "coverage-7.6.8-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3ea8bb1ab9558374c0ab591783808511d135a833c3ca64a18ec927f20c4030f0"}, + {file = "coverage-7.6.8-cp311-cp311-win32.whl", hash = "sha256:629a1ba2115dce8bf75a5cce9f2486ae483cb89c0145795603d6554bdc83e801"}, + {file = "coverage-7.6.8-cp311-cp311-win_amd64.whl", hash = "sha256:fb9fc32399dca861584d96eccd6c980b69bbcd7c228d06fb74fe53e007aa8ef9"}, + {file = "coverage-7.6.8-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e683e6ecc587643f8cde8f5da6768e9d165cd31edf39ee90ed7034f9ca0eefee"}, + {file = "coverage-7.6.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1defe91d41ce1bd44b40fabf071e6a01a5aa14de4a31b986aa9dfd1b3e3e414a"}, + {file = "coverage-7.6.8-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7ad66e8e50225ebf4236368cc43c37f59d5e6728f15f6e258c8639fa0dd8e6d"}, + {file = "coverage-7.6.8-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3fe47da3e4fda5f1abb5709c156eca207eacf8007304ce3019eb001e7a7204cb"}, + {file = "coverage-7.6.8-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:202a2d645c5a46b84992f55b0a3affe4f0ba6b4c611abec32ee88358db4bb649"}, + {file = "coverage-7.6.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4674f0daa1823c295845b6a740d98a840d7a1c11df00d1fd62614545c1583787"}, + {file = "coverage-7.6.8-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:74610105ebd6f33d7c10f8907afed696e79c59e3043c5f20eaa3a46fddf33b4c"}, + {file = "coverage-7.6.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:37cda8712145917105e07aab96388ae76e787270ec04bcb9d5cc786d7cbb8443"}, + {file = "coverage-7.6.8-cp312-cp312-win32.whl", hash = "sha256:9e89d5c8509fbd6c03d0dd1972925b22f50db0792ce06324ba069f10787429ad"}, + {file = "coverage-7.6.8-cp312-cp312-win_amd64.whl", hash = "sha256:379c111d3558272a2cae3d8e57e6b6e6f4fe652905692d54bad5ea0ca37c5ad4"}, + {file = "coverage-7.6.8-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0b0c69f4f724c64dfbfe79f5dfb503b42fe6127b8d479b2677f2b227478db2eb"}, + {file = "coverage-7.6.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c15b32a7aca8038ed7644f854bf17b663bc38e1671b5d6f43f9a2b2bd0c46f63"}, + {file = "coverage-7.6.8-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63068a11171e4276f6ece913bde059e77c713b48c3a848814a6537f35afb8365"}, + {file = "coverage-7.6.8-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f4548c5ead23ad13fb7a2c8ea541357474ec13c2b736feb02e19a3085fac002"}, + {file = "coverage-7.6.8-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b4b4299dd0d2c67caaaf286d58aef5e75b125b95615dda4542561a5a566a1e3"}, + {file = "coverage-7.6.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c9ebfb2507751f7196995142f057d1324afdab56db1d9743aab7f50289abd022"}, + {file = "coverage-7.6.8-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c1b4474beee02ede1eef86c25ad4600a424fe36cff01a6103cb4533c6bf0169e"}, + {file = "coverage-7.6.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d9fd2547e6decdbf985d579cf3fc78e4c1d662b9b0ff7cc7862baaab71c9cc5b"}, + {file = "coverage-7.6.8-cp313-cp313-win32.whl", hash = "sha256:8aae5aea53cbfe024919715eca696b1a3201886ce83790537d1c3668459c7146"}, + {file = "coverage-7.6.8-cp313-cp313-win_amd64.whl", hash = "sha256:ae270e79f7e169ccfe23284ff5ea2d52a6f401dc01b337efb54b3783e2ce3f28"}, + {file = "coverage-7.6.8-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:de38add67a0af869b0d79c525d3e4588ac1ffa92f39116dbe0ed9753f26eba7d"}, + {file = "coverage-7.6.8-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b07c25d52b1c16ce5de088046cd2432b30f9ad5e224ff17c8f496d9cb7d1d451"}, + {file = "coverage-7.6.8-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62a66ff235e4c2e37ed3b6104d8b478d767ff73838d1222132a7a026aa548764"}, + {file = "coverage-7.6.8-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09b9f848b28081e7b975a3626e9081574a7b9196cde26604540582da60235fdf"}, + {file = "coverage-7.6.8-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:093896e530c38c8e9c996901858ac63f3d4171268db2c9c8b373a228f459bbc5"}, + {file = "coverage-7.6.8-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9a7b8ac36fd688c8361cbc7bf1cb5866977ece6e0b17c34aa0df58bda4fa18a4"}, + {file = "coverage-7.6.8-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:38c51297b35b3ed91670e1e4efb702b790002e3245a28c76e627478aa3c10d83"}, + {file = "coverage-7.6.8-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2e4e0f60cb4bd7396108823548e82fdab72d4d8a65e58e2c19bbbc2f1e2bfa4b"}, + {file = "coverage-7.6.8-cp313-cp313t-win32.whl", hash = "sha256:6535d996f6537ecb298b4e287a855f37deaf64ff007162ec0afb9ab8ba3b8b71"}, + {file = "coverage-7.6.8-cp313-cp313t-win_amd64.whl", hash = "sha256:c79c0685f142ca53256722a384540832420dff4ab15fec1863d7e5bc8691bdcc"}, + {file = "coverage-7.6.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3ac47fa29d8d41059ea3df65bd3ade92f97ee4910ed638e87075b8e8ce69599e"}, + {file = "coverage-7.6.8-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:24eda3a24a38157eee639ca9afe45eefa8d2420d49468819ac5f88b10de84f4c"}, + {file = "coverage-7.6.8-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4c81ed2820b9023a9a90717020315e63b17b18c274a332e3b6437d7ff70abe0"}, + {file = "coverage-7.6.8-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bd55f8fc8fa494958772a2a7302b0354ab16e0b9272b3c3d83cdb5bec5bd1779"}, + {file = "coverage-7.6.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f39e2f3530ed1626c66e7493be7a8423b023ca852aacdc91fb30162c350d2a92"}, + {file = "coverage-7.6.8-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:716a78a342679cd1177bc8c2fe957e0ab91405bd43a17094324845200b2fddf4"}, + {file = "coverage-7.6.8-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:177f01eeaa3aee4a5ffb0d1439c5952b53d5010f86e9d2667963e632e30082cc"}, + {file = "coverage-7.6.8-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:912e95017ff51dc3d7b6e2be158dedc889d9a5cc3382445589ce554f1a34c0ea"}, + {file = "coverage-7.6.8-cp39-cp39-win32.whl", hash = "sha256:4db3ed6a907b555e57cc2e6f14dc3a4c2458cdad8919e40b5357ab9b6db6c43e"}, + {file = "coverage-7.6.8-cp39-cp39-win_amd64.whl", hash = "sha256:428ac484592f780e8cd7b6b14eb568f7c85460c92e2a37cb0c0e5186e1a0d076"}, + {file = "coverage-7.6.8-pp39.pp310-none-any.whl", hash = "sha256:5c52a036535d12590c32c49209e79cabaad9f9ad8aa4cbd875b68c4d67a9cbce"}, + {file = "coverage-7.6.8.tar.gz", hash = "sha256:8b2b8503edb06822c86d82fa64a4a5cb0760bb8f31f26e138ec743f422f37cfc"}, ] [package.extras] @@ -466,51 +466,53 @@ pytz = ">2021.1" [[package]] name = "cryptography" -version = "43.0.3" +version = "44.0.0" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false -python-versions = ">=3.7" -files = [ - {file = "cryptography-43.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e"}, - {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e"}, - {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e1ce50266f4f70bf41a2c6dc4358afadae90e2a1e5342d3c08883df1675374f"}, - {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:443c4a81bb10daed9a8f334365fe52542771f25aedaf889fd323a853ce7377d6"}, - {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:74f57f24754fe349223792466a709f8e0c093205ff0dca557af51072ff47ab18"}, - {file = "cryptography-43.0.3-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9762ea51a8fc2a88b70cf2995e5675b38d93bf36bd67d91721c309df184f49bd"}, - {file = "cryptography-43.0.3-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:81ef806b1fef6b06dcebad789f988d3b37ccaee225695cf3e07648eee0fc6b73"}, - {file = "cryptography-43.0.3-cp37-abi3-win32.whl", hash = "sha256:cbeb489927bd7af4aa98d4b261af9a5bc025bd87f0e3547e11584be9e9427be2"}, - {file = "cryptography-43.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:f46304d6f0c6ab8e52770addfa2fc41e6629495548862279641972b6215451cd"}, - {file = "cryptography-43.0.3-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:8ac43ae87929a5982f5948ceda07001ee5e83227fd69cf55b109144938d96984"}, - {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:846da004a5804145a5f441b8530b4bf35afbf7da70f82409f151695b127213d5"}, - {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f996e7268af62598f2fc1204afa98a3b5712313a55c4c9d434aef49cadc91d4"}, - {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f7b178f11ed3664fd0e995a47ed2b5ff0a12d893e41dd0494f406d1cf555cab7"}, - {file = "cryptography-43.0.3-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:c2e6fc39c4ab499049df3bdf567f768a723a5e8464816e8f009f121a5a9f4405"}, - {file = "cryptography-43.0.3-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e1be4655c7ef6e1bbe6b5d0403526601323420bcf414598955968c9ef3eb7d16"}, - {file = "cryptography-43.0.3-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:df6b6c6d742395dd77a23ea3728ab62f98379eff8fb61be2744d4679ab678f73"}, - {file = "cryptography-43.0.3-cp39-abi3-win32.whl", hash = "sha256:d56e96520b1020449bbace2b78b603442e7e378a9b3bd68de65c782db1507995"}, - {file = "cryptography-43.0.3-cp39-abi3-win_amd64.whl", hash = "sha256:0c580952eef9bf68c4747774cde7ec1d85a6e61de97281f2dba83c7d2c806362"}, - {file = "cryptography-43.0.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:d03b5621a135bffecad2c73e9f4deb1a0f977b9a8ffe6f8e002bf6c9d07b918c"}, - {file = "cryptography-43.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a2a431ee15799d6db9fe80c82b055bae5a752bef645bba795e8e52687c69efe3"}, - {file = "cryptography-43.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:281c945d0e28c92ca5e5930664c1cefd85efe80e5c0d2bc58dd63383fda29f83"}, - {file = "cryptography-43.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f18c716be16bc1fea8e95def49edf46b82fccaa88587a45f8dc0ff6ab5d8e0a7"}, - {file = "cryptography-43.0.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:4a02ded6cd4f0a5562a8887df8b3bd14e822a90f97ac5e544c162899bc467664"}, - {file = "cryptography-43.0.3-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:53a583b6637ab4c4e3591a15bc9db855b8d9dee9a669b550f311480acab6eb08"}, - {file = "cryptography-43.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1ec0bcf7e17c0c5669d881b1cd38c4972fade441b27bda1051665faaa89bdcaa"}, - {file = "cryptography-43.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2ce6fae5bdad59577b44e4dfed356944fbf1d925269114c28be377692643b4ff"}, - {file = "cryptography-43.0.3.tar.gz", hash = "sha256:315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805"}, +python-versions = "!=3.9.0,!=3.9.1,>=3.7" +files = [ + {file = "cryptography-44.0.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:84111ad4ff3f6253820e6d3e58be2cc2a00adb29335d4cacb5ab4d4d34f2a123"}, + {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b15492a11f9e1b62ba9d73c210e2416724633167de94607ec6069ef724fad092"}, + {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:831c3c4d0774e488fdc83a1923b49b9957d33287de923d58ebd3cec47a0ae43f"}, + {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:761817a3377ef15ac23cd7834715081791d4ec77f9297ee694ca1ee9c2c7e5eb"}, + {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3c672a53c0fb4725a29c303be906d3c1fa99c32f58abe008a82705f9ee96f40b"}, + {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:4ac4c9f37eba52cb6fbeaf5b59c152ea976726b865bd4cf87883a7e7006cc543"}, + {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:60eb32934076fa07e4316b7b2742fa52cbb190b42c2df2863dbc4230a0a9b385"}, + {file = "cryptography-44.0.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ed3534eb1090483c96178fcb0f8893719d96d5274dfde98aa6add34614e97c8e"}, + {file = "cryptography-44.0.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f3f6fdfa89ee2d9d496e2c087cebef9d4fcbb0ad63c40e821b39f74bf48d9c5e"}, + {file = "cryptography-44.0.0-cp37-abi3-win32.whl", hash = "sha256:eb33480f1bad5b78233b0ad3e1b0be21e8ef1da745d8d2aecbb20671658b9053"}, + {file = "cryptography-44.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:abc998e0c0eee3c8a1904221d3f67dcfa76422b23620173e28c11d3e626c21bd"}, + {file = "cryptography-44.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:660cb7312a08bc38be15b696462fa7cc7cd85c3ed9c576e81f4dc4d8b2b31591"}, + {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1923cb251c04be85eec9fda837661c67c1049063305d6be5721643c22dd4e2b7"}, + {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:404fdc66ee5f83a1388be54300ae978b2efd538018de18556dde92575e05defc"}, + {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:c5eb858beed7835e5ad1faba59e865109f3e52b3783b9ac21e7e47dc5554e289"}, + {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f53c2c87e0fb4b0c00fa9571082a057e37690a8f12233306161c8f4b819960b7"}, + {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:9e6fc8a08e116fb7c7dd1f040074c9d7b51d74a8ea40d4df2fc7aa08b76b9e6c"}, + {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9abcc2e083cbe8dde89124a47e5e53ec38751f0d7dfd36801008f316a127d7ba"}, + {file = "cryptography-44.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d2436114e46b36d00f8b72ff57e598978b37399d2786fd39793c36c6d5cb1c64"}, + {file = "cryptography-44.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a01956ddfa0a6790d594f5b34fc1bfa6098aca434696a03cfdbe469b8ed79285"}, + {file = "cryptography-44.0.0-cp39-abi3-win32.whl", hash = "sha256:eca27345e1214d1b9f9490d200f9db5a874479be914199194e746c893788d417"}, + {file = "cryptography-44.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:708ee5f1bafe76d041b53a4f95eb28cdeb8d18da17e597d46d7833ee59b97ede"}, + {file = "cryptography-44.0.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:37d76e6863da3774cd9db5b409a9ecfd2c71c981c38788d3fcfaf177f447b731"}, + {file = "cryptography-44.0.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:f677e1268c4e23420c3acade68fac427fffcb8d19d7df95ed7ad17cdef8404f4"}, + {file = "cryptography-44.0.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f5e7cb1e5e56ca0933b4873c0220a78b773b24d40d186b6738080b73d3d0a756"}, + {file = "cryptography-44.0.0-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:8b3e6eae66cf54701ee7d9c83c30ac0a1e3fa17be486033000f2a73a12ab507c"}, + {file = "cryptography-44.0.0-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:be4ce505894d15d5c5037167ffb7f0ae90b7be6f2a98f9a5c3442395501c32fa"}, + {file = "cryptography-44.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:62901fb618f74d7d81bf408c8719e9ec14d863086efe4185afd07c352aee1d2c"}, + {file = "cryptography-44.0.0.tar.gz", hash = "sha256:cd4e834f340b4293430701e772ec543b0fbe6c2dea510a5286fe0acabe153a02"}, ] [package.dependencies] cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} [package.extras] -docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] -docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] -nox = ["nox"] -pep8test = ["check-sdist", "click", "mypy", "ruff"] -sdist = ["build"] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=3.0.0)"] +docstest = ["pyenchant (>=3)", "readme-renderer (>=30.0)", "sphinxcontrib-spelling (>=7.3.1)"] +nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2)"] +pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"] +sdist = ["build (>=1.0.0)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi", "cryptography-vectors (==43.0.3)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test = ["certifi (>=2024)", "cryptography-vectors (==44.0.0)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] test-randomorder = ["pytest-randomly"] [[package]] @@ -657,13 +659,13 @@ probabilistic = ["pyprobables (>=0.6,<0.7)"] [[package]] name = "fastjsonschema" -version = "2.20.0" +version = "2.21.0" description = "Fastest Python implementation of JSON schema" optional = false python-versions = "*" files = [ - {file = "fastjsonschema-2.20.0-py3-none-any.whl", hash = "sha256:5875f0b0fa7a0043a91e93a9b8f793bcbbba9691e7fd83dca95c28ba26d21f0a"}, - {file = "fastjsonschema-2.20.0.tar.gz", hash = "sha256:3d48fc5300ee96f5d116f10fe6f28d938e6008f59a6a025c2649475b87f76a23"}, + {file = "fastjsonschema-2.21.0-py3-none-any.whl", hash = "sha256:5b23b8e7c9c6adc0ecb91c03a0768cb48cd154d9159378a69c8318532e0b5cbf"}, + {file = "fastjsonschema-2.21.0.tar.gz", hash = "sha256:a02026bbbedc83729da3bfff215564b71902757f33f60089f1abae193daa4771"}, ] [package.extras] @@ -1551,13 +1553,43 @@ doc = ["sphinx"] [[package]] name = "tomli" -version = "2.1.0" +version = "2.2.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" files = [ - {file = "tomli-2.1.0-py3-none-any.whl", hash = "sha256:a5c57c3d1c56f5ccdf89f6523458f60ef716e210fc47c4cfb188c5ba473e0391"}, - {file = "tomli-2.1.0.tar.gz", hash = "sha256:3f646cae2aec94e17d04973e4249548320197cfabdf130015d023de4b74d8ab8"}, + {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, + {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8"}, + {file = "tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff"}, + {file = "tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e"}, + {file = "tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98"}, + {file = "tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744"}, + {file = "tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec"}, + {file = "tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69"}, + {file = "tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc"}, + {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, ] [[package]] @@ -1641,13 +1673,13 @@ ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==23.2.1)", "requests (>=2.31.0)" [[package]] name = "virtualenv" -version = "20.27.1" +version = "20.28.0" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.8" files = [ - {file = "virtualenv-20.27.1-py3-none-any.whl", hash = "sha256:f11f1b8a29525562925f745563bfd48b189450f61fb34c4f9cc79dd5aa32a1f4"}, - {file = "virtualenv-20.27.1.tar.gz", hash = "sha256:142c6be10212543b32c6c45d3d3893dff89112cc588b7d0879ae5a1ec03a47ba"}, + {file = "virtualenv-20.28.0-py3-none-any.whl", hash = "sha256:23eae1b4516ecd610481eda647f3a7c09aea295055337331bb4e6892ecce47b0"}, + {file = "virtualenv-20.28.0.tar.gz", hash = "sha256:2c9c3262bb8e7b87ea801d715fae4495e6032450c71d2309be9550e7364049aa"}, ] [package.dependencies] diff --git a/pyproject.toml b/pyproject.toml index 662cf3e..76601d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "django-tasks-scheduler" packages = [ { include = "scheduler" }, ] -version = "3.0.0" +version = "3.0.0b1" description = "An async job scheduler for django using redis/valkey brokers" readme = "README.md" keywords = ["redis", "valkey", "django", "background-jobs", "job-queue", "task-queue", "redis-queue", "scheduled-jobs"]