diff --git a/.github/actions/test-coverage/action.yml b/.github/actions/test-coverage/action.yml index aed62bc..7cbfaa5 100644 --- a/.github/actions/test-coverage/action.yml +++ b/.github/actions/test-coverage/action.yml @@ -17,11 +17,11 @@ outputs: runs: using: "composite" steps: - - name: Run Tests with coverage + - name: Run regular tests with coverage shell: bash run: | cd testproject - poetry run coverage run manage.py test scheduler + poetry run coverage run manage.py test --exclude-tag multiprocess scheduler - name: Coverage report id: coverage_report shell: bash diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 79d0b61..eb1c41d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,14 +27,14 @@ jobs: run: | poetry run ruff check - test: + test-regular: needs: [ 'ruff' ] runs-on: ubuntu-latest name: "Run tests ${{ matrix.python-version }}/${{ matrix.django-version }}/${{ matrix.broker }}" strategy: max-parallel: 6 matrix: - python-version: [ '3.10', '3.11', '3.12', '3.13' ] + python-version: [ '3.11', '3.12', '3.13' ] django-version: [ '5.0.7', '5.1.7' ] broker: [ 'redis', 'fakeredis', 'valkey' ] include: @@ -110,7 +110,7 @@ jobs: else export BROKER_PORT=6379 fi - poetry run python manage.py test scheduler + poetry run python manage.py test --exclude-tag multiprocess scheduler # Steps for coverage check - name: Run tests with coverage @@ -141,7 +141,7 @@ jobs: # write permission is required for auto-labeler # otherwise, read permission is required at least pull-requests: write - needs: test + needs: test-regular runs-on: ubuntu-latest steps: - uses: release-drafter/release-drafter@v6 diff --git a/SECURITY.md b/SECURITY.md index 806683f..6b2bf97 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,9 +2,9 @@ ## Supported Versions -| Version | Supported | -|-------------|--------------------| -| 2023.latest | :white_check_mark: | +| Version | Supported | +|----------|--------------------| +| 4.latest | :white_check_mark: | ## Reporting a Vulnerability diff --git a/docs/changelog.md b/docs/changelog.md index e367bb6..bf572cb 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,41 @@ # Changelog +## v4.0.0b1 🌈 + +### Breaking Changes + +This version is a full revamp of the package. The main changes are related to removing the RQ dependency. +Worker/Queue/Job are all implemented in the package itself. This change allows for more flexibility and control over +the tasks. + +Management commands: + +- `rqstats` => `scheduler_stats` +- `rqworker` => `scheduler_worker` + +Settings: + +- `SCHEDULER_CONFIG` is now a `SchedulerConfiguration` object to help IDE guide settings. +- `SCHEDULER_QUEUES` is now a list of `QueueConfiguration` objects to help IDE guide settings. +- Configuring queue to use `SSL`/`SSL_CERT_REQS`/`SOCKET_TIMEOUT` is now done using `CONNECTION_KWARGS` in + `QueueConfiguration` + ```python + SCHEDULER_QUEUES: Dict[str, QueueConfiguration] = { + 'default': QueueConfiguration( + HOST='localhost', + PORT=6379, + USERNAME='some-user', + PASSWORD='some-password', + CONNECTION_KWARGS={ # Eventual additional Broker connection arguments + 'ssl_cert_reqs': 'required', + 'ssl':True, + }, + ), + # ... + } + ``` +- For how to configure in `settings.py`, please see the [settings documentation](./configuration.md). + ## v3.0.1 🌈 ### 🐛 Bug Fixes diff --git a/docs/commands.md b/docs/commands.md index 7020cf9..3293f43 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -1,6 +1,6 @@ # Management commands -## rqworker +## `scheduler_worker` - Create a worker Create a new worker with a scheduler for specific queues by order of priority. If no queues are specified, will run on default queue only. @@ -8,11 +8,10 @@ If no queues are specified, will run on default queue only. All queues must have the same redis settings on `SCHEDULER_QUEUES`. ```shell -usage: manage.py rqworker [-h] [--pid PIDFILE] [--burst] [--name NAME] [--worker-ttl WORKER_TTL] [--max-jobs MAX_JOBS] - [--fork-job-execution FORK_JOB_EXECUTION] [--job-class JOB_CLASS] [--sentry-dsn SENTRY_DSN] [--sentry-debug] - [--sentry-ca-certs SENTRY_CA_CERTS] [--version] [-v {0,1,2,3}] [--settings SETTINGS] [--pythonpath PYTHONPATH] - [--traceback] [--no-color] [--force-color] [--skip-checks] - [queues ...] +usage: manage.py scheduler_worker [-h] [--pid PIDFILE] [--name NAME] [--worker-ttl WORKER_TTL] [--fork-job-execution FORK_JOB_EXECUTION] [--sentry-dsn SENTRY_DSN] [--sentry-debug] [--sentry-ca-certs SENTRY_CA_CERTS] [--burst] + [--max-jobs MAX_JOBS] [--max-idle-time MAX_IDLE_TIME] [--with-scheduler] [--version] [-v {0,1,2,3}] [--settings SETTINGS] [--pythonpath PYTHONPATH] [--traceback] [--no-color] [--force-color] + [--skip-checks] + [queues ...] positional arguments: queues The queues to work on, separated by space, all queues should be using the same redis @@ -20,25 +19,25 @@ positional arguments: options: -h, --help show this help message and exit --pid PIDFILE file to write the worker`s pid into - --burst Run worker in burst mode --name NAME Name of the worker --worker-ttl WORKER_TTL Default worker timeout to be used - --max-jobs MAX_JOBS Maximum number of jobs to execute before terminating worker --fork-job-execution FORK_JOB_EXECUTION Fork job execution to another process - --job-class JOB_CLASS - Jobs class to use --sentry-dsn SENTRY_DSN Sentry DSN to use --sentry-debug Enable Sentry debug mode --sentry-ca-certs SENTRY_CA_CERTS Path to CA certs file + --burst Run worker in burst mode + --max-jobs MAX_JOBS Maximum number of jobs to execute before terminating worker + --max-idle-time MAX_IDLE_TIME + Maximum number of seconds to wait for new job before terminating worker + --with-scheduler Run worker with scheduler, default to True --version Show program's version number and exit. -v {0,1,2,3}, --verbosity {0,1,2,3} Verbosity level; 0=minimal output, 1=normal output, 2=verbose output, 3=very verbose output - --settings SETTINGS The Python path to a settings module, e.g. "myproject.settings.main". If this isn't provided, the - DJANGO_SETTINGS_MODULE environment variable will be used. + --settings SETTINGS The Python path to a settings module, e.g. "myproject.settings.main". If this isn't provided, the DJANGO_SETTINGS_MODULE environment variable will be used. --pythonpath PYTHONPATH A directory to add to the Python path, e.g. "/home/djangoprojects/myproject". --traceback Raise on CommandError exceptions. @@ -49,7 +48,7 @@ options: -## export +## `export` - Export scheduled tasks Export all scheduled tasks from django db to json/yaml format. @@ -62,7 +61,7 @@ Result should be (for json): ```json [ { - "model": "ScheduledJob", + "model": "CronTaskType", "name": "Scheduled Task 1", "callable": "scheduler.tests.test_job", "callable_args": [ @@ -83,7 +82,7 @@ Result should be (for json): ] ``` -## import +## `import` - Import scheduled tasks A json/yaml that was exported using the `export` command can be imported to django. @@ -96,7 +95,7 @@ can be imported to django. python manage.py import -f {yaml,json} --filename {SOURCE-FILE} ``` -## run_job +## `run_job` - Run a job immediately Run a method in a queue immediately. @@ -104,10 +103,54 @@ Run a method in a queue immediately. python manage.py run_job {callable} {callable args ...} ``` -## delete failed jobs +## `delete_failed_jobs` - delete failed jobs Run this to empty failed jobs registry from a queue. ```shell python manage.py delete_failed_jobs ``` + +## `scheduler_stats` - Show scheduler stats + +Prints scheduler stats as a table, json, or yaml, example: + +```shell +$ python manage.py scheduler_stats + +Django-Scheduler CLI Dashboard + +-------------------------------------------------------------------------------- +| Name | Queued | Active | Finished | Canceled | Workers | +-------------------------------------------------------------------------------- +| default | 0 | 0 | 0 | 0 | 0 | +| low | 0 | 0 | 0 | 0 | 0 | +| high | 0 | 0 | 0 | 0 | 0 | +| medium | 0 | 0 | 0 | 0 | 0 | +| another | 0 | 0 | 0 | 0 | 0 | +-------------------------------------------------------------------------------- +``` + +```shell +usage: manage.py scheduler_stats [-h] [-j] [-y] [-i INTERVAL] [--version] [-v {0,1,2,3}] [--settings SETTINGS] [--pythonpath PYTHONPATH] [--traceback] [--no-color] [--force-color] [--skip-checks] + +Print statistics + +options: + -h, --help show this help message and exit + -j, --json Output statistics as JSON + -y, --yaml Output statistics as YAML + -i INTERVAL, --interval INTERVAL + Poll statistics every N seconds + --version Show program's version number and exit. + -v {0,1,2,3}, --verbosity {0,1,2,3} + Verbosity level; 0=minimal output, 1=normal output, 2=verbose output, 3=very verbose output + --settings SETTINGS The Python path to a settings module, e.g. "myproject.settings.main". If this isn't provided, the DJANGO_SETTINGS_MODULE environment variable will be used. + --pythonpath PYTHONPATH + A directory to add to the Python path, e.g. "/home/djangoprojects/myproject". + --traceback Raise on CommandError exceptions. + --no-color Don't colorize the command output. + --force-color Force colorization of the command output. + --skip-checks Skip system checks. + +``` \ No newline at end of file diff --git a/docs/configuration.md b/docs/configuration.md index da5c5a9..fb9bdbc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -5,35 +5,41 @@ All default settings for scheduler can be in one dictionary in `settings.py`: ```python -SCHEDULER_CONFIG = { - 'EXECUTIONS_IN_PAGE': 20, - 'DEFAULT_RESULT_TTL': 500, - 'DEFAULT_TIMEOUT': 300, # 5 minutes - 'SCHEDULER_INTERVAL': 10, # 10 seconds - 'BROKER': 'redis', -} -SCHEDULER_QUEUES = { - 'default': { - 'HOST': 'localhost', - 'PORT': 6379, - 'DB': 0, - 'USERNAME': 'some-user', - 'PASSWORD': 'some-password', - 'DEFAULT_TIMEOUT': 360, - 'CLIENT_KWARGS': { # Eventual additional Redis connection arguments - 'ssl_cert_reqs': None, +from typing import Dict + +from scheduler.settings_types import SchedulerConfig, Broker, QueueConfiguration, UnixSignalDeathPenalty + + +SCHEDULER_CONFIG = SchedulerConfig( + EXECUTIONS_IN_PAGE=20, + SCHEDULER_INTERVAL=10, + BROKER=Broker.REDIS, + CALLBACK_TIMEOUT=60, # Callback timeout in seconds (success/failure/stopped) + # Default values, can be overriden per task/job + DEFAULT_SUCCESS_TTL=10 * 60, # Time To Live (TTL) in seconds to keep successful job results + DEFAULT_FAILURE_TTL=365 * 24 * 60 * 60, # Time To Live (TTL) in seconds to keep job failure information + DEFAULT_JOB_TTL=10 * 60, # Time To Live (TTL) in seconds to keep job information + DEFAULT_JOB_TIMEOUT=5 * 60, # timeout (seconds) for a job + # General configuration values + DEFAULT_WORKER_TTL=10 * 60, # Time To Live (TTL) in seconds to keep worker information after last heartbeat + DEFAULT_MAINTENANCE_TASK_INTERVAL=10 * 60, # The interval to run maintenance tasks in seconds. 10 minutes. + DEFAULT_JOB_MONITORING_INTERVAL=30, # The interval to monitor jobs in seconds. + SCHEDULER_FALLBACK_PERIOD_SECS=120, # Period (secs) to wait before requiring to reacquire locks + DEATH_PENALTY_CLASS=UnixSignalDeathPenalty, +) +SCHEDULER_QUEUES: Dict[str, QueueConfiguration] = { + 'default': QueueConfiguration( + HOST='localhost', + PORT=6379, + USERNAME='some-user', + PASSWORD='some-password', + CONNECTION_KWARGS={ # Eventual additional Broker connection arguments + 'ssl_cert_reqs': 'required', + 'ssl': True, }, - 'TOKEN_VALIDATION_METHOD': None, # Method to validate auth-header - }, - 'high': { - 'URL': os.getenv('REDISTOGO_URL', 'redis://localhost:6379/0'), # If you're on Heroku - 'DEFAULT_TIMEOUT': 500, - }, - 'low': { - 'HOST': 'localhost', - 'PORT': 6379, - 'DB': 0, - } + ), + 'high': QueueConfiguration(URL=os.getenv('REDISTOGO_URL', 'redis://localhost:6379/0')), + 'low': QueueConfiguration(HOST='localhost', PORT=6379, DB=0, ASYNC=False), } ``` diff --git a/docs/drt-model.md b/docs/drt-model.md index d11c328..545658e 100644 --- a/docs/drt-model.md +++ b/docs/drt-model.md @@ -1,6 +1,6 @@ # Worker related flows -Running `python manage.py startworker --name 'X' --queues high default low` +Running `python manage.py scheduler_worker --name 'X' --queues high default low` ## Register new worker for queues ```mermaid @@ -48,8 +48,8 @@ sequenceDiagram note over worker,job: Find next job loop over queueKeys until job to run is found or all queues are empty - worker ->>+ queue: get next job id and remove it or None (zrange+zpop) - queue -->>- worker: job id / nothing + worker ->>+ queue: get next job name and remove it or None (zrange+zpop) + queue -->>- worker: job name / nothing end note over worker,job: Execute job or sleep diff --git a/docs/index.md b/docs/index.md index 7235283..c4d74c6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,12 +13,55 @@ This allows remembering scheduled tasks, their parameters, etc. The goal is to simplify. Make sure to follow [the migration guide](migrate_to_v3.md) +## Architecture and terminology -## Terminology +```mermaid +flowchart TD + subgraph Django Process + task[Scheduled Task
django-model] + end + db[(Relational
Database)] + subgraph Worker + worker[Worker
Queue listener
Job Execution] + commands[Worker
commands
Listener] + scheduler[Scheduler] + scheduler ~~~ commands ~~~ worker + end + + subgraph Broker + job[Job] + commandsChannel[Workers
Commands
Channel] + subgraph Queue + direction TB + scheduled[Scheduled Jobs] + queued[Queued jobs] + active[Active jobs] + finished[Finished jobs] + failed[Failed jobs] + canceled[Canceled jobs] + scheduled ~~~ queued ~~~ active + active ~~~ finished + active ~~~ failed + queued ~~~ canceled + end + job ~~~ commandsChannel + end + + task --> db + task -->|Create instance of executing a task| job + job -->|Queuing a job to be executed| scheduled + scheduled -.->|Queue jobs| scheduler -.-> queued + queued -.->|Worker picking up job to execute| worker + worker -.->|Moves it to active jobs| active + active -.->|Once terminated successfully| finished + active -.->|Once terminated unsuccessfully or stopped| failed + queued -...->|In case job is stopped before starting| canceled +``` ### Scheduled Task -Starting v3.0.0, django-tasks-scheduler is using a single `Task` model with different task types, the task types are: +django-tasks-scheduler is using a single `Task` django-model with different task types, the task types +are: - `ONCE` - Run the task once at a scheduled time. - `REPEATABLE` - Run the task multiple times (limited number of times or infinite times) based on a time interval. @@ -29,42 +72,52 @@ reduces the number of overall queries. An `Task` instance contains all relevant information about a task to enable the users to schedule using django-admin and track their status. -Previously, there were three different models for ScheduledTask. These exist for legacy purposes and are scheduled to -be removed. +### Job + +A job is a record in the broker, containing all information required to execute a piece of code, usually representing a +task, but not necessarily. -* `Scheduled Task` - Run a task once, on a specific time (can be immediate). -* `Repeatable Task` - Run a task multiple times (limited number of times or infinite times) based on an interval -* `Cron Task` - Run a task multiple times (limited number of times or infinite times) based on a cron string +It contains the following information: -Scheduled tasks are scheduled when the django application starts, and after a scheduled task is executed. +- Name of the job (that is unique, and passed in different queues). +- Link to the task. +- Reference to the method to be executed. +- Callbacks (In case of failure/success/stopped). +- Timeout details (for method to be executed, for callbacks) +- Successful/Failed result time-to-live. ### Queue A queue of messages between processes (main django-app process and worker usually). -This is implemented in `rq` package. +It is a collection of different registries for different purposes: -* A queue contains multiple registries for scheduled tasks, finished jobs, failed jobs, etc. +- Scheduled jobs: Jobs that are scheduled to run +- Queued jobs: Jobs waiting to be picked up by a worker to run. +- Active jobs: Jobs that are currently being executed. +- Finished jobs: Jobs that have been successfully executed +- Failed jobs: Jobs that have failed to execute or have been stopped +- Canceled jobs: Jobs that have been stopped/canceled before they were executed ### Worker -A process listening to one or more queues **for jobs to be executed**, and executing jobs queued to be -executed. +A process listening to one or more queues **for jobs to be executed**, and executing jobs queued to be executed. -### Scheduler +- A worker has a thread listening to a channel where it can get specific commands. +- A worker can have, by default, a subprocess for the scheduler. -A process listening to one or more queues for **jobs to be scheduled for execution**, and schedule them -to be executed by a worker. +### Scheduler (Worker sub-process) -This is a subprocess of worker. +A process listening to one or more queues for **jobs to be scheduled for execution**, and schedule them to be executed +by a worker (i.e., move them from scheduled-jobs registry to queued-jobs registry). -### Queued Job Execution +This is a sub-process of worker. -Once a worker listening to the queue becomes available, the job will be executed +### Job -### Scheduled Job Execution +Once a worker listening to the queue becomes available, the job will be executed. A scheduler checking the queue periodically will check whether the time the job should be executed has come, and if so, -it will queue it. +it will queue it, i.e., add it to the queued-jobs registry. * A job is considered scheduled if it is queued to be executed, or scheduled to be executed. * If there is no scheduler, the job will not be queued to run. @@ -74,24 +127,27 @@ it will queue it. ```mermaid sequenceDiagram autonumber + box DB + participant db as Database + end box Worker participant scheduler as Scheduler Process end - box DB - participant db as Database - + box Broker + participant job as Job end - box Redis queue - participant queue as Queue - participant schedule as Queue scheduled tasks + box Broker Queue + participant schedule as Scheduled jobs + participant queue as Queued jobs end loop Scheduler process - loop forever - note over scheduler, schedule: Database interaction + note over db, schedule: Database interaction scheduler ->> db: Check for enabled tasks that should be scheduled critical There are tasks to be scheduled - scheduler ->> schedule: Create a job for task that should be scheduled + scheduler ->> job: Create job for task that should be scheduled + scheduler ->> schedule: Add the job to the scheduled-jobs registry end - note over scheduler, schedule: Redis queues interaction + note over scheduler, queue: Broker queues interaction scheduler ->> schedule: check whether there are scheduled tasks that should be executed critical there are jobs that are scheduled to be executed scheduler ->> schedule: remove jobs to be scheduled @@ -109,23 +165,35 @@ sequenceDiagram box Worker participant worker as Worker Process end - box Redis queue - participant queue as Queue - participant finished as Queue finished jobs - participant failed as Queue failed jobs + box Queue + participant queue as Queued jobs + participant finished as Finished jobs + participant failed as Failed jobs + end + box Broker + participant job as Job + participant result as Result end loop Worker process - loop forever worker ->>+ queue: get the first job to be executed queue -->>- worker: A job to be executed or nothing critical There is a job to be executed - worker ->> queue: Remove job from queue + note over worker, result: There is a job to be executed + worker ->> queue: Remove job from queued registry worker ->> worker: Execute job critical Job ended successfully - worker ->> finished: Write job result + worker ->> worker: Execute successful callbacks + worker ->> finished: Move job to finished-jobs registry + worker ->> job: Update job details + worker ->> result: Write result option Job ended unsuccessfully - worker ->> failed: Write job result + worker ->> worker: Execute failure callbacks + worker ->> failed: Move job to failed-jobs registry + worker ->> job: Update job details + worker ->> result: Write result end option No job to be executed + note over worker, result: No job to be executed worker ->> worker: sleep end end @@ -141,7 +209,8 @@ Please report issues via [GitHub Issues][issues] . ## Acknowledgements -A lot of django-admin views and their tests were adopted from [django-rq][django-rq]. +- Some django-admin views and their tests were adopted from [django-rq][django-rq]. +- Worker and Queue implementation was inspired by [rq][rq]. [badge]:https://github.com/django-commons/django-tasks-scheduler/actions/workflows/test.yml/badge.svg @@ -155,4 +224,6 @@ A lot of django-admin views and their tests were adopted from [django-rq][django [issues]:https://github.com/django-commons/django-tasks-scheduler/issues -[django-rq]:https://github.com/rq/django-rq \ No newline at end of file +[django-rq]:https://github.com/rq/django-rq + +[rq]:https://github.com/rq/rq \ No newline at end of file diff --git a/docs/installation.md b/docs/installation.md index 13573b0..5eb8d5a 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -15,61 +15,64 @@ ``` 3. Configure your queues. - Add at least one Redis Queue to your `settings.py`: + Add at least one Redis Queue to your `settings.py`. + Note that the usage of `QueueConfiguration` is optional, you can use a simple dictionary, but `QueueConfiguration` + helps preventing configuration errors. ```python - import os - SCHEDULER_QUEUES = { - 'default': { - 'HOST': 'localhost', - 'PORT': 6379, - 'DB': 0, - 'USERNAME': 'some-user', - 'PASSWORD': 'some-password', - 'DEFAULT_TIMEOUT': 360, - 'CLIENT_KWARGS': { # Eventual additional Redis connection arguments - 'ssl_cert_reqs': None, - }, - }, - 'with-sentinel': { - 'SENTINELS': [('localhost', 26736), ('localhost', 26737)], - 'MASTER_NAME': 'redismaster', - 'DB': 0, - # Redis username/password - 'USERNAME': 'redis-user', - 'PASSWORD': 'secret', - 'SOCKET_TIMEOUT': 0.3, - 'CONNECTION_KWARGS': { # Eventual additional Redis connection arguments - 'ssl': True - }, - 'SENTINEL_KWARGS': { # Eventual Sentinel connection arguments - # If Sentinel also has auth, username/password can be passed here - 'username': 'sentinel-user', - 'password': 'secret', - }, - }, - 'high': { - 'URL': os.getenv('REDISTOGO_URL', 'redis://localhost:6379/0'), # If you're on Heroku - 'DEFAULT_TIMEOUT': 500, - }, - 'low': { - 'HOST': 'localhost', - 'PORT': 6379, - 'DB': 0, - } - } + from typing import Dict + from scheduler.settings_types import QueueConfiguration + + SCHEDULER_QUEUES: Dict[str, QueueConfiguration] = { + 'default': QueueConfiguration( + HOST='localhost', + PORT=6379, + USERNAME='some-user', + PASSWORD='some-password', + CONNECTION_KWARGS={ # Eventual additional Broker connection arguments + 'ssl_cert_reqs': 'required', + 'ssl': True, + }, + ), + 'with-sentinel': QueueConfiguration( + SENTINELS= [('localhost', 26736), ('localhost', 26737)], + MASTER_NAME= 'redismaster', + DB= 0, + USERNAME= 'redis-user', + PASSWORD= 'secret', + CONNECTION_KWARGS= { + 'ssl': True}, + SENTINEL_KWARGS= { + 'username': 'sentinel-user', + 'password': 'secret', + }), + 'high': QueueConfiguration(URL=os.getenv('REDISTOGO_URL', 'redis://localhost:6379/0')), + 'low': QueueConfiguration(HOST='localhost', PORT=6379, DB=0, ASYNC=False), + } ``` - + 4. Optional: Configure default values for queuing jobs from code: ```python - SCHEDULER_CONFIG = { - 'EXECUTIONS_IN_PAGE': 20, - 'DEFAULT_RESULT_TTL': 500, - 'DEFAULT_TIMEOUT': 300, # 5 minutes - 'SCHEDULER_INTERVAL': 10, # 10 seconds - 'BROKER': 'redis', # - } + from scheduler.settings_types import SchedulerConfig, Broker, UnixSignalDeathPenalty + + SCHEDULER_CONFIG = SchedulerConfig( + EXECUTIONS_IN_PAGE=20, + SCHEDULER_INTERVAL=10, + BROKER=Broker.REDIS, + CALLBACK_TIMEOUT=60, # Callback timeout in seconds (success/failure/stopped) + # Default values, can be overriden per task/job + DEFAULT_SUCCESS_TTL=10 * 60, # Time To Live (TTL) in seconds to keep successful job results + DEFAULT_FAILURE_TTL=365 * 24 * 60 * 60, # Time To Live (TTL) in seconds to keep job failure information + DEFAULT_JOB_TTL=10 * 60, # Time To Live (TTL) in seconds to keep job information + DEFAULT_JOB_TIMEOUT=5 * 60, # timeout (seconds) for a job + # General configuration values + DEFAULT_WORKER_TTL=10 * 60, # Time To Live (TTL) in seconds to keep worker information after last heartbeat + DEFAULT_MAINTENANCE_TASK_INTERVAL=10 * 60, # The interval to run maintenance tasks in seconds. 10 minutes. + DEFAULT_JOB_MONITORING_INTERVAL=30, # The interval to monitor jobs in seconds. + SCHEDULER_FALLBACK_PERIOD_SECS=120, # Period (secs) to wait before requiring to reacquire locks + DEATH_PENALTY_CLASS=UnixSignalDeathPenalty, + ) ``` - + 5. Add `scheduler.urls` to your django application `urls.py`: ```python from django.urls import path, include diff --git a/docs/usage.md b/docs/usage.md index dacd016..7749e90 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -6,7 +6,7 @@ from scheduler import job -@job +@job() def long_running_func(): pass @@ -117,11 +117,10 @@ python manage.py run_job -q {queue} -t {timeout} -r {result_ttl} {callable} {arg Create a worker to execute queued jobs on specific queues using: ```shell -python manage.py rqworker [-h] [--pid PIDFILE] [--burst] [--name NAME] [--worker-ttl WORKER_TTL] [--max-jobs MAX_JOBS] [--fork-job-execution FORK_JOB_EXECUTION] - [--job-class JOB_CLASS] [--version] [-v {0,1,2,3}] [--settings SETTINGS] [--pythonpath PYTHONPATH] [--traceback] [--no-color] [--force-color] - [--skip-checks] - [queues ...] - +usage: manage.py scheduler_worker [-h] [--pid PIDFILE] [--name NAME] [--worker-ttl WORKER_TTL] [--fork-job-execution FORK_JOB_EXECUTION] [--sentry-dsn SENTRY_DSN] [--sentry-debug] [--sentry-ca-certs SENTRY_CA_CERTS] [--burst] + [--max-jobs MAX_JOBS] [--max-idle-time MAX_IDLE_TIME] [--with-scheduler] [--version] [-v {0,1,2,3}] [--settings SETTINGS] [--pythonpath PYTHONPATH] [--traceback] [--no-color] [--force-color] + [--skip-checks] + [queues ...] ``` More information about the different parameters can be found in the [commands documentation](commands.md). @@ -129,29 +128,29 @@ More information about the different parameters can be found in the [commands do ### Running multiple workers as unix/linux services using systemd You can have multiple workers running as system services. -To have multiple rqworkers, edit the `/etc/systemd/system/rqworker@.service` +To have multiple scheduler workers, edit the `/etc/systemd/system/scheduler_worker@.service` file, make sure it ends with `@.service`, the following is example: ```ini -# /etc/systemd/system/rqworker@.service +# /etc/systemd/system/scheduler_worker@.service [Unit] -Description = rqworker daemon +Description = scheduler_worker daemon After = network.target [Service] WorkingDirectory = {{ path_to_your_project_folder } } ExecStart = /home/ubuntu/.virtualenv/{ { your_virtualenv } }/bin/python \ {{ path_to_your_project_folder } }/manage.py \ - rqworker high default low + scheduler_worker high default low # Optional -# {{user to run rqworker as}} +# {{user to run scheduler_worker as}} User = ubuntu -# {{group to run rqworker as}} +# {{group to run scheduler_worker as}} Group = www-data # Redirect logs to syslog StandardOutput = syslog StandardError = syslog -SyslogIdentifier = rqworker +SyslogIdentifier = scheduler_worker Environment = OBJC_DISABLE_INITIALIZE_FORK_SAFETY = YES Environment = LC_ALL = en_US.UTF-8 Environment = LANG = en_US.UTF-8 @@ -164,11 +163,11 @@ After you are done editing the file, reload the settings and start the new worke ```shell sudo systemctl daemon-reload -sudo systemctl start rqworker@{1..3} +sudo systemctl start scheduler_worker@{1..3} ``` You can target a specific worker using its number: ```shell -sudo systemctl stop rqworker@2 +sudo systemctl stop scheduler_worker@2 ``` \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index d3771cb..a62e609 100644 --- a/poetry.lock +++ b/poetry.lock @@ -13,7 +13,6 @@ files = [ ] [package.dependencies] -exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} idna = ">=2.8" sniffio = ">=1.1" typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} @@ -35,9 +34,6 @@ files = [ {file = "asgiref-3.8.1.tar.gz", hash = "sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590"}, ] -[package.dependencies] -typing-extensions = {version = ">=4", markers = "python_version < \"3.11\""} - [package.extras] tests = ["mypy (>=0.800)", "pytest", "pytest-asyncio"] @@ -52,7 +48,7 @@ files = [ {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, ] -markers = {main = "python_version < \"3.11.3\"", dev = "python_full_version < \"3.11.3\""} +markers = {main = "extra == \"valkey\" and python_version < \"3.11.3\"", dev = "python_full_version < \"3.11.3\""} [[package]] name = "backports-tarfile" @@ -85,10 +81,8 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "os_name == \"nt\""} -importlib-metadata = {version = ">=4.6", markers = "python_full_version < \"3.10.2\""} packaging = ">=19.1" pyproject_hooks = "*" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} [package.extras] docs = ["furo (>=2023.08.17)", "sphinx (>=7.0,<8.0)", "sphinx-argparse-cli (>=1.5)", "sphinx-autodoc-typehints (>=1.10)", "sphinx-issues (>=3.0.0)"] @@ -125,11 +119,12 @@ version = "2025.1.31" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe"}, {file = "certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651"}, ] +markers = {main = "extra == \"sentry\""} [[package]] name = "cffi" @@ -621,22 +616,6 @@ https = ["urllib3 (>=1.24.1)"] paramiko = ["paramiko"] pgp = ["gpg"] -[[package]] -name = "exceptiongroup" -version = "1.2.2" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -groups = ["dev"] -markers = "python_version < \"3.11\"" -files = [ - {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, - {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, -] - -[package.extras] -test = ["pytest (>=6)"] - [[package]] name = "fakeredis" version = "2.27.0" @@ -653,7 +632,6 @@ files = [ lupa = {version = ">=2.1,<3.0", optional = true, markers = "extra == \"lua\""} redis = {version = ">=4.3", markers = "python_full_version > \"3.8.0\""} sortedcontainers = ">=2,<3" -typing-extensions = {version = ">=4.7,<5.0", markers = "python_version < \"3.11\""} [package.extras] bf = ["pyprobables (>=0.6,<0.7)"] @@ -1229,7 +1207,6 @@ pyproject-hooks = ">=1.0.0,<2.0.0" requests = ">=2.26,<3.0" requests-toolbelt = ">=1.0.0,<2.0.0" shellingham = ">=1.5,<2.0" -tomli = {version = ">=2.0.1,<3.0.0", markers = "python_version < \"3.11\""} tomlkit = ">=0.11.4,<1.0.0" trove-classifiers = ">=2022.5.19" virtualenv = ">=20.26.6,<21.0.0" @@ -1488,7 +1465,7 @@ version = "5.2.1" description = "Python client for Redis database and key-value store" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["dev"] files = [ {file = "redis-5.2.1-py3-none-any.whl", hash = "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4"}, {file = "redis-5.2.1.tar.gz", hash = "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f"}, @@ -1538,22 +1515,6 @@ files = [ [package.dependencies] requests = ">=2.0.1,<3.0.0" -[[package]] -name = "rq" -version = "1.16.2" -description = "RQ is a simple, lightweight, library for creating background jobs, and processing them." -optional = false -python-versions = ">=3.7" -groups = ["main"] -files = [ - {file = "rq-1.16.2-py3-none-any.whl", hash = "sha256:52e619f6cb469b00e04da74305045d244b75fecb2ecaa4f26422add57d3c5f09"}, - {file = "rq-1.16.2.tar.gz", hash = "sha256:5c5b9ad5fbaf792b8fada25cc7627f4d206a9a4455aced371d4f501cc3f13b34"}, -] - -[package.dependencies] -click = ">=5" -redis = ">=3.5" - [[package]] name = "ruff" version = "0.11.2" @@ -1599,6 +1560,64 @@ files = [ cryptography = ">=2.0" jeepney = ">=0.6" +[[package]] +name = "sentry-sdk" +version = "2.24.1" +description = "Python client for Sentry (https://sentry.io)" +optional = true +python-versions = ">=3.6" +groups = ["main"] +markers = "extra == \"sentry\"" +files = [ + {file = "sentry_sdk-2.24.1-py2.py3-none-any.whl", hash = "sha256:36baa6a1128b9d98d2adc5e9b2f887eff0a6af558fc2b96ed51919042413556d"}, + {file = "sentry_sdk-2.24.1.tar.gz", hash = "sha256:8ba3c29990fa48865b908b3b9dc5ae7fa7e72407c7c9e91303e5206b32d7b8b1"}, +] + +[package.dependencies] +certifi = "*" +urllib3 = ">=1.26.11" + +[package.extras] +aiohttp = ["aiohttp (>=3.5)"] +anthropic = ["anthropic (>=0.16)"] +arq = ["arq (>=0.23)"] +asyncpg = ["asyncpg (>=0.23)"] +beam = ["apache-beam (>=2.12)"] +bottle = ["bottle (>=0.12.13)"] +celery = ["celery (>=3)"] +celery-redbeat = ["celery-redbeat (>=2)"] +chalice = ["chalice (>=1.16.0)"] +clickhouse-driver = ["clickhouse-driver (>=0.2.0)"] +django = ["django (>=1.8)"] +falcon = ["falcon (>=1.4)"] +fastapi = ["fastapi (>=0.79.0)"] +flask = ["blinker (>=1.1)", "flask (>=0.11)", "markupsafe"] +grpcio = ["grpcio (>=1.21.1)", "protobuf (>=3.8.0)"] +http2 = ["httpcore[http2] (==1.*)"] +httpx = ["httpx (>=0.16.0)"] +huey = ["huey (>=2)"] +huggingface-hub = ["huggingface_hub (>=0.22)"] +langchain = ["langchain (>=0.0.210)"] +launchdarkly = ["launchdarkly-server-sdk (>=9.8.0)"] +litestar = ["litestar (>=2.0.0)"] +loguru = ["loguru (>=0.5)"] +openai = ["openai (>=1.0.0)", "tiktoken (>=0.3.0)"] +openfeature = ["openfeature-sdk (>=0.7.1)"] +opentelemetry = ["opentelemetry-distro (>=0.35b0)"] +opentelemetry-experimental = ["opentelemetry-distro"] +pure-eval = ["asttokens", "executing", "pure_eval"] +pymongo = ["pymongo (>=3.1)"] +pyspark = ["pyspark (>=2.4.4)"] +quart = ["blinker (>=1.1)", "quart (>=0.16.1)"] +rq = ["rq (>=0.6)"] +sanic = ["sanic (>=0.8)"] +sqlalchemy = ["sqlalchemy (>=1.2)"] +starlette = ["starlette (>=0.19.1)"] +starlite = ["starlite (>=1.48)"] +statsig = ["statsig (>=0.55.3)"] +tornado = ["tornado (>=6)"] +unleash = ["UnleashClient (>=6.0.1)"] + [[package]] name = "shellingham" version = "1.5.4" @@ -1664,48 +1683,74 @@ dev = ["build", "hatch"] doc = ["sphinx"] [[package]] -name = "tomli" -version = "2.2.1" -description = "A lil' TOML parser" +name = "time-machine" +version = "2.16.0" +description = "Travel through time in your tests." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev"] -markers = "python_version < \"3.11\"" -files = [ - {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, - {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, - {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a"}, - {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee"}, - {file = "tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e"}, - {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4"}, - {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106"}, - {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8"}, - {file = "tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff"}, - {file = "tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b"}, - {file = "tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea"}, - {file = "tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8"}, - {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192"}, - {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222"}, - {file = "tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77"}, - {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6"}, - {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd"}, - {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e"}, - {file = "tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98"}, - {file = "tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4"}, - {file = "tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7"}, - {file = "tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c"}, - {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13"}, - {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281"}, - {file = "tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272"}, - {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140"}, - {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2"}, - {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744"}, - {file = "tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec"}, - {file = "tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69"}, - {file = "tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc"}, - {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, +files = [ + {file = "time_machine-2.16.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:09531af59fdfb39bfd24d28bd1e837eff5a5d98318509a31b6cfd57d27801e52"}, + {file = "time_machine-2.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:92d0b0f3c49f34dd76eb462f0afdc61ed1cb318c06c46d03e99b44ebb489bdad"}, + {file = "time_machine-2.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c29616e18e2349a8766d5b6817920fc74e39c00fa375d202231e9d525a1b882"}, + {file = "time_machine-2.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c1ceb6035a64cb00650e3ab203cf3faffac18576a3f3125c24df468b784077c7"}, + {file = "time_machine-2.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64c205ea37b8c4ba232645335fc3b75bc2d03ce30f0a34649e36cae85652ee96"}, + {file = "time_machine-2.16.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dfe92412bd11104c4f0fb2da68653e6c45b41f7217319a83a8b66ed4f20148b3"}, + {file = "time_machine-2.16.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:d5fe7a6284e3dce87ae13a25029c53542dd27a28d151f3ef362ec4dd9c3e45fd"}, + {file = "time_machine-2.16.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c0fca3025266d88d1b48be162a43b7c2d91c81cc5b3bee9f01194678ffb9969a"}, + {file = "time_machine-2.16.0-cp310-cp310-win32.whl", hash = "sha256:4149e17018af07a5756a1df84aea71e6e178598c358c860c6bfec42170fa7970"}, + {file = "time_machine-2.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:01bc257e9418980a4922de94775be42a966e1a082fb01a1635917f9afc7b84ca"}, + {file = "time_machine-2.16.0-cp310-cp310-win_arm64.whl", hash = "sha256:6895e3e84119594ab12847c928f619d40ae9cedd0755515dc154a5b5dc6edd9f"}, + {file = "time_machine-2.16.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8f936566ef9f09136a3d5db305961ef6d897b76b240c9ff4199144aed6dd4fe5"}, + {file = "time_machine-2.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5886e23ede3478ca2a3e0a641f5d09dd784dfa9e48c96e8e5e31fc4fe77b6dc0"}, + {file = "time_machine-2.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c76caf539fa4941e1817b7c482c87c65c52a1903fea761e84525955c6106fafb"}, + {file = "time_machine-2.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:298aa423e07c8b21b991782f01d7749c871c792319c2af3e9755f9ab49033212"}, + {file = "time_machine-2.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3391ae9c484736850bb44ef125cbad52fe2d1b69e42c95dc88c43af8ead2cc7"}, + {file = "time_machine-2.16.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:503e7ff507c2089699d91885fc5b9c8ff16774a7b6aff48b4dcee0c0a0685b61"}, + {file = "time_machine-2.16.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:eee7b0fc4fbab2c6585ea17606c6548be83919c70deea0865409fe9fc2d8cdce"}, + {file = "time_machine-2.16.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9db5e5b3ccdadaafa5730c2f9db44c38b013234c9ad01f87738907e19bdba268"}, + {file = "time_machine-2.16.0-cp311-cp311-win32.whl", hash = "sha256:2552f0767bc10c9d668f108fef9b487809cdeb772439ce932e74136365c69baf"}, + {file = "time_machine-2.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:12474fcdbc475aa6fe5275fe7224e685c5b9777f5939647f35980e9614ae7558"}, + {file = "time_machine-2.16.0-cp311-cp311-win_arm64.whl", hash = "sha256:ac2df0fa564356384515ed62cb6679f33f1f529435b16b0ec0f88414635dbe39"}, + {file = "time_machine-2.16.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:84788f4d62a8b1bf5e499bb9b0e23ceceea21c415ad6030be6267ce3d639842f"}, + {file = "time_machine-2.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:15ec236b6571730236a193d9d6c11d472432fc6ab54e85eac1c16d98ddcd71bf"}, + {file = "time_machine-2.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cedc989717c8b44a3881ac3d68ab5a95820448796c550de6a2149ed1525157f0"}, + {file = "time_machine-2.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9d26d79de1c63a8c6586c75967e09b0ff306aa7e944a1eaddb74595c9b1839ca"}, + {file = "time_machine-2.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:317b68b56a9c3731e0cf8886e0f94230727159e375988b36c60edce0ddbcb44a"}, + {file = "time_machine-2.16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:43e1e18279759897be3293a255d53e6b1cb0364b69d9591d0b80c51e461c94b0"}, + {file = "time_machine-2.16.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e43adb22def972a29d2b147999b56897116085777a0fea182fd93ee45730611e"}, + {file = "time_machine-2.16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0c766bea27a0600e36806d628ebc4b47178b12fcdfb6c24dc0a566a9c06bfe7f"}, + {file = "time_machine-2.16.0-cp312-cp312-win32.whl", hash = "sha256:6dae82ab647d107817e013db82223e20a9853fa88543fec853ae326382d03c2e"}, + {file = "time_machine-2.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:265462c77dc9576267c3c7f20707780a171a9fdbac93ac22e608c309efd68c33"}, + {file = "time_machine-2.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:ef768e14768eebe3bb1196c0dece8e14c1c6991605721214a0c3c68cf77eb216"}, + {file = "time_machine-2.16.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:7751bf745d54e9e8b358c0afa332815da9b8a6194b26d0fd62876ab6c4d5c9c0"}, + {file = "time_machine-2.16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1784edf173ca840ba154de6eed000b5727f65ab92972c2f88cec5c4d6349c5f2"}, + {file = "time_machine-2.16.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f5876a5682ce1f517e55d7ace2383432627889f6f7e338b961f99d684fd9e8d"}, + {file = "time_machine-2.16.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:806672529a2e255cd901f244c9033767dc1fa53466d0d3e3e49565a1572a64fe"}, + {file = "time_machine-2.16.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:667b150fedb54acdca2a4bea5bf6da837b43e6dd12857301b48191f8803ba93f"}, + {file = "time_machine-2.16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:da3ae1028af240c0c46c79adf9c1acffecc6ed1701f2863b8132f5ceae6ae4b5"}, + {file = "time_machine-2.16.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:520a814ea1b2706c89ab260a54023033d3015abef25c77873b83e3d7c1fafbb2"}, + {file = "time_machine-2.16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8243664438bb468408b29c6865958662d75e51f79c91842d2794fa22629eb697"}, + {file = "time_machine-2.16.0-cp313-cp313-win32.whl", hash = "sha256:32d445ce20d25c60ab92153c073942b0bac9815bfbfd152ce3dcc225d15ce988"}, + {file = "time_machine-2.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:f6927dda86425f97ffda36131f297b1a601c64a6ee6838bfa0e6d3149c2f0d9f"}, + {file = "time_machine-2.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:4d3843143c46dddca6491a954bbd0abfd435681512ac343169560e9bab504129"}, + {file = "time_machine-2.16.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:23c5283c01b4f80b7dfbc88f3d8088c06c301b94b7c35366be498c2d7b308549"}, + {file = "time_machine-2.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ac95ae4529d7d85b251f9cf0f961a8a408ba285875811268f469d824a3b0b15a"}, + {file = "time_machine-2.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfb76674db946a74f0ca6e3b81caa8265e35dafe9b7005c7d2b8dd5bbd3825cf"}, + {file = "time_machine-2.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d0b6ff3ccde9b16bbc694a2b5facf2d8890554f3135ff626ed1429e270e3cc4f"}, + {file = "time_machine-2.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1906ec6e26e6b803cd6aab28d420c87285b9c209ff2a69f82d12f82278f78bb"}, + {file = "time_machine-2.16.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:e46bd09c944ec7a20868abd2b83d7d7abdaf427775e9df3089b9226a122b340f"}, + {file = "time_machine-2.16.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:cac3e2b4101db296b150cb665e5461c03621e6ede6117fc9d5048c0ec96d6e7c"}, + {file = "time_machine-2.16.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1e0dcc97cfec12ae306e3036746e7631cc7ef65c31889f7264c25217d4938367"}, + {file = "time_machine-2.16.0-cp39-cp39-win32.whl", hash = "sha256:c761d32d0c5d1fe5b71ac502e1bd5edec4598a7fc6f607b9b906b98e911148ce"}, + {file = "time_machine-2.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:ddfab1c622342f2945942c5c2d6be327656980e8f2d2b2ce0c022d0aa3711361"}, + {file = "time_machine-2.16.0-cp39-cp39-win_arm64.whl", hash = "sha256:2e08a4015d5d1aab2cb46c780e85b33efcd5cbe880bb363b282a6972e617b8bb"}, + {file = "time_machine-2.16.0.tar.gz", hash = "sha256:4a99acc273d2f98add23a89b94d4dd9e14969c01214c8514bfa78e4e9364c7e2"}, ] +[package.dependencies] +python-dateutil = "*" + [[package]] name = "tomlkit" version = "0.13.2" @@ -1736,12 +1781,12 @@ version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["dev"] +markers = "python_version < \"3.13\"" files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] -markers = {main = "python_version < \"3.11\"", dev = "python_version < \"3.13\""} [[package]] name = "tzdata" @@ -1762,11 +1807,12 @@ version = "2.3.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df"}, {file = "urllib3-2.3.0.tar.gz", hash = "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d"}, ] +markers = {main = "extra == \"sentry\""} [package.extras] brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] @@ -2037,10 +2083,11 @@ cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\ cffi = ["cffi (>=1.11)"] [extras] +sentry = ["sentry-sdk"] valkey = ["valkey"] yaml = ["pyyaml"] [metadata] lock-version = "2.1" -python-versions = "^3.10" -content-hash = "91d003aa67d25bcfcdc886beefc43db1ca776b8918ce6a012025bd0b868a1ed8" +python-versions = "^3.11" +content-hash = "36dbf5925f57d2aace06194ca1477b733d868fab07c518b30521812237a00f2c" diff --git a/pyproject.toml b/pyproject.toml index 47e34cc..42f4051 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "django-tasks-scheduler" packages = [ { include = "scheduler" }, ] -version = "3.0.1" +version = "4.0.0b1" description = "An async job scheduler for django using redis/valkey brokers" readme = "README.md" keywords = ["redis", "valkey", "django", "background-jobs", "job-queue", "task-queue", "redis-queue", "scheduled-jobs"] @@ -41,15 +41,16 @@ documentation = "https://django-tasks-scheduler.readthedocs.io/en/latest/" "Funding" = "https://github.com/sponsors/cunla" [tool.poetry.dependencies] -python = "^3.10" +python = "^3.11" django = ">=5" croniter = ">=2.0" click = "^8.1" -rq = "^1.16" pyyaml = { version = "^6.0", optional = true } valkey = { version = "^6.0.2", optional = true } +sentry-sdk = { version = "^2.19", optional = true } [tool.poetry.group.dev.dependencies] +time-machine = "^2.16.0" poetry = "^2.0.1" ruff = "^0.11" coverage = "^7.6" @@ -60,6 +61,7 @@ freezegun = "^1.5" [tool.poetry.extras] yaml = ["pyyaml"] valkey = ["valkey"] +sentry = ["sentry-sdk"] [tool.ruff] line-length = 120 @@ -68,7 +70,7 @@ exclude = [ 'testproject', '.venv', '.github', - '__pycache', + '__pycache__', ] [tool.ruff.format] diff --git a/scheduler/__init__.py b/scheduler/__init__.py index e7010c5..76e7ec2 100644 --- a/scheduler/__init__.py +++ b/scheduler/__init__.py @@ -2,8 +2,11 @@ __version__ = importlib.metadata.version("django-tasks-scheduler") -from .decorators import job - __all__ = [ + "QueueConfiguration", + "SchedulerConfig", "job", ] + +from scheduler.settings_types import QueueConfiguration, SchedulerConfig +from scheduler.decorators import job \ No newline at end of file diff --git a/scheduler/admin/ephemeral_models.py b/scheduler/admin/ephemeral_models.py index 4bb1f13..06a886f 100644 --- a/scheduler/admin/ephemeral_models.py +++ b/scheduler/admin/ephemeral_models.py @@ -38,4 +38,4 @@ class WorkerAdmin(ImmutableAdmin): def changelist_view(self, request, extra_context=None): """The 'change list' admin view for this model.""" - return views.workers(request) + return views.workers_list(request) diff --git a/scheduler/admin/task_admin.py b/scheduler/admin/task_admin.py index 9308930..cf90cc1 100644 --- a/scheduler/admin/task_admin.py +++ b/scheduler/admin/task_admin.py @@ -1,13 +1,31 @@ +from typing import List + from django.contrib import admin, messages from django.contrib.contenttypes.admin import GenericStackedInline +from django.db.models import QuerySet +from django.http import HttpRequest from django.utils.translation import gettext_lazy as _ -from scheduler import tools from scheduler.broker_types import ConnectionErrorTypes +from scheduler.helpers import tools +from scheduler.helpers.queues import get_queue from scheduler.models.args import TaskArg, TaskKwarg -from scheduler.models.task import Task +from scheduler.models.task import Task, TaskType +from scheduler.redis_models import JobModel from scheduler.settings import SCHEDULER_CONFIG, logger -from scheduler.tools import get_job_executions_for_task, TaskType + + +def job_execution_of(job: JobModel, task: Task) -> bool: + return job.scheduled_task_id == task.id and job.task_type == task.task_type + + +def get_job_executions_for_task(queue_name: str, scheduled_task: Task) -> List[JobModel]: + queue = get_queue(queue_name) + job_list: List[JobModel] = JobModel.get_many(queue.get_all_job_names(), connection=queue.connection) + res = sorted(list(filter(lambda j: job_execution_of(j, scheduled_task), job_list)), + key=lambda j: j.created_at, + reverse=True) + return res class JobArgInline(GenericStackedInline): @@ -32,10 +50,7 @@ class TaskAdmin(admin.ModelAdmin): """TaskAdmin admin view for all task models.""" class Media: - js = ( - "admin/js/jquery.init.js", - "admin/js/select-fields.js", - ) + js = ("admin/js/jquery.init.js", "admin/js/select-fields.js",) save_on_top = True change_form_template = "admin/scheduler/change_form.html" @@ -52,7 +67,7 @@ class Media: list_display = ( "enabled", "name", - "job_id", + "job_name", "function_string", "is_scheduled", "queue", @@ -65,7 +80,7 @@ class Media: ) list_display_links = ("name",) readonly_fields = ( - "job_id", + "job_name", "successful_runs", "last_successful_run", "failed_runs", @@ -75,32 +90,18 @@ class Media: fieldsets = ( ( None, - dict( - fields=( - "name", - "callable", - "task_type", - ("enabled", "timeout", "result_ttl"), - ) - ), - ), - ( - None, - dict(fields=("scheduled_time",), classes=("tasktype-OnceTaskType",)), - ), - ( - None, - dict(fields=("cron_string",), classes=("tasktype-CronTaskType",)), - ), - ( - None, - dict(fields=("interval", "interval_unit", "repeat"), classes=("tasktype-RepeatableTaskType",)), - ), - (_("RQ Settings"), dict(fields=(("queue", "at_front"), "job_id"))), - ( - _("Previous runs info"), - dict(fields=(("successful_runs", "last_successful_run"), ("failed_runs", "last_failed_run"))), + dict(fields=( + "name", "callable", + ("enabled", "timeout", "result_ttl"), + "task_type", + )), ), + (None, dict(fields=("scheduled_time",), classes=("tasktype-OnceTaskType",)),), + (None, dict(fields=("cron_string",), classes=("tasktype-CronTaskType",)),), + (None, dict(fields=(("interval", "interval_unit",), "repeat"), classes=("tasktype-RepeatableTaskType",)),), + (_("Queue settings"), dict(fields=(("queue", "at_front"), "job_name"))), + (_("Previous runs info"), + dict(fields=(("successful_runs", "last_successful_run"), ("failed_runs", "last_failed_run"))),), ) @admin.display(description="Schedule") @@ -109,16 +110,16 @@ def task_schedule(self, o: Task) -> str: return f"Run once: {o.scheduled_time:%Y-%m-%d %H:%M:%S}" elif o.task_type == TaskType.CRON.value: return f"Cron: {o.cron_string}" - elif o.task_type == TaskType.REPEATABLE.value: + else: # if o.task_type == TaskType.REPEATABLE.value: if o.interval is None or o.interval_unit is None: return "" - return "Repeatable: {} {}".format(o.interval, o.get_interval_unit_display()) + return f"Repeatable: {o.interval} {o.get_interval_unit_display()}" @admin.display(description="Next run") def next_run(self, o: Task) -> str: return tools.get_next_cron_time(o.cron_string) - def change_view(self, request, object_id, form_url="", extra_context=None): + def change_view(self, request: HttpRequest, object_id, form_url="", extra_context=None): extra = extra_context or {} obj = self.get_object(request, object_id) try: @@ -142,17 +143,17 @@ def change_view(self, request, object_id, form_url="", extra_context=None): return super(TaskAdmin, self).change_view(request, object_id, form_url, extra_context=extra) - def delete_queryset(self, request, queryset): + def delete_queryset(self, request: HttpRequest, queryset: QuerySet) -> None: for job in queryset: job.unschedule() super(TaskAdmin, self).delete_queryset(request, queryset) - def delete_model(self, request, obj): + def delete_model(self, request: HttpRequest, obj: Task) -> None: obj.unschedule() super(TaskAdmin, self).delete_model(request, obj) @admin.action(description=_("Disable selected %(verbose_name_plural)s"), permissions=("change",)) - def disable_selected(self, request, queryset): + def disable_selected(self, request: HttpRequest, queryset: QuerySet) -> None: rows_updated = 0 for obj in queryset.filter(enabled=True).iterator(): obj.enabled = False @@ -165,7 +166,7 @@ def disable_selected(self, request, queryset): ) @admin.action(description=_("Enable selected %(verbose_name_plural)s"), permissions=("change",)) - def enable_selected(self, request, queryset): + def enable_selected(self, request: HttpRequest, queryset: QuerySet) -> None: rows_updated = 0 for obj in queryset.filter(enabled=False).iterator(): obj.enabled = True @@ -176,7 +177,7 @@ def enable_selected(self, request, queryset): self.message_user(request, f"{get_message_bit(rows_updated)} successfully enabled and scheduled.", level=level) @admin.action(description="Enqueue now", permissions=("change",)) - def enqueue_job_now(self, request, queryset): + def enqueue_job_now(self, request: HttpRequest, queryset: QuerySet) -> None: task_names = [] for task in queryset: task.enqueue_to_run() diff --git a/scheduler/broker_types.py b/scheduler/broker_types.py index eb779fb..3801b73 100644 --- a/scheduler/broker_types.py +++ b/scheduler/broker_types.py @@ -1,6 +1,7 @@ # This is a helper module to obfuscate types used by different broker implementations. from collections import namedtuple -from typing import Union, Dict, Tuple +from typing import Any, Callable, TypeVar, Union +from typing import Dict, Tuple import redis @@ -11,20 +12,32 @@ valkey.Valkey = redis.Redis valkey.StrictValkey = redis.StrictRedis -from scheduler.settings import Broker +from scheduler.settings_types import Broker ConnectionErrorTypes = (redis.ConnectionError, valkey.ConnectionError) ResponseErrorTypes = (redis.ResponseError, valkey.ResponseError) +TimeoutErrorTypes = (redis.TimeoutError, valkey.TimeoutError) +WatchErrorTypes = (redis.WatchError, valkey.WatchError) ConnectionType = Union[redis.Redis, valkey.Valkey] PipelineType = Union[redis.client.Pipeline, valkey.client.Pipeline] SentinelType = Union[redis.sentinel.Sentinel, valkey.sentinel.Sentinel] +FunctionReferenceType = TypeVar("FunctionReferenceType", str, Callable[..., Any]) -BrokerMetaDataType = namedtuple("BrokerMetaDataType", ["connection_type", "sentinel_type", "ssl_prefix"]) +BrokerMetaDataType = namedtuple("BrokerMetaDataType", ["connection_type", "sentinel_type"]) BrokerMetaData: Dict[Tuple[Broker, bool], BrokerMetaDataType] = { - # Map of (Broker, Strict flag) => Connection Class, Sentinel Class, SSL Connection Prefix - (Broker.REDIS, False): BrokerMetaDataType(redis.Redis, redis.sentinel.Sentinel, "rediss"), - (Broker.VALKEY, False): BrokerMetaDataType(valkey.Valkey, valkey.sentinel.Sentinel, "valkeys"), - (Broker.REDIS, True): BrokerMetaDataType(redis.StrictRedis, redis.sentinel.Sentinel, "rediss"), - (Broker.VALKEY, True): BrokerMetaDataType(valkey.StrictValkey, valkey.sentinel.Sentinel, "valkeys"), + # Map of (Broker, Strict flag) => Connection Class, Sentinel Class + (Broker.REDIS, False): BrokerMetaDataType(redis.Redis, redis.sentinel.Sentinel), + (Broker.VALKEY, False): BrokerMetaDataType(valkey.Valkey, valkey.sentinel.Sentinel), + (Broker.REDIS, True): BrokerMetaDataType(redis.StrictRedis, redis.sentinel.Sentinel), + (Broker.VALKEY, True): BrokerMetaDataType(valkey.StrictValkey, valkey.sentinel.Sentinel), } + +MODEL_NAMES = [ + "Task", +] +TASK_TYPES = ["OnceTaskType", "RepeatableTaskType", "CronTaskType"] + + +def is_pipeline(conn: ConnectionType) -> bool: + return isinstance(conn, redis.client.Pipeline) or isinstance(conn, valkey.client.Pipeline) diff --git a/scheduler/decorators.py b/scheduler/decorators.py index c8f7e94..936db89 100644 --- a/scheduler/decorators.py +++ b/scheduler/decorators.py @@ -1,43 +1,96 @@ -from scheduler import settings -from .queues import get_queue, QueueNotFoundError -from .rq_classes import rq_job_decorator +from functools import wraps +from typing import Any, Callable, Dict, Optional, Union + +from scheduler.broker_types import ConnectionType +from scheduler.helpers.callback import Callback JOB_METHODS_LIST = list() -def job(*args, **kwargs): - """ - The same as rq package's job decorator, but it automatically works out - the ``connection`` argument from SCHEDULER_QUEUES. +class job: + def __init__( + self, + queue: Union["Queue", str, None] = None, # noqa: F821 + connection: Optional[ConnectionType] = None, + timeout: Optional[int] = None, + result_ttl: Optional[int] = None, + job_info_ttl: Optional[int] = None, + at_front: bool = False, + meta: Optional[Dict[Any, Any]] = None, + description: Optional[str] = None, + on_failure: Optional[Union["Callback", Callable[..., Any]]] = None, + on_success: Optional[Union["Callback", Callable[..., Any]]] = None, + on_stopped: Optional[Union["Callback", Callable[..., Any]]] = None, + ): + """A decorator that adds a ``delay`` method to the decorated function, which in turn creates a RQ job when + called. Accepts a required ``queue`` argument that can be either a ``Queue`` instance or a string + denoting the queue name. For example:: + - And also, it allows simplified ``@job`` syntax to put a job into the default queue. + >>> @job(queue='default') + >>> def simple_add(x, y): + >>> return x + y + >>> ... + >>> # Puts `simple_add` function into queue + >>> simple_add.delay(1, 2) - """ - if len(args) == 0: - func = None - queue = "default" - else: - if callable(args[0]): - func = args[0] + :param queue: The queue to use, can be the Queue class itself, or the queue name (str) + :type queue: Union['Queue', str] + :param connection: Broker Connection + :param timeout: Job timeout + :param result_ttl: Result time to live + :param job_info_ttl: Time to live for job info + :param at_front: Whether to enqueue the job at front of the queue + :param meta: Arbitraty metadata about the job + :param description: Job description + :param on_failure: Callable to run on failure + :param on_success: Callable to run on success + :param on_stopped: Callable to run when stopped + """ + from scheduler.helpers.queues import get_queue + if queue is None: queue = "default" - else: - func = None - queue = args[0] - args = args[1:] - - if isinstance(queue, str): - try: - queue = get_queue(queue) - if "connection" not in kwargs: - kwargs["connection"] = queue.connection - except KeyError: - raise QueueNotFoundError(f"Queue {queue} does not exist") - - kwargs.setdefault("result_ttl", settings.SCHEDULER_CONFIG.DEFAULT_RESULT_TTL) - kwargs.setdefault("timeout", settings.SCHEDULER_CONFIG.DEFAULT_TIMEOUT) - - decorator = rq_job_decorator(queue, *args, **kwargs) - if func: - JOB_METHODS_LIST.append(f"{func.__module__}.{func.__name__}") - return decorator(func) - return decorator + self.queue = get_queue(queue) if isinstance(queue, str) else queue + self.connection = connection + self.timeout = timeout + self.result_ttl = result_ttl + self.job_info_ttl = job_info_ttl + self.meta = meta + self.at_front = at_front + self.description = description + self.on_success = on_success + self.on_failure = on_failure + self.on_stopped = on_stopped + + def __call__(self, f): + @wraps(f) + def delay(*args, **kwargs): + from scheduler.helpers.queues import get_queue + + queue = get_queue(self.queue) if isinstance(self.queue, str) else self.queue + + job_name = kwargs.pop("job_name", None) + at_front = kwargs.pop("at_front", False) + + if not at_front: + at_front = self.at_front + + return queue.create_and_enqueue_job( + f, + args=args, + kwargs=kwargs, + timeout=self.timeout, + result_ttl=self.result_ttl, + job_info_ttl=self.job_info_ttl, + name=job_name, + at_front=at_front, + meta=self.meta, + description=self.description, + on_failure=self.on_failure, + on_success=self.on_success, + on_stopped=self.on_stopped, + ) + + JOB_METHODS_LIST.append(f"{f.__module__}.{f.__name__}") + f.delay = delay + return f diff --git a/scheduler/helpers/__init__.py b/scheduler/helpers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scheduler/helpers/callback.py b/scheduler/helpers/callback.py new file mode 100644 index 0000000..0d812b5 --- /dev/null +++ b/scheduler/helpers/callback.py @@ -0,0 +1,77 @@ +import importlib +import inspect +from typing import Union, Callable, Any, Optional + +from scheduler.timeouts import JobTimeoutException + + +class CallbackSetupError(Exception): + pass + + +class Callback: + def __init__(self, func: Union[str, Callable[..., Any]], timeout: Optional[int] = None): + from scheduler.settings import SCHEDULER_CONFIG + + self.timeout = timeout or SCHEDULER_CONFIG.CALLBACK_TIMEOUT + if not isinstance(self.timeout, int) or self.timeout < 0: + raise CallbackSetupError(f"Callback `timeout` must be a positive int, but received {self.timeout}") + if not isinstance(func, str) and not inspect.isfunction(func) and not inspect.isbuiltin(func): + raise CallbackSetupError(f"Callback `func` must be a string or function, received {func}") + if isinstance(func, str): + func = _import_attribute(func) + self.func: Callable[..., Any] = func + + @property + def name(self) -> str: + return f"{self.func.__module__}.{self.func.__qualname__}" + + def __call__(self, *args, **kwargs): + from scheduler.settings import SCHEDULER_CONFIG + + with SCHEDULER_CONFIG.DEATH_PENALTY_CLASS(self.timeout, JobTimeoutException): + return self.func(*args, **kwargs) + + +def _import_attribute(name: str) -> Callable[..., Any]: + """Returns an attribute from a dotted path name. Example: `path.to.func`. + + When the attribute we look for is a staticmethod, module name in its dotted path is not the last-before-end word + E.g.: package_a.package_b.module_a.ClassA.my_static_method + Thus we remove the bits from the end of the name until we can import it + + :param name: The name (reference) to the path. + :raises ValueError: If no module is found or invalid attribute name. + :returns: An attribute (normally a Callable) + """ + name_bits = name.split(".") + module_name_bits, attribute_bits = name_bits[:-1], [name_bits[-1]] + module = None + while len(module_name_bits) > 0: + try: + module_name = ".".join(module_name_bits) + module = importlib.import_module(module_name) + break + except ImportError: + attribute_bits.insert(0, module_name_bits.pop()) + + if module is None: # maybe it's a builtin + try: + return __builtins__[name] + except KeyError: + raise CallbackSetupError(f"Invalid attribute name: {name}") + + attribute_name = ".".join(attribute_bits) + if hasattr(module, attribute_name): + return getattr(module, attribute_name) + # staticmethods + attribute_name = attribute_bits.pop() + attribute_owner_name = ".".join(attribute_bits) + try: + attribute_owner = getattr(module, attribute_owner_name) + except: # noqa + raise CallbackSetupError(f"Invalid attribute name: {attribute_name}") + + if not hasattr(attribute_owner, attribute_name): + raise CallbackSetupError(f"Invalid attribute name: {name}") + return getattr(attribute_owner, attribute_name) diff --git a/scheduler/helpers/queues/__init__.py b/scheduler/helpers/queues/__init__.py new file mode 100644 index 0000000..af2f09e --- /dev/null +++ b/scheduler/helpers/queues/__init__.py @@ -0,0 +1,11 @@ +__all__ = [ + "Queue", + "InvalidJobOperation", + "get_queue", + "get_all_workers", + "get_queues", + "perform_job", +] + +from .getters import get_queue, get_all_workers, get_queues +from .queue_logic import Queue, InvalidJobOperation, perform_job diff --git a/scheduler/helpers/queues/getters.py b/scheduler/helpers/queues/getters.py new file mode 100644 index 0000000..4f7eb5b --- /dev/null +++ b/scheduler/helpers/queues/getters.py @@ -0,0 +1,104 @@ +from typing import List, Set + +from scheduler.broker_types import ConnectionErrorTypes, BrokerMetaData +from scheduler.redis_models.worker import WorkerModel +from scheduler.settings import ( + SCHEDULER_CONFIG, + get_queue_names, + get_queue_configuration, + QueueConfiguration, + logger, + Broker, +) +from .queue_logic import Queue + + +class QueueConnectionDiscrepancyError(Exception): + pass + + +_BAD_QUEUE_CONFIGURATION = set() + + +def _get_connection(config: QueueConfiguration, use_strict_broker=False): + """Returns a Broker connection to use based on parameters in SCHEDULER_QUEUES""" + """ + Returns a redis connection from a connection config + """ + if SCHEDULER_CONFIG.BROKER == Broker.FAKEREDIS: + import fakeredis + + broker_cls = fakeredis.FakeRedis if not use_strict_broker else fakeredis.FakeStrictRedis + else: + broker_cls = BrokerMetaData[(SCHEDULER_CONFIG.BROKER, use_strict_broker)].connection_type + if config.URL: + return broker_cls.from_url(config.URL, db=config.DB, **(config.CONNECTION_KWARGS or {})) + if config.UNIX_SOCKET_PATH: + return broker_cls(unix_socket_path=config.UNIX_SOCKET_PATH, db=config.DB) + + if config.SENTINELS: + connection_kwargs = { + "db": config.DB, + "password": config.PASSWORD, + "username": config.USERNAME, + } + connection_kwargs.update(config.CONNECTION_KWARGS or {}) + sentinel_kwargs = config.SENTINEL_KWARGS or {} + SentinelClass = BrokerMetaData[(SCHEDULER_CONFIG.BROKER, use_strict_broker)].sentinel_type + sentinel = SentinelClass(config.SENTINELS, sentinel_kwargs=sentinel_kwargs, **connection_kwargs) + return sentinel.master_for( + service_name=config.MASTER_NAME, + redis_class=broker_cls, + ) + + return broker_cls( + host=config.HOST, + port=config.PORT, + db=config.DB, + username=config.USERNAME, + password=config.PASSWORD, + **(config.CONNECTION_KWARGS or {}), + ) + + +def get_queue(name="default") -> Queue: + """Returns an DjangoQueue using parameters defined in `SCHEDULER_QUEUES`""" + queue_settings = get_queue_configuration(name) + is_async = queue_settings.ASYNC + connection = _get_connection(queue_settings) + return Queue(name=name, connection=connection, is_async=is_async) + + +def get_all_workers() -> Set[WorkerModel]: + queue_names = get_queue_names() + + workers_set: Set[WorkerModel] = set() + for queue_name in queue_names: + if queue_name in _BAD_QUEUE_CONFIGURATION: + continue + connection = _get_connection(get_queue_configuration(queue_name)) + try: + curr_workers: Set[WorkerModel] = set(WorkerModel.all(connection=connection)) + workers_set.update(curr_workers) + except ConnectionErrorTypes as e: + logger.error(f"Could not connect for queue {queue_name}: {e}") + _BAD_QUEUE_CONFIGURATION.add(queue_name) + return workers_set + + +def get_queues(*queue_names: str) -> List[Queue]: + """Return queue instances from specified queue names. All instances must use the same Broker configuration.""" + + queue_config = get_queue_configuration(queue_names[0]) + queues = [get_queue(queue_names[0])] + # perform consistency checks while building return list + for queue_name in queue_names[1:]: + curr_queue_config = get_queue_configuration(queue_name) + if not queue_config.same_connection_params(curr_queue_config): + raise QueueConnectionDiscrepancyError( + f'Queues must have the same broker connection. "{queue_name}" and "{queue_names[0]}" have different connection settings' + ) + queue = get_queue(queue_name) + queues.append(queue) + + return queues diff --git a/scheduler/helpers/queues/queue_logic.py b/scheduler/helpers/queues/queue_logic.py new file mode 100644 index 0000000..d9fc53d --- /dev/null +++ b/scheduler/helpers/queues/queue_logic.py @@ -0,0 +1,481 @@ +import asyncio +import sys +import traceback +from datetime import datetime +from functools import total_ordering +from typing import Dict, List, Optional, Tuple, Union, Self, Any + +from redis import WatchError + +from scheduler.broker_types import ConnectionType, FunctionReferenceType +from scheduler.helpers.callback import Callback +from scheduler.helpers.utils import utcnow, current_timestamp +from scheduler.redis_models import ( + JobNamesRegistry, + FinishedJobRegistry, + ActiveJobRegistry, + FailedJobRegistry, + CanceledJobRegistry, + ScheduledJobRegistry, + QueuedJobRegistry, + NoSuchJobError, +) +from scheduler.redis_models import JobStatus, SchedulerLock, Result, ResultType, JobModel +from scheduler.settings import logger, SCHEDULER_CONFIG + + +class InvalidJobOperation(Exception): + pass + + +def perform_job(job_model: JobModel, connection: ConnectionType) -> Any: # noqa + """The main execution method. Invokes the job function with the job arguments. + + :returns: The job's return value + """ + job_model.persist(connection=connection) + _job_stack.append(job_model) + + try: + result = job_model.func(*job_model.args, **job_model.kwargs) + if asyncio.iscoroutine(result): + loop = asyncio.new_event_loop() + coro_result = loop.run_until_complete(result) + result = coro_result + if job_model.success_callback: + job_model.success_callback(job_model, connection, result) # type: ignore + return result + except: + if job_model.failure_callback: + job_model.failure_callback(job_model, connection, *sys.exc_info()) # type: ignore + raise + finally: + assert job_model is _job_stack.pop() + + +_job_stack = [] + + +@total_ordering +class Queue: + REGISTRIES = dict( + finished="finished_job_registry", + failed="failed_job_registry", + scheduled="scheduled_job_registry", + active="active_job_registry", + canceled="canceled_job_registry", + queued="queued_job_registry", + ) + + def __init__(self, connection: Optional[ConnectionType], name: str, is_async: bool = True) -> None: + """Initializes a Queue object. + + :param name: The queue name + :param connection: Broker connection + :param is_async: Whether jobs should run "async" (using the worker). + """ + self.connection = connection + self.name = name + self._is_async = is_async + self.queued_job_registry = QueuedJobRegistry(connection=self.connection, name=self.name) + self.active_job_registry = ActiveJobRegistry(connection=self.connection, name=self.name) + self.failed_job_registry = FailedJobRegistry(connection=self.connection, name=self.name) + self.finished_job_registry = FinishedJobRegistry(connection=self.connection, name=self.name) + self.scheduled_job_registry = ScheduledJobRegistry(connection=self.connection, name=self.name) + self.canceled_job_registry = CanceledJobRegistry(connection=self.connection, name=self.name) + + def __len__(self): + return self.count + + @property + def scheduler_pid(self) -> int: + lock = SchedulerLock(self.name) + pid = lock.value(self.connection) + return int(pid.decode()) if pid is not None else None + + def clean_registries(self, timestamp: Optional[float] = None) -> None: + """Remove abandoned jobs from registry and add them to FailedJobRegistry. + + Removes jobs with an expiry time earlier than current_timestamp, specified as seconds since the Unix epoch. + Removed jobs are added to the global failed job queue. + """ + before_score = timestamp or current_timestamp() + started_jobs: List[Tuple[str, float]] = self.active_job_registry.get_job_names_before( + self.connection, before_score + ) + + with self.connection.pipeline() as pipeline: + for job_name, job_score in started_jobs: + job = JobModel.get(job_name, connection=self.connection) + if job is None or job.failure_callback is None or job_score + job.timeout > before_score: + continue + + logger.debug(f"Running failure callbacks for {job.name}") + try: + job.failure_callback(job, self.connection, traceback.extract_stack()) + except Exception: # noqa + logger.exception(f"Job {self.name}: error while executing failure callback") + raise + + + else: + logger.warning( + f"{self.__class__.__name__} cleanup: Moving job to {self.failed_job_registry.key} " + f"(due to AbandonedJobError)" + ) + job.set_status(JobStatus.FAILED, connection=pipeline) + exc_string = ( + f"Moved to {self.failed_job_registry.key}, due to AbandonedJobError, at {datetime.now()}" + ) + job.save(connection=pipeline) + job.expire(ttl=-1, connection=pipeline) + score = current_timestamp() + SCHEDULER_CONFIG.DEFAULT_FAILURE_TTL + Result.create( + connection=pipeline, + job_name=job.name, + worker_name=job.worker_name, + _type=ResultType.FAILED, + ttl=SCHEDULER_CONFIG.DEFAULT_FAILURE_TTL, + exc_string=exc_string, + ) + self.failed_job_registry.add(pipeline, job.name, score) + job.save(connection=pipeline) + job.expire(connection=pipeline, ttl=SCHEDULER_CONFIG.DEFAULT_FAILURE_TTL) + + for registry in self.REGISTRIES.values(): + getattr(self, registry).cleanup(connection=self.connection, timestamp=before_score) + pipeline.execute() + + def first_queued_job_name(self) -> Optional[str]: + return self.queued_job_registry.get_first() + + def empty(self): + """Removes all queued jobs from the queue.""" + queued_jobs_count = self.queued_job_registry.count(connection=self.connection) + with self.connection.pipeline() as pipe: + for offset in range(0, queued_jobs_count, 1000): + job_names = self.queued_job_registry.all(offset, 1000) + for job_name in job_names: + self.queued_job_registry.delete(connection=pipe, job_name=job_name) + JobModel.delete_many(job_names, connection=pipe) + pipe.execute() + + @property + def count(self) -> int: + """Returns a count of all messages in the queue.""" + res = 0 + for registry in self.REGISTRIES.values(): + res += getattr(self, registry).count(connection=self.connection) + return res + + def get_registry(self, name: str) -> Union[None, JobNamesRegistry]: + name = name.lower() + if name in Queue.REGISTRIES: + return getattr(self, Queue.REGISTRIES[name]) + return None + + def get_all_job_names(self) -> List[str]: + res = list() + res.extend(self.queued_job_registry.all()) + res.extend(self.finished_job_registry.all()) + res.extend(self.active_job_registry.all()) + res.extend(self.failed_job_registry.all()) + res.extend(self.scheduled_job_registry.all()) + res.extend(self.canceled_job_registry.all()) + return res + + def get_all_jobs(self) -> List[JobModel]: + job_names = self.get_all_job_names() + return JobModel.get_many(job_names, connection=self.connection) + + def create_and_enqueue_job( + self, + func: FunctionReferenceType, + args: Union[Tuple, List, None] = None, + kwargs: Optional[Dict] = None, + timeout: Optional[int] = None, + result_ttl: Optional[int] = None, + job_info_ttl: Optional[int] = None, + description: Optional[str] = None, + name: Optional[str] = None, + at_front: bool = False, + meta: Optional[Dict] = None, + on_success: Optional[Callback] = None, + on_failure: Optional[Callback] = None, + on_stopped: Optional[Callback] = None, + task_type: Optional[str] = None, + scheduled_task_id: Optional[int] = None, + when: Optional[datetime] = None, + pipeline: Optional[ConnectionType] = None, + ) -> JobModel: + """Creates a job to represent the delayed function call and enqueues it. + :param when: When to schedule the job (None to enqueue immediately) + :param func: The reference to the function + :param args: The `*args` to pass to the function + :param kwargs: The `**kwargs` to pass to the function + :param timeout: Function timeout + :param result_ttl: Result time to live + :param job_info_ttl: Time to live + :param description: The job description + :param name: The job name + :param at_front: Whether to enqueue the job at the front + :param meta: Metadata to attach to the job + :param on_success: Callback for on success + :param on_failure: Callback for on failure + :param on_stopped: Callback for on stopped + :param task_type: The task type + :param scheduled_task_id: The scheduled task id + :param pipeline: The Broker Pipeline + :returns: The enqueued Job + """ + status = JobStatus.QUEUED if when is None else JobStatus.SCHEDULED + job_model = JobModel.create( + connection=self.connection, + func=func, + args=args, + kwargs=kwargs, + result_ttl=result_ttl, + job_info_ttl=job_info_ttl, + description=description, + name=name, + meta=meta, + status=status, + timeout=timeout, + on_success=on_success, + on_failure=on_failure, + on_stopped=on_stopped, + queue_name=self.name, + task_type=task_type, + scheduled_task_id=scheduled_task_id, + ) + if when is None: + job_model = self.enqueue_job(job_model, connection=pipeline, at_front=at_front) + else: + job_model.save(connection=self.connection) + self.scheduled_job_registry.schedule(self.connection, job_model, when) + return job_model + + def job_handle_success(self, job: JobModel, result: Any, result_ttl: int, connection: ConnectionType): + """Saves and cleanup job after successful execution""" + job.after_execution( + result_ttl, JobStatus.FINISHED, + prev_registry=self.active_job_registry, + new_registry=self.finished_job_registry, connection=connection) + Result.create(connection, job_name=job.name, worker_name=job.worker_name, _type=ResultType.SUCCESSFUL, + return_value=result, ttl=result_ttl) + + def job_handle_failure(self, status: JobStatus, job: JobModel, exc_string: str, connection: ConnectionType): + # Does not set job status since the job might be stopped + job.after_execution( + SCHEDULER_CONFIG.DEFAULT_FAILURE_TTL, status, + prev_registry=self.active_job_registry, + new_registry=self.failed_job_registry, + connection=connection) + Result.create( + connection, job.name, job.worker_name, + ResultType.FAILED, SCHEDULER_CONFIG.DEFAULT_FAILURE_TTL, + exc_string=exc_string + ) + + def run_job(self, job: JobModel) -> JobModel: + """Run the job + :param job: The job to run + :returns: The job result + """ + try: + result = perform_job(job, self.connection) + + result_ttl = job.success_ttl + with self.connection.pipeline() as pipeline: + self.job_handle_success(job, result=result, result_ttl=result_ttl, connection=pipeline) + job.expire(result_ttl, connection=pipeline) + pipeline.execute() + except Exception as e: + logger.warning(f"Job {job.name} failed with exception: {e}") + with self.connection.pipeline() as pipeline: + exc_string = "".join(traceback.format_exception(*sys.exc_info())) + self.job_handle_failure(JobStatus.FAILED, job, exc_string, pipeline) + pipeline.execute() + return job + + def enqueue_job( + self, job_model: JobModel, connection: Optional[ConnectionType] = None, at_front: bool = False + ) -> JobModel: + """Enqueues a job for delayed execution without checking dependencies. + + If Queue is instantiated with is_async=False, job is executed immediately. + :param job_model: The job redis model + :param connection: The Redis Pipeline + :param at_front: Whether to enqueue the job at the front + + :returns: The enqueued JobModel + """ + + pipe = connection if connection is not None else self.connection.pipeline() + + # Add Queue key set + job_model.status = JobStatus.QUEUED + job_model.enqueued_at = utcnow() + job_model.save(connection=pipe) + + if self._is_async: + if at_front: + score = current_timestamp() + else: + score = self.queued_job_registry.get_last_timestamp() or current_timestamp() + self.scheduled_job_registry.delete(connection=pipe, job_name=job_model.name) + self.queued_job_registry.add(connection=pipe, score=score, job_name=job_model.name) + pipe.execute() + logger.debug(f"Pushed job {job_model.name} into {self.name} queued-jobs registry") + else: # sync mode + pipe.execute() + job_model = self.run_sync(job_model) + job_model.expire(ttl=job_model.job_info_ttl, connection=pipe) + pipe.execute() + + return job_model + + def run_sync(self, job: JobModel) -> JobModel: + """Run a job synchronously, meaning on the same process the method was called.""" + job.prepare_for_execution("sync", self.active_job_registry, self.connection) + + try: + self.run_job(job) + except: # noqa + with self.connection.pipeline() as pipeline: + exc_string = "".join(traceback.format_exception(*sys.exc_info())) + self.job_handle_failure(JobStatus.FAILED, job, exc_string, pipeline) + pipeline.execute() + return job + + @classmethod + def dequeue_any( + cls, + queues: List[Self], + timeout: Optional[int], + connection: Optional[ConnectionType] = None, + ) -> Tuple[Optional[JobModel], Optional[Self]]: + """Class method returning a Job instance at the front of the given set of Queues, where the order of the queues + is important. + + When all the Queues are empty, depending on the `timeout` argument, either blocks execution of this function + for the duration of the timeout or until new messages arrive on any of the queues, or returns None. + + :param queues: List of Queue objects + :param timeout: Timeout for the pop operation + :param connection: Broker Connection + :returns: Tuple of Job, Queue + """ + + while True: + registries = [q.queued_job_registry for q in queues] + + registry_key, job_name = QueuedJobRegistry.pop(connection, registries, timeout) + if job_name is None: + return None, None + + queue = next(filter(lambda q: q.queued_job_registry.key == registry_key, queues), None) + if queue is None: + logger.warning(f"Could not find queue for registry key {registry_key} in queues") + return None, None + + job = JobModel.get(job_name, connection=connection) + if job is None: + continue + return job, queue + return None, None + + def __eq__(self, other: Self) -> bool: + if not isinstance(other, Queue): + raise TypeError("Cannot compare queues to other objects") + return self.name == other.name + + def __lt__(self, other: Self) -> bool: + if not isinstance(other, Queue): + raise TypeError("Cannot compare queues to other objects") + return self.name < other.name + + def __hash__(self) -> int: + return hash(self.name) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.name!r})" + + def __str__(self) -> str: + return f"<{self.__class__.__name__} {self.name}>" + + def _remove_from_registries(self, job_name: str, connection: ConnectionType) -> None: + """Removes the job from all registries besides failed_job_registry""" + self.finished_job_registry.delete(connection=connection, job_name=job_name) + self.scheduled_job_registry.delete(connection=connection, job_name=job_name) + self.active_job_registry.delete(connection=connection, job_name=job_name) + self.canceled_job_registry.delete(connection=connection, job_name=job_name) + self.queued_job_registry.delete(connection=connection, job_name=job_name) + + def cancel_job(self, job_name: str) -> None: + """Cancels the given job, which will prevent the job from ever running (or inspected). + + This method merely exists as a high-level API call to cancel jobs without worrying about the internals required + to implement job cancellation. + + :param job_name: The job name to cancel. + :raises NoSuchJobError: If the job does not exist. + :raises InvalidJobOperation: If the job has already been canceled. + """ + job = JobModel.get(job_name, connection=self.connection) + if job is None: + raise NoSuchJobError(f"No such job: {job_name}") + if job.status == JobStatus.CANCELED: + raise InvalidJobOperation(f"Cannot cancel already canceled job: {job.name}") + + pipe = self.connection.pipeline() + new_status = JobStatus.CANCELED if job.status == JobStatus.QUEUED else JobStatus.STOPPED + + while True: + try: + job.set_field("status", new_status, connection=pipe) + self._remove_from_registries(job_name, connection=pipe) + pipe.execute() + if new_status == JobStatus.CANCELED: + self.canceled_job_registry.add(pipe, job_name, 0) + else: + self.finished_job_registry.add(pipe, job_name, + current_timestamp() + SCHEDULER_CONFIG.DEFAULT_FAILURE_TTL) + pipe.execute() + break + except WatchError: + # if the pipeline comes from the caller, we re-raise the exception as it is the responsibility of the + # caller to handle it + raise + + def delete_job(self, job_name: str, expire_job_model: bool = True) -> None: + """Deletes the given job from the queue and all its registries""" + pipe = self.connection.pipeline() + + while True: + try: + self._remove_from_registries(job_name, connection=pipe) + self.failed_job_registry.delete(connection=pipe, job_name=job_name) + if expire_job_model: + job_model = JobModel.get(job_name, connection=self.connection) + if job_model is not None: + job_model.expire(ttl=job_model.job_info_ttl, connection=pipe) + pipe.execute() + break + except WatchError: + pass + + def requeue_jobs(self, *job_names: str, at_front: bool = False) -> int: + jobs = JobModel.get_many(job_names, connection=self.connection) + jobs_requeued = 0 + with self.connection.pipeline() as pipe: + for job in jobs: + if job is None: + continue + job.started_at = None + job.ended_at = None + job.save(connection=pipe) + self.enqueue_job(job, connection=pipe, at_front=at_front) + jobs_requeued += 1 + pipe.execute() + return jobs_requeued diff --git a/scheduler/helpers/tools.py b/scheduler/helpers/tools.py new file mode 100644 index 0000000..567e370 --- /dev/null +++ b/scheduler/helpers/tools.py @@ -0,0 +1,77 @@ +"""Helper functions for the scheduler that require django loaded""" + +import os +from typing import Any, Optional + +import croniter +from django.apps import apps +from django.utils import timezone + +from scheduler.broker_types import TASK_TYPES +from scheduler.helpers.queues import get_queues +from scheduler.models.task import Task, TaskType +from scheduler.redis_models import WorkerModel +from scheduler.settings import SCHEDULER_CONFIG, Broker, logger +from scheduler.worker.worker import Worker + + +def get_next_cron_time(cron_string: Optional[str]) -> Optional[timezone.datetime]: + """Calculate the next scheduled time by creating a crontab object with a cron string""" + if cron_string is None: + return None + now = timezone.now() + itr = croniter.croniter(cron_string, now) + next_itr = itr.get_next(timezone.datetime) + return next_itr + + +def get_scheduled_task(task_type_str: str, task_id: int) -> Task: + # Try with new model names + model = apps.get_model(app_label="scheduler", model_name="Task") + if task_type_str in TASK_TYPES: + try: + task_type = TaskType(task_type_str) + task = model.objects.filter(task_type=task_type, id=task_id).first() + if task is None: + raise ValueError(f"Job {task_type}:{task_id} does not exit") + return task + except ValueError: + raise ValueError(f"Invalid task type {task_type_str}") + raise ValueError(f"Job Model {task_type_str} does not exist, choices are {TASK_TYPES}") + + +def run_task(task_model: str, task_id: int) -> Any: + """Run a scheduled job""" + if isinstance(task_id, str): + task_id = int(task_id) + scheduled_task = get_scheduled_task(task_model, task_id) + logger.debug(f"Running task {str(scheduled_task)}") + args = scheduled_task.parse_args() + kwargs = scheduled_task.parse_kwargs() + res = scheduled_task.callable_func()(*args, **kwargs) + return res + + +def _calc_worker_name(existing_worker_names) -> str: + hostname = os.uname()[1] + c = 1 + worker_name = f"{hostname}-worker.{c}" + while worker_name in existing_worker_names: + c += 1 + worker_name = f"{hostname}-worker.{c}" + return worker_name + + +def create_worker(*queue_names: str, **kwargs) -> Worker: + """Returns a Django worker for all queues or specified ones.""" + queues = get_queues(*queue_names) + existing_worker_names = WorkerModel.all_names(connection=queues[0].connection) + kwargs.setdefault("fork_job_execution", SCHEDULER_CONFIG.BROKER != Broker.FAKEREDIS) + if kwargs.get("name", None) is None: + kwargs["name"] = _calc_worker_name(existing_worker_names) + if kwargs["name"] in existing_worker_names: + raise ValueError(f"Worker {kwargs['name']} already exists") + kwargs["name"] = kwargs["name"].replace("/", ".") + kwargs.setdefault("with_scheduler", False) + worker = Worker(queues, connection=queues[0].connection, **kwargs) + return worker diff --git a/scheduler/helpers/utils.py b/scheduler/helpers/utils.py new file mode 100644 index 0000000..1d7e8f4 --- /dev/null +++ b/scheduler/helpers/utils.py @@ -0,0 +1,23 @@ +import datetime +import importlib +import time +from typing import Callable + + +def current_timestamp() -> int: + """Returns current UTC timestamp in secs""" + return int(time.time()) + + +def utcnow(): + """Return now in UTC""" + return datetime.datetime.now(datetime.UTC) + + +def callable_func(callable_str: str) -> Callable: + path = callable_str.split(".") + module = importlib.import_module(".".join(path[:-1])) + func = getattr(module, path[-1]) + if callable(func) is False: + raise TypeError(f"'{callable_str}' is not callable") + return func diff --git a/scheduler/management/commands/delete_failed_executions.py b/scheduler/management/commands/delete_failed_executions.py index 01224e0..6f41980 100644 --- a/scheduler/management/commands/delete_failed_executions.py +++ b/scheduler/management/commands/delete_failed_executions.py @@ -1,8 +1,8 @@ import click from django.core.management.base import BaseCommand -from scheduler.queues import get_queue -from scheduler.rq_classes import JobExecution +from scheduler.helpers.queues import get_queue +from scheduler.redis_models import JobModel class Command(BaseCommand): @@ -15,15 +15,15 @@ def add_arguments(self, parser): def handle(self, *args, **options): queue = get_queue(options.get("queue", "default")) - job_ids = queue.failed_job_registry.get_job_ids() - jobs = JobExecution.fetch_many(job_ids, connection=queue.connection) + job_names = queue.failed_job_registry.all() + jobs = JobModel.get_many(job_names, connection=queue.connection) func_name = options.get("func", None) if func_name is not None: jobs = [job for job in jobs if job.func_name == func_name] dry_run = options.get("dry_run", False) click.echo(f"Found {len(jobs)} failed jobs") - for job in jobs: - click.echo(f"Deleting {job.id}") + for job in job_names: + click.echo(f"Deleting {job}") if not dry_run: - job.delete() + queue.delete_job(job) click.echo(f"Deleted {len(jobs)} failed jobs") diff --git a/scheduler/management/commands/export.py b/scheduler/management/commands/export.py index bb2b249..f28223f 100644 --- a/scheduler/management/commands/export.py +++ b/scheduler/management/commands/export.py @@ -1,16 +1,13 @@ import sys import click -from django.apps import apps from django.core.management.base import BaseCommand -from scheduler.tools import MODEL_NAMES +from scheduler.models.task import Task class Command(BaseCommand): - """ - Export all scheduled jobs - """ + """Export all scheduled jobs""" help = __doc__ @@ -43,13 +40,12 @@ def add_arguments(self, parser): def handle(self, *args, **options): file = open(options.get("filename"), "w") if options.get("filename") else sys.stdout res = list() - for model_name in MODEL_NAMES: - model = apps.get_model(app_label="scheduler", model_name=model_name) - jobs = model.objects.all() - if options.get("enabled"): - jobs = jobs.filter(enabled=True) - for job in jobs: - res.append(job.to_dict()) + + tasks = Task.objects.all() + if options.get("enabled"): + tasks = tasks.filter(enabled=True) + for task in tasks: + res.append(task.to_dict()) if options.get("format") == "json": import json diff --git a/scheduler/management/commands/import.py b/scheduler/management/commands/import.py index 8171781..7fe2940 100644 --- a/scheduler/management/commands/import.py +++ b/scheduler/management/commands/import.py @@ -2,7 +2,6 @@ from typing import Dict, Any, Optional import click -from django.apps import apps from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.core.management.base import BaseCommand @@ -10,7 +9,6 @@ from scheduler.models.task import TaskArg, TaskKwarg, Task from scheduler.models.task import TaskType -from scheduler.tools import MODEL_NAMES def job_model_str(model_str: str) -> str: @@ -139,9 +137,7 @@ def handle(self, *args, **options): jobs = yaml.load(file, yaml.SafeLoader) if options.get("reset"): - for model_name in MODEL_NAMES: - model = apps.get_model(app_label="scheduler", model_name=model_name) - model.objects.all().delete() + Task.objects.all().delete() for job in jobs: create_task_from_dict(job, update=options.get("update")) diff --git a/scheduler/management/commands/run_job.py b/scheduler/management/commands/run_job.py index 48c7458..9495a99 100644 --- a/scheduler/management/commands/run_job.py +++ b/scheduler/management/commands/run_job.py @@ -1,7 +1,7 @@ import click from django.core.management.base import BaseCommand -from scheduler.queues import get_queue +from scheduler.helpers.queues import get_queue class Command(BaseCommand): @@ -32,6 +32,6 @@ def handle(self, **options): queue = get_queue(options.get("queue")) func = options.get("callable") args = options.get("args") - job = queue.enqueue_call(func, args=args, timeout=timeout, result_ttl=result_ttl) + job = queue.create_and_enqueue_job(func, args=args, timeout=timeout, result_ttl=result_ttl) if verbosity: - click.echo(f"Job {job.id} created") + click.echo(f"Job {job.name} created") diff --git a/scheduler/management/commands/rqstats.py b/scheduler/management/commands/scheduler_stats.py similarity index 89% rename from scheduler/management/commands/rqstats.py rename to scheduler/management/commands/scheduler_stats.py index 13a7de8..0abb973 100644 --- a/scheduler/management/commands/rqstats.py +++ b/scheduler/management/commands/scheduler_stats.py @@ -9,13 +9,11 @@ ANSI_LIGHT_WHITE = "\033[1;37m" ANSI_RESET = "\033[0m" -KEYS = ("jobs", "started_jobs", "deferred_jobs", "finished_jobs", "canceled_jobs", "workers") +KEYS = ("queued_jobs", "started_jobs", "finished_jobs", "canceled_jobs", "workers") class Command(BaseCommand): - """ - Print statistics - """ + """Print statistics""" help = __doc__ @@ -59,7 +57,7 @@ def _print_stats_dashboard(self, statistics, prev_stats=None): click.echo("Django-Scheduler CLI Dashboard") click.echo() self._print_separator() - click.echo(f"| {'Name':<16} | Queued | Active | Deferred | Finished | Canceled | Workers |") + click.echo(f"| {'Name':<16} | Queued | Active | Finished | Canceled | Workers |") self._print_separator() for ind, queue in enumerate(statistics["queues"]): vals = list((queue[k] for k in KEYS)) @@ -82,6 +80,9 @@ def _print_stats_dashboard(self, statistics, prev_stats=None): click.echo("Press 'Ctrl+c' to quit") def handle(self, *args, **options): + if options.get("json") and options.get("yaml"): + click.secho("Aborting. Cannot output as both json and yaml", err=True, fg="red") + return if options.get("json"): import json diff --git a/scheduler/management/commands/rqworker.py b/scheduler/management/commands/scheduler_worker.py similarity index 70% rename from scheduler/management/commands/rqworker.py rename to scheduler/management/commands/scheduler_worker.py index ce6201b..d031232 100644 --- a/scheduler/management/commands/rqworker.py +++ b/scheduler/management/commands/scheduler_worker.py @@ -5,11 +5,10 @@ import click from django.core.management.base import BaseCommand from django.db import connections -from rq.logutils import setup_loghandlers from scheduler.broker_types import ConnectionErrorTypes -from scheduler.rq_classes import register_sentry -from scheduler.tools import create_worker +from scheduler.helpers.tools import create_worker +from scheduler.settings import logger VERBOSITY_TO_LOG_LEVEL = { 0: logging.CRITICAL, @@ -19,21 +18,16 @@ } WORKER_ARGUMENTS = { + "queues", "name", - "default_result_ttl", "connection", - "exc_handler", - "exception_handlers", - "default_worker_ttl", "maintenance_interval", - "job_class", - "queue_class", - "log_job_description", "job_monitoring_interval", + "dequeue_strategy", "disable_default_exception_handler", - "prepare_for_work", - "serializer", - "work_horse_killed_handler", + "fork_job_execution", + "with_scheduler", + "burst", } @@ -42,23 +36,63 @@ def reset_db_connections(): c.close() +def register_sentry(sentry_dsn, **opts): + try: + import sentry_sdk + from sentry_sdk.integrations.rq import RqIntegration + except ImportError: + logger.error("Sentry SDK not installed. Skipping Sentry Integration") + return + + sentry_sdk.init(sentry_dsn, integrations=[RqIntegration()], **opts) + + class Command(BaseCommand): - """ - Runs RQ workers on specified queues. Note that all queues passed into a - single rqworker command must share the same connection. + """Runs scheduler workers on specified queues. + Note that all queues passed into a single scheduler_worker command must share the same connection. Example usage: - python manage.py rqworker high medium low + python manage.py scheduler_worker high medium low """ args = "" - def add_arguments(self, parser): + def _add_sentry_args(self, parser): + parser.add_argument("--sentry-dsn", action="store", dest="sentry_dsn", help="Sentry DSN to use") + parser.add_argument("--sentry-debug", action="store_true", dest="sentry_debug", help="Enable Sentry debug mode") + parser.add_argument("--sentry-ca-certs", action="store", dest="sentry_ca_certs", help="Path to CA certs file") + + def _add_work_args(self, parser): parser.add_argument( - "--pid", action="store", dest="pidfile", default=None, help="file to write the worker`s pid into" + "--burst", action="store_true", dest="burst", default=False, help="Run worker in burst mode" ) parser.add_argument( - "--burst", action="store_true", dest="burst", default=False, help="Run worker in burst mode" + "--max-jobs", + action="store", + default=None, + dest="max_jobs", + type=int, + help="Maximum number of jobs to execute before terminating worker", + ) + parser.add_argument( + "--max-idle-time", + action="store", + default=None, + dest="max_idle_time", + type=int, + help="Maximum number of seconds to wait for new job before terminating worker", + ) + parser.add_argument( + "--with-scheduler", + action="store_true", + default=True, + dest="with_scheduler", + help="Run worker with scheduler, default to True", + ) + + def add_arguments(self, parser): + parser.add_argument( + "--pid", action="store", dest="pidfile", default=None, help="file to write the worker`s pid into" ) parser.add_argument("--name", action="store", dest="name", default=None, help="Name of the worker") parser.add_argument( @@ -69,14 +103,6 @@ def add_arguments(self, parser): default=420, help="Default worker timeout to be used", ) - parser.add_argument( - "--max-jobs", - action="store", - default=None, - dest="max_jobs", - type=int, - help="Maximum number of jobs to execute before terminating worker", - ) parser.add_argument( "--fork-job-execution", action="store", @@ -85,16 +111,14 @@ def add_arguments(self, parser): type=bool, help="Fork job execution to another process", ) - parser.add_argument("--job-class", action="store", dest="job_class", help="Jobs class to use") parser.add_argument( "queues", nargs="*", type=str, help="The queues to work on, separated by space, all queues should be using the same redis", ) - parser.add_argument("--sentry-dsn", action="store", dest="sentry_dsn", help="Sentry DSN to use") - parser.add_argument("--sentry-debug", action="store_true", dest="sentry_debug", help="Enable Sentry debug mode") - parser.add_argument("--sentry-ca-certs", action="store", dest="sentry_ca_certs", help="Path to CA certs file") + self._add_sentry_args(parser) + self._add_work_args(parser) def handle(self, **options): queues = options.pop("queues", []) @@ -109,9 +133,9 @@ def handle(self, **options): fp.write(str(os.getpid())) # Verbosity is defined by default in BaseCommand for all commands - verbosity = options.pop("verbosity", 1) + verbosity = options.pop("verbosity", 3) log_level = VERBOSITY_TO_LOG_LEVEL.get(verbosity, logging.INFO) - setup_loghandlers(log_level) + logger.setLevel(log_level) init_options = {k: v for k, v in options.items() if k in WORKER_ARGUMENTS} @@ -128,9 +152,8 @@ def handle(self, **options): register_sentry(options.get("sentry_dsn"), **sentry_opts) w.work( - burst=options.get("burst", False), - logging_level=log_level, max_jobs=options["max_jobs"], + max_idle_time=options.get("max_idle_time", None), ) except ConnectionErrorTypes as e: click.echo(str(e), err=True) diff --git a/scheduler/migrations/0021_remove_task_job_id_task_job_name.py b/scheduler/migrations/0021_remove_task_job_id_task_job_name.py new file mode 100644 index 0000000..3c03f51 --- /dev/null +++ b/scheduler/migrations/0021_remove_task_job_id_task_job_name.py @@ -0,0 +1,22 @@ +# Generated by Django 5.1.7 on 2025-03-24 14:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('scheduler', '0020_remove_repeatabletask_new_task_id_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='task', + name='job_id', + ), + migrations.AddField( + model_name='task', + name='job_name', + field=models.CharField(blank=True, editable=False, help_text='Current job_name on queue', max_length=128, null=True, verbose_name='job name'), + ), + ] diff --git a/scheduler/models/args.py b/scheduler/models/args.py index f7cd57b..ac2d700 100644 --- a/scheduler/models/args.py +++ b/scheduler/models/args.py @@ -7,7 +7,7 @@ from django.db import models from django.utils.translation import gettext_lazy as _ -from scheduler import tools +from scheduler.helpers import utils ARG_TYPE_TYPES_DICT = { "str": str, @@ -48,7 +48,7 @@ def clean(self): ) try: if self.arg_type == "callable": - tools.callable_func(self.val) + utils.callable_func(self.val) elif self.arg_type == "datetime": datetime.fromisoformat(self.val) elif self.arg_type == "bool": @@ -71,7 +71,7 @@ def delete(self, **kwargs): def value(self): if self.arg_type == "callable": - res = tools.callable_func(self.val)() + res = utils.callable_func(self.val)() elif self.arg_type == "datetime": res = datetime.fromisoformat(self.val) elif self.arg_type == "bool": diff --git a/scheduler/models/task.py b/scheduler/models/task.py index 1bf5750..67d6ce8 100644 --- a/scheduler/models/task.py +++ b/scheduler/models/task.py @@ -1,10 +1,8 @@ import math -import uuid from datetime import timedelta, datetime -from typing import Dict +from typing import Dict, Any, Optional import croniter -from django.apps import apps from django.conf import settings as django_settings from django.contrib import admin from django.contrib.contenttypes.fields import GenericRelation @@ -18,58 +16,60 @@ from django.utils.translation import gettext_lazy as _ from scheduler import settings -from scheduler import tools +from scheduler.broker_types import ConnectionType +from scheduler.helpers import tools, utils +from scheduler.helpers.callback import Callback +from scheduler.helpers.queues import Queue +from scheduler.helpers.queues import get_queue from scheduler.models.args import TaskArg, TaskKwarg -from scheduler.queues import get_queue -from scheduler.rq_classes import DjangoQueue -from scheduler.settings import QUEUES -from scheduler.settings import logger -from scheduler.tools import TaskType +from scheduler.redis_models import JobModel +from scheduler.settings import logger, get_queue_names SCHEDULER_INTERVAL = settings.SCHEDULER_CONFIG.SCHEDULER_INTERVAL -def failure_callback(job, connection, result, *args, **kwargs): - task_type = job.meta.get("task_type", None) - if task_type is None: - return - task = Task.objects.filter(job_id=job.id).first() +def _get_task_for_job(job: JobModel) -> Optional["Task"]: + if job.task_type is None or job.scheduled_task_id is None: + return None + task = Task.objects.filter(id=job.scheduled_task_id).first() + return task + + +def failure_callback(job: JobModel, connection, result, *args, **kwargs): + task = _get_task_for_job(job) if task is None: - logger.warn(f"Could not find task for job {job.id}") + logger.warn(f"Could not find task for job {job.name}") return mail_admins( f"Task {task.id}/{task.name} has failed", "See django-admin for logs", ) - task.job_id = None + task.job_name = None task.failed_runs += 1 task.last_failed_run = timezone.now() task.save(schedule_job=True) -def success_callback(job, connection, result, *args, **kwargs): - task_type = job.meta.get("task_type", None) - if task_type is None: - return - - task = Task.objects.filter(job_id=job.id).first() - if task is None: - try: - model = apps.get_model(app_label="scheduler", model_name=task_type) - task = model.objects.filter(job_id=job.id).first() - except LookupError: - pass +def success_callback(job: JobModel, connection: ConnectionType, result: Any, *args, **kwargs): + task = _get_task_for_job(job) if task is None: - logger.warn(f"Could not find task for job {task_type}/{job.id}") + logger.warn(f"Could not find task for job {job.name}") return - task.job_id = None + task.job_name = None task.successful_runs += 1 task.last_successful_run = timezone.now() task.save(schedule_job=True) def get_queue_choices(): - return [(queue, queue) for queue in QUEUES.keys()] + queue_names = get_queue_names() + return [(queue, queue) for queue in queue_names] + + +class TaskType(models.TextChoices): + CRON = "CronTaskType", _("Cron Task") + REPEATABLE = "RepeatableTaskType", _("Repeatable Task") + ONCE = "OnceTaskType", _("Run once") class Task(models.Model): @@ -95,8 +95,8 @@ class TimeUnits(models.TextChoices): ), ) queue = models.CharField(_("queue"), max_length=255, choices=get_queue_choices, help_text=_("Queue name")) - job_id = models.CharField( - _("job id"), max_length=128, editable=False, blank=True, null=True, help_text=_("Current job_id on queue") + job_name = models.CharField( + _("job name"), max_length=128, editable=False, blank=True, null=True, help_text=_("Current job_name on queue") ) at_front = models.BooleanField( _("At front"), @@ -180,22 +180,23 @@ class TimeUnits(models.TextChoices): def callable_func(self): """Translate callable string to callable""" - return tools.callable_func(self.callable) + return utils.callable_func(self.callable) @admin.display(boolean=True, description=_("is scheduled?")) def is_scheduled(self) -> bool: """Check whether a next job for this task is queued/scheduled to be executed""" - if self.job_id is None: # no job_id => is not scheduled + if self.job_name is None: # no job_id => is not scheduled return False # check whether job_id is in scheduled/queued/active jobs - scheduled_jobs = self.rqueue.scheduled_job_registry.get_job_ids() - enqueued_jobs = self.rqueue.get_job_ids() - active_jobs = self.rqueue.started_job_registry.get_job_ids() - res = (self.job_id in scheduled_jobs) or (self.job_id in enqueued_jobs) or (self.job_id in active_jobs) + res = ( + (self.job_name in self.rqueue.scheduled_job_registry.all()) + or (self.job_name in self.rqueue.queued_job_registry.all()) + or (self.job_name in self.rqueue.active_job_registry.all()) + ) # If the job_id is not scheduled/queued/started, # update the job_id to None. (The job_id belongs to a previous run which is completed) if not res: - self.job_id = None + self.job_name = None super(Task, self).save() return res @@ -218,32 +219,29 @@ def parse_kwargs(self): return dict([kwarg.value() for kwarg in kwargs]) def _next_job_id(self): - addition = uuid.uuid4().hex[-10:] - name = self.name.replace("/", ".") - return f"{self.queue}:{name}:{addition}" + addition = timezone.now().strftime("%Y%m%d%H%M%S%f") + return f"{self.queue}:{self.id}:{addition}" def _enqueue_args(self) -> Dict: - """Args for DjangoQueue.enqueue. - Set all arguments for DjangoQueue.enqueue/enqueue_at. - Particularly: + """Args for Queue.enqueue_call. + Set all arguments for Queue.enqueue. Particularly: - set job timeout and ttl - ensure a callback to reschedule the job next iteration. - Set job-id to proper format - set job meta """ res = dict( - meta=dict( - task_type=self.task_type, - scheduled_task_id=self.id, - ), - on_success=success_callback, - on_failure=failure_callback, - job_id=self._next_job_id(), + meta=dict(), + task_type=self.task_type, + scheduled_task_id=self.id, + on_success=Callback(success_callback), + on_failure=Callback(failure_callback), + name=self._next_job_id(), ) if self.at_front: res["at_front"] = self.at_front if self.timeout: - res["job_timeout"] = self.timeout + res["timeout"] = self.timeout if self.result_ttl is not None: res["result_ttl"] = self.result_ttl if self.task_type == TaskType.REPEATABLE: @@ -252,20 +250,18 @@ def _enqueue_args(self) -> Dict: return res @property - def rqueue(self) -> DjangoQueue: + def rqueue(self) -> Queue: """Returns django-queue for job""" return get_queue(self.queue) def enqueue_to_run(self) -> bool: - """Enqueue task to run now.""" + """Enqueue task to run now as a different instance from the scheduled task.""" kwargs = self._enqueue_args() - job = self.rqueue.enqueue( + self.rqueue.create_and_enqueue_job( tools.run_task, args=(self.task_type, self.id), **kwargs, ) - self.job_id = job.id - self.save(schedule_job=False) return True def unschedule(self) -> bool: @@ -273,12 +269,9 @@ def unschedule(self) -> bool: If a job is queued to be executed or scheduled to be executed, it will remove it. """ - queue = self.rqueue - if self.job_id is None: - return True - queue.remove(self.job_id) - queue.scheduled_job_registry.remove(self.job_id) - self.job_id = None + if self.job_name is not None: + self.rqueue.delete_job(self.job_name) + self.job_name = None self.save(schedule_job=False) return True @@ -358,17 +351,17 @@ def _schedule(self) -> bool: if not self.enabled: logger.debug(f"Task {str(self)} disabled, enable task before scheduling") return False - if self.task_type in {TaskType.REPEATABLE, TaskType.ONCE} and self._schedule_time() < timezone.now(): - return False schedule_time = self._schedule_time() + if self.task_type in {TaskType.REPEATABLE, TaskType.ONCE} and schedule_time < timezone.now(): + return False kwargs = self._enqueue_args() - job = self.rqueue.enqueue_at( - schedule_time, + job = self.rqueue.create_and_enqueue_job( tools.run_task, args=(self.task_type, self.id), + when=schedule_time, **kwargs, ) - self.job_id = job.id + self.job_name = job.name super(Task, self).save() return True @@ -394,19 +387,19 @@ def interval_seconds(self): def clean_callable(self): try: - tools.callable_func(self.callable) + utils.callable_func(self.callable) except Exception: raise ValidationError( {"callable": ValidationError(_("Invalid callable, must be importable"), code="invalid")} ) def clean_queue(self): - queue_keys = settings.QUEUES.keys() - if self.queue not in queue_keys: + queue_names = settings.get_queue_names() + if self.queue not in queue_names: raise ValidationError( { "queue": ValidationError( - _("Invalid queue, must be one of: {}".format(", ".join(queue_keys))), code="invalid" + "Invalid queue, must be one of: {}".format(", ".join(queue_names)), code="invalid" ) } ) diff --git a/scheduler/queues.py b/scheduler/queues.py deleted file mode 100644 index f7796db..0000000 --- a/scheduler/queues.py +++ /dev/null @@ -1,150 +0,0 @@ -from typing import List, Dict, Set - -from .broker_types import ConnectionErrorTypes, BrokerMetaData -from .rq_classes import JobExecution, DjangoQueue, DjangoWorker -from .settings import SCHEDULER_CONFIG -from .settings import logger, Broker - -_CONNECTION_PARAMS = { - "URL", - "DB", - "USE_REDIS_CACHE", - "UNIX_SOCKET_PATH", - "HOST", - "PORT", - "PASSWORD", - "SENTINELS", - "MASTER_NAME", - "SOCKET_TIMEOUT", - "SSL", - "CONNECTION_KWARGS", -} - - -class QueueNotFoundError(Exception): - pass - - -def _get_broker_connection(config, use_strict_broker=False): - """ - Returns a redis connection from a connection config - """ - if SCHEDULER_CONFIG.BROKER == Broker.FAKEREDIS: - import fakeredis - - broker_cls = fakeredis.FakeRedis if not use_strict_broker else fakeredis.FakeStrictRedis - else: - broker_cls = BrokerMetaData[(SCHEDULER_CONFIG.BROKER, use_strict_broker)].connection_type - logger.debug(f"Getting connection for {config}") - if "URL" in config: - ssl_url_protocol = BrokerMetaData[(SCHEDULER_CONFIG.BROKER, use_strict_broker)].ssl_prefix - if config.get("SSL") or config.get("URL").startswith(f"{ssl_url_protocol}://"): - return broker_cls.from_url( - config["URL"], - db=config.get("DB"), - ssl_cert_reqs=config.get("SSL_CERT_REQS", "required"), - ) - else: - return broker_cls.from_url( - config["URL"], - db=config.get("DB"), - ) - if "UNIX_SOCKET_PATH" in config: - return broker_cls(unix_socket_path=config["UNIX_SOCKET_PATH"], db=config["DB"]) - - if "SENTINELS" in config: - connection_kwargs = { - "db": config.get("DB"), - "password": config.get("PASSWORD"), - "username": config.get("USERNAME"), - "socket_timeout": config.get("SOCKET_TIMEOUT"), - } - connection_kwargs.update(config.get("CONNECTION_KWARGS", {})) - sentinel_kwargs = config.get("SENTINEL_KWARGS", {}) - SentinelClass = BrokerMetaData[(SCHEDULER_CONFIG.BROKER, use_strict_broker)].sentinel_type - sentinel = SentinelClass(config["SENTINELS"], sentinel_kwargs=sentinel_kwargs, **connection_kwargs) - return sentinel.master_for( - service_name=config["MASTER_NAME"], - redis_class=broker_cls, - ) - - return broker_cls( - host=config["HOST"], - port=config["PORT"], - db=config.get("DB", 0), - username=config.get("USERNAME", None), - password=config.get("PASSWORD"), - ssl=config.get("SSL", False), - ssl_cert_reqs=config.get("SSL_CERT_REQS", "required"), - **config.get("CLIENT_KWARGS", {}), - ) - - -def get_connection(queue_settings, use_strict_redis=False): - """Returns a Broker connection to use based on parameters in SCHEDULER_QUEUES""" - return _get_broker_connection(queue_settings, use_strict_redis) - - -def get_queue( - name="default", default_timeout=None, is_async=None, autocommit=None, connection=None, **kwargs -) -> DjangoQueue: - """Returns an DjangoQueue using parameters defined in `SCHEDULER_QUEUES`""" - from .settings import QUEUES - - if name not in QUEUES: - raise QueueNotFoundError(f"Queue {name} not found, queues={QUEUES.keys()}") - queue_settings = QUEUES[name] - if is_async is None: - is_async = queue_settings.get("ASYNC", True) - - if default_timeout is None: - default_timeout = queue_settings.get("DEFAULT_TIMEOUT") - if connection is None: - connection = get_connection(queue_settings) - return DjangoQueue( - name, default_timeout=default_timeout, connection=connection, is_async=is_async, autocommit=autocommit, **kwargs - ) - - -def get_all_workers() -> Set[DjangoWorker]: - from .settings import QUEUES - - workers_set: Set[DjangoWorker] = set() - for queue_name in QUEUES: - connection = get_connection(QUEUES[queue_name]) - try: - curr_workers: Set[DjangoWorker] = set(DjangoWorker.all(connection=connection)) - workers_set.update(curr_workers) - except ConnectionErrorTypes as e: - logger.error(f"Could not connect for queue {queue_name}: {e}") - return workers_set - - -def _queues_share_connection_params(q1_params: Dict, q2_params: Dict): - """Check that both queues share the same connection parameters""" - return all( - ((p not in q1_params and p not in q2_params) or (q1_params.get(p, None) == q2_params.get(p, None))) - for p in _CONNECTION_PARAMS - ) - - -def get_queues(*queue_names, **kwargs) -> List[DjangoQueue]: - """Return queue instances from specified queue names. - All instances must use the same Redis connection. - """ - from .settings import QUEUES - - kwargs["job_class"] = JobExecution - queue_params = QUEUES[queue_names[0]] - queues = [get_queue(queue_names[0], **kwargs)] - # perform consistency checks while building return list - for name in queue_names[1:]: - if not _queues_share_connection_params(queue_params, QUEUES[name]): - raise ValueError( - f'Queues must have the same broker connection. "{name}" and' - f' "{queue_names[0]}" have different connections' - ) - queue = get_queue(name, **kwargs) - queues.append(queue) - - return queues diff --git a/scheduler/redis_models/__init__.py b/scheduler/redis_models/__init__.py new file mode 100644 index 0000000..8c7336a --- /dev/null +++ b/scheduler/redis_models/__init__.py @@ -0,0 +1,35 @@ +__all__ = [ + "Result", + "ResultType", + "as_str", + "SchedulerLock", + "WorkerModel", + "DequeueTimeout", + "KvLock", + "JobStatus", + "JobModel", + "JobNamesRegistry", + "FinishedJobRegistry", + "ActiveJobRegistry", + "FailedJobRegistry", + "CanceledJobRegistry", + "ScheduledJobRegistry", + "QueuedJobRegistry", + "NoSuchJobError", +] + +from .base import as_str +from .job import JobStatus, JobModel +from .lock import SchedulerLock, KvLock +from .registry.base_registry import DequeueTimeout, JobNamesRegistry +from .registry.queue_registries import ( + FinishedJobRegistry, + ActiveJobRegistry, + FailedJobRegistry, + CanceledJobRegistry, + ScheduledJobRegistry, + QueuedJobRegistry, + NoSuchJobError, +) +from .result import Result, ResultType +from .worker import WorkerModel diff --git a/scheduler/redis_models/base.py b/scheduler/redis_models/base.py new file mode 100644 index 0000000..a7e40e2 --- /dev/null +++ b/scheduler/redis_models/base.py @@ -0,0 +1,243 @@ +import dataclasses +import json +from collections.abc import Sequence +from datetime import datetime, UTC +from enum import Enum +from typing import List, Optional, Union, Self, Dict, Collection, Any, ClassVar, Set, Type + +from redis import Redis + +from scheduler.broker_types import ConnectionType +from scheduler.settings import logger + +MAX_KEYS = 1000 + + +def as_str(v: Union[bytes, str]) -> Optional[str]: + """Converts a `bytes` value to a string using `utf-8`. + + :param v: The value (None/bytes/str) + :raises: ValueError: If the value is not `bytes` or `str` + :returns: Either the decoded string or None + """ + if v is None or isinstance(v, str): + return v + if isinstance(v, bytes): + return v.decode("utf-8") + raise ValueError("Unknown type %r" % type(v)) + + +def decode_dict(d: Dict[bytes, bytes], exclude_keys: Set[str]) -> Dict[str, str]: + return {k.decode(): v.decode() for (k, v) in d.items() if k.decode() not in exclude_keys} + + +def _serialize(value: Any) -> Optional[Any]: + if value is None: + return None + if isinstance(value, bool): + value = int(value) + elif isinstance(value, Enum): + value = value.value + elif isinstance(value, datetime): + value = value.isoformat() + elif isinstance(value, dict): + value = json.dumps(value) + elif isinstance(value, (int, float)): + return value + elif isinstance(value, (list, set, tuple)): + return json.dumps(value, default=str) + return str(value) + + +def _deserialize(value: str, _type: Type) -> Any: + if value is None: + return None + try: + if _type is str or _type == Optional[str]: + return as_str(value) + if _type is datetime or _type == Optional[datetime]: + return datetime.fromisoformat(as_str(value)) + elif _type is bool: + return bool(int(value)) + elif _type is int or _type == Optional[int]: + return int(value) + elif _type is float or _type == Optional[float]: + return float(value) + elif _type in {List[Any], List[str], Dict[str, str]}: + return json.loads(value) + elif _type == Optional[Any]: + return json.loads(value) + elif issubclass(_type, Enum): + return _type(as_str(value)) + except (ValueError, TypeError) as e: + logger.warning(f"Failed to deserialize {value} as {_type}: {e}") + return value + + +@dataclasses.dataclass(slots=True, kw_only=True) +class BaseModel: + name: str + _element_key_template: ClassVar[str] = ":element:{}" + + @classmethod + def key_for(cls, name: str) -> str: + return cls._element_key_template.format(name) + + @property + def _key(self) -> str: + return self._element_key_template.format(self.name) + + def serialize(self, with_nones: bool = False) -> Dict[str, str]: + data = dataclasses.asdict( + self, dict_factory=lambda fields: {key: value for (key, value) in fields if not key.startswith("_")} + ) + if not with_nones: + data = {k: v for k, v in data.items() if v is not None} + for k in data: + data[k] = _serialize(data[k]) + return data + + @classmethod + def deserialize(cls, data: Dict[str, Any]) -> Self: + types = {f.name: f.type for f in dataclasses.fields(cls)} + for k in data: + if k not in types: + logger.warning(f"Unknown field {k} in {cls.__name__}") + continue + data[k] = _deserialize(data[k], types[k]) + return cls(**data) + + +@dataclasses.dataclass(slots=True, kw_only=True) +class HashModel(BaseModel): + created_at: Optional[datetime] = None + parent: Optional[str] = None + _dirty_fields: Set[str] = dataclasses.field(default_factory=set) # fields that were changed + _save_all: bool = True # Save all fields to broker, after init, or after delete + _list_key: ClassVar[str] = ":list_all:" + _children_key_template: ClassVar[str] = ":children:{}:" + + def __post_init__(self): + self._dirty_fields = set() + self._save_all = True + + def __setattr__(self, key, value): + if key != "_dirty_fields" and hasattr(self, "_dirty_fields"): + self._dirty_fields.add(key) + super(HashModel, self).__setattr__(key, value) + + @property + def _parent_key(self) -> Optional[str]: + if self.parent is None: + return None + return self._children_key_template.format(self.parent) + + @classmethod + def all_names(cls, connection: Redis, parent: Optional[str] = None) -> Collection[str]: + collection_key = cls._children_key_template.format(parent) if parent else cls._list_key + collection_members = connection.smembers(collection_key) + return [r.decode() for r in collection_members] + + @classmethod + def all(cls, connection: Redis, parent: Optional[str] = None) -> List[Self]: + keys = cls.all_names(connection, parent) + items = [cls.get(k, connection) for k in keys] + return [w for w in items if w is not None] + + @classmethod + def exists(cls, name: str, connection: ConnectionType) -> bool: + return connection.exists(cls._element_key_template.format(name)) > 0 + + @classmethod + def delete_many(cls, names: List[str], connection: ConnectionType) -> None: + for name in names: + connection.delete(cls._element_key_template.format(name)) + + @classmethod + def get(cls, name: str, connection: ConnectionType) -> Optional[Self]: + res = connection.hgetall(cls._element_key_template.format(name)) + if not res: + return None + try: + return cls.deserialize(decode_dict(res, set())) + except Exception as e: + logger.warning(f"Failed to deserialize {name}: {e}") + return None + + @classmethod + def get_many(cls, names: Sequence[str], connection: ConnectionType) -> List[Self]: + pipeline = connection.pipeline() + for name in names: + pipeline.hgetall(cls._element_key_template.format(name)) + values = pipeline.execute() + return [(cls.deserialize(decode_dict(v, set())) if v else None) for v in values] + + def save(self, connection: ConnectionType) -> None: + connection.sadd(self._list_key, self.name) + if self._parent_key is not None: + connection.sadd(self._parent_key, self.name) + mapping = self.serialize(with_nones=True) + if not self._save_all and len(self._dirty_fields) > 0: + mapping = {k: v for k, v in mapping.items() if k in self._dirty_fields} + none_values = {k for k, v in mapping.items() if v is None} + if none_values: + connection.hdel(self._key, *none_values) + mapping = {k: v for k, v in mapping.items() if v is not None} + if mapping: + connection.hset(self._key, mapping=mapping) + self._dirty_fields = set() + self._save_all = False + + def delete(self, connection: ConnectionType) -> None: + connection.srem(self._list_key, self._key) + if self._parent_key is not None: + connection.srem(self._parent_key, 0, self._key) + connection.delete(self._key) + self._save_all = True + + @classmethod + def count(cls, connection: ConnectionType, parent: Optional[str] = None) -> int: + if parent is not None: + result = connection.scard(cls._children_key_template.format(parent)) + else: + result = connection.scard(cls._list_key) + return result + + def get_field(self, field: str, connection: ConnectionType) -> Any: + types = {f.name: f.type for f in dataclasses.fields(self)} + res = connection.hget(self._key, field) + return _deserialize(res, types[field]) + + def set_field(self, field: str, value: Any, connection: ConnectionType, set_attribute: bool = True) -> None: + if not hasattr(self, field): + raise AttributeError(f"Field {field} does not exist") + if set_attribute: + setattr(self, field, value) + if value is None: + connection.hdel(self._key, field) + return + value = _serialize(value) + connection.hset(self._key, field, value) + + +@dataclasses.dataclass(slots=True, kw_only=True) +class StreamModel(BaseModel): + _children_key_template: ClassVar[str] = ":children:{}:" + + def __init__(self, name: str, parent: str, created_at: Optional[datetime] = None): + self.name = name + self.created_at: datetime = created_at or datetime.now(UTC) + self.parent: str = parent + + @property + def _parent_key(self) -> str: + return self._children_key_template.format(self.parent) + + @classmethod + def all(cls, connection: ConnectionType, parent: str) -> List[Self]: + results = connection.xrevrange(cls._children_key_template.format(parent), "+", "-") + return [cls.deserialize(decode_dict(result[1], exclude_keys=set())) for result in results] + + def save(self, connection: ConnectionType) -> bool: + result = connection.xadd(self._parent_key, self.serialize(), maxlen=10) + return bool(result) diff --git a/scheduler/redis_models/job.py b/scheduler/redis_models/job.py new file mode 100644 index 0000000..9be975e --- /dev/null +++ b/scheduler/redis_models/job.py @@ -0,0 +1,312 @@ +import dataclasses +import inspect +import numbers +from datetime import datetime +from enum import Enum +from typing import ClassVar, Dict, Optional, List, Callable, Any, Union, Tuple, Self + +from scheduler.broker_types import ConnectionType, FunctionReferenceType +from scheduler.helpers import utils +from scheduler.helpers.callback import Callback +from scheduler.redis_models.base import HashModel, as_str +from scheduler.settings import SCHEDULER_CONFIG, logger +from .registry.base_registry import JobNamesRegistry +from ..helpers.utils import current_timestamp + + +class TimeoutFormatError(Exception): + pass + + +class JobStatus(str, Enum): + """The Status of Job within its lifecycle at any given time.""" + + QUEUED = "queued" + FINISHED = "finished" + FAILED = "failed" + STARTED = "started" + SCHEDULED = "scheduled" + STOPPED = "stopped" + CANCELED = "canceled" + + +@dataclasses.dataclass(slots=True, kw_only=True) +class JobModel(HashModel): + _list_key: ClassVar[str] = ":jobs:ALL:" + _children_key_template: ClassVar[str] = ":{}:jobs:" + _element_key_template: ClassVar[str] = ":jobs:{}" + + queue_name: str + description: str + func_name: str + + args: List[Any] + kwargs: Dict[str, str] + timeout: int = SCHEDULER_CONFIG.DEFAULT_JOB_TIMEOUT + success_ttl: int = SCHEDULER_CONFIG.DEFAULT_SUCCESS_TTL + job_info_ttl: int = SCHEDULER_CONFIG.DEFAULT_JOB_TTL + status: JobStatus + created_at: datetime + meta: Dict[str, str] + at_front: bool = False + last_heartbeat: Optional[datetime] = None + worker_name: Optional[str] = None + started_at: Optional[datetime] = None + enqueued_at: Optional[datetime] = None + ended_at: Optional[datetime] = None + success_callback_name: Optional[str] = None + success_callback_timeout: int = SCHEDULER_CONFIG.CALLBACK_TIMEOUT + failure_callback_name: Optional[str] = None + failure_callback_timeout: int = SCHEDULER_CONFIG.CALLBACK_TIMEOUT + stopped_callback_name: Optional[str] = None + stopped_callback_timeout: int = SCHEDULER_CONFIG.CALLBACK_TIMEOUT + task_type: Optional[str] = None + scheduled_task_id: Optional[int] = None + + def serialize(self, with_nones: bool = False) -> Dict[str, str]: + res = super(JobModel, self).serialize() + return res + + def __hash__(self): + return hash(self.name) + + def __eq__(self, other): # noqa + return isinstance(other, self.__class__) and self.name == other.name + + def __str__(self): + return f"{self.name}: {self.description}" + + def get_status(self, connection: ConnectionType) -> JobStatus: + return self.get_field("status", connection=connection) + + def set_status(self, status: JobStatus, connection: ConnectionType) -> None: + """Set's the Job Status""" + self.set_field("status", status, connection=connection) + + @property + def is_queued(self) -> bool: + return self.status == JobStatus.QUEUED + + @property + def is_canceled(self) -> bool: + return self.status == JobStatus.CANCELED + + @property + def is_failed(self) -> bool: + return self.status == JobStatus.FAILED + + @property + def func(self) -> Callable[[Any], Any]: + return utils.callable_func(self.func_name) + + @property + def is_scheduled_task(self) -> bool: + return self.scheduled_task_id is not None + + def expire(self, ttl: int, connection: ConnectionType) -> None: + """Expire the Job Model if ttl >= 0""" + if ttl == 0: + self.delete(connection=connection) + elif ttl > 0: + connection.expire(self._key, ttl) + + def persist(self, connection: ConnectionType) -> None: + connection.persist(self._key) + + def prepare_for_execution(self, worker_name: str, registry: JobNamesRegistry, connection: ConnectionType) -> None: + """Prepares the job for execution, setting the worker name, + heartbeat information, status and other metadata before execution begins. + :param worker_name: The name of the worker + :param registry: The registry to add the job to + :param current_pid: The current process id + :param connection: The connection to the broker + """ + self.worker_name = worker_name + self.last_heartbeat = utils.utcnow() + self.started_at = self.last_heartbeat + self.status = JobStatus.STARTED + registry.add(connection, self.name, self.last_heartbeat.timestamp()) + self.save(connection=connection) + + def after_execution(self, result_ttl: int, status: JobStatus, connection: ConnectionType, + prev_registry: Optional[JobNamesRegistry] = None, + new_registry: Optional[JobNamesRegistry] = None) -> None: + """After the job is executed, update the status, heartbeat, and other metadata.""" + self.status = status + self.ended_at = utils.utcnow() + self.last_heartbeat = self.ended_at + if prev_registry is not None: + prev_registry.delete(connection, self.name) + if new_registry is not None and result_ttl != 0: + new_registry.add(connection, self.name, current_timestamp() + result_ttl) + self.save(connection=connection) + + @property + def failure_callback(self) -> Optional[Callback]: + if self.failure_callback_name is None: + return None + logger.debug(f"Running failure callbacks for {self.name}") + return Callback(self.failure_callback_name, self.failure_callback_timeout) + + @property + def success_callback(self) -> Optional[Callable[..., Any]]: + if self.success_callback_name is None: + return None + logger.debug(f"Running success callbacks for {self.name}") + return Callback(self.success_callback_name, self.success_callback_timeout) + + @property + def stopped_callback(self) -> Optional[Callable[..., Any]]: + if self.stopped_callback_name is None: + return None + logger.debug(f"Running stopped callbacks for {self.name}") + return Callback(self.stopped_callback_name, self.stopped_callback_timeout) + + def get_call_string(self): + return _get_call_string(self.func_name, self.args, self.kwargs) + + @classmethod + def create( + cls, + connection: ConnectionType, + func: FunctionReferenceType, + queue_name: str, + args: Union[List[Any], Optional[Tuple]] = None, + kwargs: Optional[Dict[str, Any]] = None, + result_ttl: Optional[int] = None, + job_info_ttl: Optional[int] = None, + status: Optional[JobStatus] = None, + description: Optional[str] = None, + timeout: Optional[int] = None, + name: Optional[str] = None, + task_type: Optional[str] = None, + scheduled_task_id: Optional[int] = None, + meta: Optional[Dict[str, Any]] = None, + *, + on_success: Optional[Callback] = None, + on_failure: Optional[Callback] = None, + on_stopped: Optional[Callback] = None, + at_front: Optional[bool] = None, + ) -> Self: + """Creates a new job-model for the given function, arguments, and keyword arguments. + :returns: A job-model instance. + """ + args = args or [] + kwargs = kwargs or {} + timeout = _parse_timeout(timeout) or SCHEDULER_CONFIG.DEFAULT_JOB_TIMEOUT + if timeout == 0: + raise ValueError("0 timeout is not allowed. Use -1 for infinite timeout") + job_info_ttl = _parse_timeout(job_info_ttl if job_info_ttl is not None else SCHEDULER_CONFIG.DEFAULT_JOB_TTL) + result_ttl = _parse_timeout(result_ttl) + if not isinstance(args, (tuple, list)): + raise TypeError(f"{args!r} is not a valid args list") + if not isinstance(kwargs, dict): + raise TypeError(f"{kwargs!r} is not a valid kwargs dict") + if on_success and not isinstance(on_success, Callback): + raise ValueError("on_success must be a Callback object") + if on_failure and not isinstance(on_failure, Callback): + raise ValueError("on_failure must be a Callback object") + if on_stopped and not isinstance(on_stopped, Callback): + raise ValueError("on_stopped must be a Callback object") + if name is not None and JobModel.exists(name, connection=connection): + raise ValueError(f"Job with name {name} already exists") + if name is None: + date_str = utils.utcnow().strftime("%Y%m%d%H%M%S%f") + name = f"{queue_name}:{scheduled_task_id or ''}:{date_str}" + + if inspect.ismethod(func): + _func_name = func.__name__ + + elif inspect.isfunction(func) or inspect.isbuiltin(func): + _func_name = f"{func.__module__}.{func.__qualname__}" + elif isinstance(func, str): + _func_name = as_str(func) + elif not inspect.isclass(func) and hasattr(func, "__call__"): # a callable class instance + _func_name = "__call__" + else: + raise TypeError(f"Expected a callable or a string, but got: {func}") + description = description or _get_call_string(func, args or [], kwargs or {}, max_length=75) + job_info_ttl = job_info_ttl if job_info_ttl is not None else SCHEDULER_CONFIG.DEFAULT_JOB_TTL + model = JobModel( + created_at=utils.utcnow(), + name=name, + queue_name=queue_name, + description=description, + func_name=_func_name, + args=args or [], + kwargs=kwargs or {}, + at_front=at_front, + task_type=task_type, + scheduled_task_id=scheduled_task_id, + success_callback_name=on_success.name if on_success else None, + success_callback_timeout=on_success.timeout if on_success else None, + failure_callback_name=on_failure.name if on_failure else None, + failure_callback_timeout=on_failure.timeout if on_failure else None, + stopped_callback_name=on_stopped.name if on_stopped else None, + stopped_callback_timeout=on_stopped.timeout if on_stopped else None, + success_ttl=result_ttl, + job_info_ttl=job_info_ttl, + timeout=timeout, + status=status, + last_heartbeat=None, + meta=meta or {}, + worker_name=None, + enqueued_at=None, + started_at=None, + ended_at=None, + ) + model.save(connection=connection) + return model + + +def _get_call_string( + func_name: Optional[str], args: Any, kwargs: Dict[Any, Any], max_length: Optional[int] = None +) -> Optional[str]: + """ + Returns a string representation of the call, formatted as a regular + Python function invocation statement. If max_length is not None, truncate + arguments with representation longer than max_length. + + :param func_name: The function name + :param args: The function arguments + :param kwargs: The function kwargs + :param max_length: The max length of the return string + :return: A string representation of the function call + """ + if func_name is None: + return None + + arg_list = [as_str(_truncate_long_string(repr(arg), max_length)) for arg in args] + + list_kwargs = [f"{k}={as_str(_truncate_long_string(repr(v), max_length))}" for k, v in kwargs.items()] + arg_list += sorted(list_kwargs) + args = ", ".join(arg_list) + + return f"{func_name}({args})" + + +def _truncate_long_string(data: str, max_length: Optional[int] = None) -> str: + """Truncate arguments with representation longer than max_length""" + if max_length is None: + return data + return (data[:max_length] + "...") if len(data) > max_length else data + + +def _parse_timeout(timeout: Union[int, float, str]) -> int: + """Transfer all kinds of timeout format to an integer representing seconds""" + if not isinstance(timeout, numbers.Integral) and timeout is not None: + try: + timeout = int(timeout) + except ValueError: + digit, unit = timeout[:-1], (timeout[-1:]).lower() + unit_second = {"d": 86400, "h": 3600, "m": 60, "s": 1} + try: + timeout = int(digit) * unit_second[unit] + except (ValueError, KeyError): + raise TimeoutFormatError( + "Timeout must be an integer or a string representing an integer, or " + 'a string with format: digits + unit, unit can be "d", "h", "m", "s", ' + 'such as "1h", "23m".' + ) + + return timeout diff --git a/scheduler/redis_models/lock.py b/scheduler/redis_models/lock.py new file mode 100644 index 0000000..e5deeb5 --- /dev/null +++ b/scheduler/redis_models/lock.py @@ -0,0 +1,31 @@ +from typing import Optional, Any + +from scheduler.broker_types import ConnectionType + + +class KvLock(object): + def __init__(self, name: str) -> None: + self.name = name + self.acquired = False + + @property + def _locking_key(self) -> str: + return f"_lock:{self.name}" + + def acquire(self, val: Any, connection: ConnectionType, expire: Optional[int] = None) -> bool: + self.acquired = connection.set(self._locking_key, val, nx=True, ex=expire) + return self.acquired + + def expire(self, connection: ConnectionType, expire: Optional[int] = None) -> bool: + return connection.expire(self._locking_key, expire) + + def release(self, connection: ConnectionType): + connection.delete(self._locking_key) + + def value(self, connection: ConnectionType) -> Any: + return connection.get(self._locking_key) + + +class SchedulerLock(KvLock): + def __init__(self, queue_name: str) -> None: + super().__init__(f"lock:scheduler:{queue_name}") diff --git a/scheduler/redis_models/registry/__init__.py b/scheduler/redis_models/registry/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scheduler/redis_models/registry/base_registry.py b/scheduler/redis_models/registry/base_registry.py new file mode 100644 index 0000000..fe5e78e --- /dev/null +++ b/scheduler/redis_models/registry/base_registry.py @@ -0,0 +1,118 @@ +import dataclasses +from collections.abc import Sequence +from typing import ClassVar, Optional, List, Self, Tuple, Any + +from scheduler.broker_types import ConnectionType +from scheduler.helpers.utils import current_timestamp +from scheduler.redis_models.base import as_str, BaseModel +from scheduler.settings import logger + + +class DequeueTimeout(Exception): + pass + + +@dataclasses.dataclass(slots=True, kw_only=True) +class ZSetModel(BaseModel): + def cleanup(self, connection: ConnectionType, timestamp: Optional[float] = None) -> None: + """Remove expired jobs from registry.""" + score = timestamp or current_timestamp() + connection.zremrangebyscore(self._key, 0, score) + + def count(self, connection: ConnectionType) -> int: + """Returns the number of jobs in this registry""" + self.cleanup(connection=connection) + return connection.zcard(self._key) + + def add(self, connection: ConnectionType, job_name: str, score: float, update_existing_only: bool = False) -> int: + return connection.zadd(self._key, {job_name: float(score)}, xx=update_existing_only) + + def delete(self, connection: ConnectionType, job_name: str) -> None: + connection.zrem(self._key, job_name) + + +class JobNamesRegistry(ZSetModel): + _element_key_template: ClassVar[str] = ":registry:{}" + + def __init__(self, connection: ConnectionType, name: str) -> None: + super().__init__(name=name) + self.connection = connection + + def __len__(self) -> int: + return self.count(self.connection) + + def __contains__(self, item: str) -> bool: + return self.connection.zrank(self._key, item) is not None + + def all(self, start: int = 0, end: int = -1) -> List[str]: + """Returns list of all job names. + + :param start: Start score/timestamp, default to 0. + :param end: End score/timestamp, default to -1 (i.e., no max score). + :returns: Returns list of all job names with timestamp from start to end + """ + self.cleanup(self.connection) + res = [as_str(job_name) for job_name in self.connection.zrange(self._key, start, end)] + logger.debug(f"Getting jobs for registry {self._key}: {len(res)} found.") + return res + + def all_with_timestamps(self, start: int = 0, end: int = -1) -> List[tuple[str, float]]: + """Returns list of all job names with their timestamps. + + :param start: Start score/timestamp, default to 0. + :param end: End score/timestamp, default to -1 (i.e., no max score). + :returns: Returns list of all job names with timestamp from start to end + """ + self.cleanup(self.connection) + res = self.connection.zrange(self._key, start, end, withscores=True) + logger.debug(f"Getting jobs for registry {self._key}: {len(res)} found.") + return [(as_str(job_name), timestamp) for job_name, timestamp in res] + + def get_first(self) -> Optional[str]: + """Returns the first job in the registry.""" + self.cleanup(self.connection) + first_job = self.connection.zrange(self._key, 0, 0) + return first_job[0].decode() if first_job else None + + def get_last_timestamp(self) -> Optional[float]: + """Returns the last timestamp in the registry.""" + self.cleanup(self.connection) + last_timestamp = self.connection.zrange(self._key, -1, -1, withscores=True) + return last_timestamp[0][1] if last_timestamp else None + + @property + def key(self) -> str: + return self._key + + @classmethod + def pop( + cls, connection: ConnectionType, registries: Sequence[Self], timeout: Optional[int] + ) -> Tuple[Optional[str], Optional[str]]: + """Helper method to abstract away from some Redis API details + + :param connection: Broker connection + :param registries: List of registries to pop from + :param timeout: Timeout in seconds + :raises ValueError: If timeout of 0 was passed + :raises DequeueTimeout: BLPOP Timeout + :returns: Tuple of registry key and job name + """ + if timeout == 0: + raise ValueError("Indefinite timeout not supported. Please pick a timeout value > 0") + registry_keys = [r.key for r in registries] + if timeout is not None: # blocking variant + colored_registries = ",".join(map(str, [str(registry) for registry in registry_keys])) + logger.debug(f"Starting BZMPOP operation for queues {colored_registries} with timeout of {timeout}") + result = connection.bzpopmin(registry_keys, timeout) + if not result: + logger.debug(f"BZMPOP timeout, no jobs found on queues {colored_registries}") + raise DequeueTimeout(timeout, registry_keys) + registry_key, job_name, timestamp = result + return as_str(registry_key), as_str(job_name) + else: # non-blocking variant + for registry_key in registry_keys: + results: List[Any] = connection.zpopmin(registry_key) + if results: + job_name, timestamp = results[0] + return as_str(registry_key), as_str(job_name) + return None, None diff --git a/scheduler/redis_models/registry/queue_registries.py b/scheduler/redis_models/registry/queue_registries.py new file mode 100644 index 0000000..cd5c086 --- /dev/null +++ b/scheduler/redis_models/registry/queue_registries.py @@ -0,0 +1,116 @@ +import time +from datetime import datetime, timedelta, timezone +from typing import ClassVar, Optional, List, Tuple + +from scheduler.broker_types import ConnectionType +from scheduler.helpers.utils import current_timestamp +from .base_registry import JobNamesRegistry +from .. import as_str +from ..job import JobModel + + +class NoSuchJobError(Exception): + pass + + +class QueuedJobRegistry(JobNamesRegistry): + _element_key_template: ClassVar[str] = ":registry:{}:queued_jobs" + + def cleanup(self, connection: ConnectionType, timestamp: Optional[float] = None) -> None: + """This method is only here to prevent errors because this method is automatically called by `count()` + and `all()` methods implemented in JobIdsRegistry.""" + pass + + def compact(self): + """Removes all "dead" jobs from the queue by cycling through it, while guaranteeing FIFO semantics.""" + compact_queue_name = f"{self._key}:compact" + + jobs_with_ts = self.all_with_timestamps() + + self.connection.rename(self._key, compact_queue_name) + + for job_name, timestamp in jobs_with_ts: + if job_name is None: + continue + if JobModel.exists(job_name, self.connection): + self.delete(connection=self.connection, job_name=job_name) + + +class FinishedJobRegistry(JobNamesRegistry): + _element_key_template: ClassVar[str] = ":registry:{}:finished_jobs" + + +class FailedJobRegistry(JobNamesRegistry): + _element_key_template: ClassVar[str] = ":registry:{}:failed_jobs" + + +class CanceledJobRegistry(JobNamesRegistry): + _element_key_template: ClassVar[str] = ":registry:{}:canceled_jobs" + + def cleanup(self, connection: ConnectionType, timestamp: Optional[float] = None) -> None: + """This method is only here to prevent errors because this method is automatically called by `count()` + and `all()` methods implemented in JobIdsRegistry.""" + pass + + +class ScheduledJobRegistry(JobNamesRegistry): + _element_key_template: ClassVar[str] = ":registry:{}:scheduled_jobs" + + def cleanup(self, connection: ConnectionType, timestamp: Optional[float] = None) -> None: + """This method is only here to prevent errors because this method is automatically called by `count()` + and `all()` methods implemented in JobIdsRegistry.""" + pass + + def schedule(self, connection: ConnectionType, job: JobModel, scheduled_datetime): + """ + Adds job to registry, scored by its execution time (in UTC). + If datetime has no tzinfo, it will assume localtimezone. + """ + # If datetime has no timezone, assume server's local timezone + if not scheduled_datetime.tzinfo: + tz = timezone(timedelta(seconds=-(time.timezone if time.daylight == 0 else time.altzone))) + scheduled_datetime = scheduled_datetime.replace(tzinfo=tz) + + timestamp = scheduled_datetime.timestamp() + return connection.zadd(self._key, {job.name: timestamp}) + + def get_jobs_to_schedule(self, timestamp: int, chunk_size: int = 1000) -> List[str]: + """Gets a list of job names that should be scheduled. + + :param timestamp: timestamp/score of jobs in SortedSet. + :param chunk_size: Max results to return. + :returns: A list of job names + """ + jobs_to_schedule = self.connection.zrangebyscore(self._key, 0, max=timestamp, start=0, num=chunk_size) + return [as_str(job_name) for job_name in jobs_to_schedule] + + def get_scheduled_time(self, job_name: str) -> datetime: + """Returns datetime (UTC) at which job is scheduled to be enqueued + + :param job_name: Job name + :raises NoSuchJobError: If the job was not found + :returns: The scheduled time as datetime object + """ + + score = self.connection.zscore(self._key, job_name) + if not score: + raise NoSuchJobError + + return datetime.fromtimestamp(score, tz=timezone.utc) + + +class ActiveJobRegistry(JobNamesRegistry): + """Registry of currently executing jobs. Each queue maintains a ActiveJobRegistry.""" + + _element_key_template: ClassVar[str] = ":registry:{}:active" + + def get_job_names_before(self, connection: ConnectionType, timestamp: Optional[float]) -> List[Tuple[str, float]]: + """Returns job names whose score is lower than a timestamp timestamp. + + Returns names for jobs with an expiry time earlier than timestamp, + specified as seconds since the Unix epoch. + timestamp defaults to calltime if unspecified. + """ + score = timestamp or current_timestamp() + jobs_before = connection.zrangebyscore(self._key, 0, score, withscores=True) + return [(as_str(job_name), score) for (job_name, score) in jobs_before] diff --git a/scheduler/redis_models/result.py b/scheduler/redis_models/result.py new file mode 100644 index 0000000..fdba94c --- /dev/null +++ b/scheduler/redis_models/result.py @@ -0,0 +1,75 @@ +import dataclasses +from datetime import datetime +from enum import Enum +from typing import Optional, Any, Self, ClassVar, List + +from scheduler.broker_types import ConnectionType +from scheduler.helpers.utils import utcnow +from scheduler.redis_models.base import StreamModel, decode_dict + + +class ResultType(Enum): + SUCCESSFUL = "successful" + FAILED = "failed" + STOPPED = "stopped" + + +@dataclasses.dataclass(slots=True, kw_only=True) +class Result(StreamModel): + parent: str + type: ResultType + worker_name: str + ttl: Optional[int] = 0 + name: Optional[str] = None + created_at: datetime = dataclasses.field(default_factory=utcnow) + return_value: Optional[Any] = None + exc_string: Optional[str] = None + + _list_key: ClassVar[str] = ":job-results:" + _children_key_template: ClassVar[str] = ":job-results:{}:" + _element_key_template: ClassVar[str] = ":job-results:{}" + + @classmethod + def create( + cls, + connection: ConnectionType, + job_name: str, + worker_name: str, + _type: ResultType, + ttl: int, + return_value: Any = None, + exc_string: Optional[str] = None, + ) -> Self: + result = cls( + parent=job_name, + ttl=ttl, type=_type, return_value=return_value, exc_string=exc_string, + worker_name=worker_name, ) + result.save(connection) + return result + + @classmethod + def fetch_latest(cls, connection: ConnectionType, job_name: str) -> Optional["Result"]: + """Returns the latest result for given job_name. + + :param connection: Broker connection. + :param job_name: Job name. + :return: Result instance or None if no result is available. + """ + response: List[Any] = connection.xrevrange(cls._children_key_template.format(job_name), "+", "-", count=1) + if not response: + return None + result_id, payload = response[0] + res = cls.deserialize(decode_dict(payload, set())) + return res + + def __repr__(self): + return f"Result(name={self.name}, type={self.type.name})" + + def __eq__(self, other: Self) -> bool: + try: + return self.name == other.name + except AttributeError: + return False + + def __bool__(self) -> bool: + return bool(self.name) diff --git a/scheduler/redis_models/worker.py b/scheduler/redis_models/worker.py new file mode 100644 index 0000000..78b4563 --- /dev/null +++ b/scheduler/redis_models/worker.py @@ -0,0 +1,121 @@ +import dataclasses +from datetime import datetime +from enum import Enum +from typing import List, Optional, Self, ClassVar, Any, Generator + +from scheduler.broker_types import ConnectionType +from scheduler.helpers.utils import utcnow +from scheduler.redis_models.base import HashModel, MAX_KEYS +from scheduler.settings import logger + +DEFAULT_WORKER_TTL = 420 + + +class WorkerStatus(str, Enum): + CREATED = "created" + STARTING = "starting" + STARTED = "started" + SUSPENDED = "suspended" + BUSY = "busy" + IDLE = "idle" + + +@dataclasses.dataclass(slots=True, kw_only=True) +class WorkerModel(HashModel): + name: str + queue_names: List[str] + pid: int + hostname: str + ip_address: str + version: str + python_version: str + state: WorkerStatus + job_execution_process_pid: int = 0 + successful_job_count: int = 0 + failed_job_count: int = 0 + completed_jobs: int = 0 + birth: Optional[datetime] = None + last_heartbeat: Optional[datetime] = None + is_suspended: bool = False + current_job_name: Optional[str] = None + stopped_job_name: Optional[str] = None + total_working_time_ms: float = 0.0 + current_job_working_time: float = 0 + last_cleaned_at: Optional[datetime] = None + shutdown_requested_date: Optional[datetime] = None + has_scheduler: bool = False + death: Optional[datetime] = None + + _list_key: ClassVar[str] = ":workers:ALL:" + _children_key_template: ClassVar[str] = ":queue-workers:{}:" + _element_key_template: ClassVar[str] = ":workers:{}" + + def save(self, connection: ConnectionType) -> None: + pipeline = connection.pipeline() + super(WorkerModel, self).save(pipeline) + for queue_name in self.queue_names: + pipeline.sadd(self._children_key_template.format(queue_name), self.name) + pipeline.expire(self._key, DEFAULT_WORKER_TTL + 60) + pipeline.execute() + + def delete(self, connection: ConnectionType) -> None: + logger.debug(f"Deleting worker {self.name}") + pipeline = connection.pipeline() + now = utcnow() + self.death = now + pipeline.hset(self._key, "death", now.isoformat()) + pipeline.expire(self._key, 60) + pipeline.srem(self._list_key, self.name) + for queue_name in self.queue_names: + pipeline.srem(self._children_key_template.format(queue_name), self.name) + pipeline.execute() + + def __eq__(self, other: Self) -> bool: + if not isinstance(other, self.__class__): + raise TypeError("Cannot compare workers to other types (of workers)") + return self._key == other._key + + def __hash__(self): + """The hash does not take the database/connection into account""" + return hash((self._key, ",".join(self.queue_names))) + + def set_current_job_working_time(self, job_execution_time: int, connection: ConnectionType) -> None: + self.set_field("current_job_working_time", job_execution_time, connection=connection) + + def heartbeat(self, connection: ConnectionType, timeout: Optional[int] = None) -> None: + timeout = timeout or DEFAULT_WORKER_TTL + 60 + connection.expire(self._key, timeout) + now = utcnow() + self.set_field("last_heartbeat", now, connection=connection) + logger.debug(f"Next heartbeat for worker {self._key} should arrive in {timeout} seconds.") + + @classmethod + def cleanup(cls, connection: ConnectionType, queue_name: Optional[str] = None): + worker_names = cls.all_names(connection, queue_name) + worker_keys = [cls.key_for(worker_name) for worker_name in worker_names] + with connection.pipeline() as pipeline: + for worker_key in worker_keys: + pipeline.exists(worker_key) + worker_exist = pipeline.execute() + invalid_workers = list() + for i, worker_key in enumerate(worker_keys): + if not worker_exist[i]: + invalid_workers.append(worker_key) + if len(invalid_workers) == 0: + return + for invalid_subset in _split_list(invalid_workers, MAX_KEYS): + pipeline.srem(cls._list_key, *invalid_subset) + if queue_name: + pipeline.srem(cls._children_key_template.format(queue_name), *invalid_subset) + pipeline.execute() + + +def _split_list(a_list: List[str], segment_size: int) -> Generator[list[str], Any, None]: + """Splits a list into multiple smaller lists having size `segment_size` + + :param a_list: The list to split + :param segment_size: The segment size to split into + :returns: The list split into smaller lists + """ + for i in range(0, len(a_list), segment_size): + yield a_list[i : i + segment_size] diff --git a/scheduler/rq_classes.py b/scheduler/rq_classes.py deleted file mode 100644 index 2014ca3..0000000 --- a/scheduler/rq_classes.py +++ /dev/null @@ -1,279 +0,0 @@ -from typing import List, Optional, Union - -import django -from django.apps import apps -from rq import Worker -from rq.command import send_stop_job_command -from rq.decorators import job -from rq.exceptions import InvalidJobOperation -from rq.job import Job, JobStatus -from rq.job import get_current_job # noqa -from rq.queue import Queue, logger -from rq.registry import ( - DeferredJobRegistry, - FailedJobRegistry, - FinishedJobRegistry, - ScheduledJobRegistry, - StartedJobRegistry, - CanceledJobRegistry, - BaseRegistry, -) -from rq.scheduler import RQScheduler -from rq.worker import WorkerStatus - -from scheduler import settings -from scheduler.broker_types import PipelineType, ConnectionType - -MODEL_NAMES = ["Task"] -TASK_TYPES = ["OnceTaskType", "RepeatableTaskType", "CronTaskType"] - -rq_job_decorator = job -ExecutionStatus = JobStatus -InvalidJobOperation = InvalidJobOperation - - -def register_sentry(sentry_dsn, **opts): - from rq.contrib.sentry import register_sentry as rq_register_sentry - - rq_register_sentry(sentry_dsn, **opts) - - -def as_str(v: Union[bytes, str]) -> Optional[str]: - """Converts a `bytes` value to a string using `utf-8`. - - :param v: The value (None/bytes/str) - :raises: ValueError: If the value is not `bytes` or `str` - :returns: Either the decoded string or None - """ - if v is None: - return None - if isinstance(v, bytes): - return v.decode("utf-8") - if isinstance(v, str): - return v - raise ValueError("Unknown type %r" % type(v)) - - -class JobExecution(Job): - def __eq__(self, other) -> bool: - return isinstance(other, Job) and self.id == other.id - - @property - def is_scheduled_task(self) -> bool: - return self.meta.get("scheduled_task_id", None) is not None - - def is_execution_of(self, task: "Task") -> bool: # noqa: F821 - return ( - self.meta.get("task_type", None) == task.task_type and self.meta.get("scheduled_task_id", None) == task.id - ) - - def stop_execution(self, connection: ConnectionType): - send_stop_job_command(connection, self.id) - - -class DjangoWorker(Worker): - def __init__(self, *args, **kwargs): - self.fork_job_execution = kwargs.pop("fork_job_execution", True) - job_class = kwargs.get("job_class") or JobExecution - if not isinstance(job_class, type) or not issubclass(job_class, JobExecution): - raise ValueError("job_class must be a subclass of JobExecution") - - # Update kwargs with the potentially modified job_class - kwargs["job_class"] = job_class - kwargs["queue_class"] = DjangoQueue - super(DjangoWorker, self).__init__(*args, **kwargs) - - def __eq__(self, other): - return isinstance(other, Worker) and self.key == other.key and self.name == other.name - - def __hash__(self): - return hash((self.name, self.key, ",".join(self.queue_names()))) - - def __str__(self): - return f"{self.name}/{','.join(self.queue_names())}" - - def _start_scheduler( - self, - burst: bool = False, - logging_level: str = "INFO", - date_format: str = "%H:%M:%S", - log_format: str = "%(asctime)s %(message)s", - ) -> None: - """Starts the scheduler process. - This is specifically designed to be run by the worker when running the `work()` method. - Instantiates the DjangoScheduler and tries to acquire a lock. - If the lock is acquired, start scheduler. - If the worker is on burst mode, just enqueues scheduled jobs and quits, - otherwise, starts the scheduler in a separate process. - - :param burst (bool, optional): Whether to work on burst mode. Defaults to False. - :param logging_level (str, optional): Logging level to use. Defaults to "INFO". - :param date_format (str, optional): Date Format. Defaults to DEFAULT_LOGGING_DATE_FORMAT. - :param log_format (str, optional): Log Format. Defaults to DEFAULT_LOGGING_FORMAT. - """ - self.scheduler = DjangoScheduler( - self.queues, - connection=self.connection, - logging_level=logging_level, - date_format=date_format, - log_format=log_format, - serializer=self.serializer, - ) - self.scheduler.acquire_locks() - if self.scheduler.acquired_locks: - if burst: - self.scheduler.enqueue_scheduled_jobs() - self.scheduler.release_locks() - else: - proc = self.scheduler.start() - self._set_property("scheduler_pid", proc.pid) - - def execute_job(self, job: "Job", queue: "Queue") -> None: - if self.fork_job_execution: - super(DjangoWorker, self).execute_job(job, queue) - else: - self.set_state(WorkerStatus.BUSY) - self.perform_job(job, queue) - self.set_state(WorkerStatus.IDLE) - - def work(self, **kwargs) -> bool: - kwargs.setdefault("with_scheduler", True) - return super(DjangoWorker, self).work(**kwargs) - - def _set_property(self, prop_name: str, val, pipeline: Optional[PipelineType] = None) -> None: - connection = pipeline if pipeline is not None else self.connection - if val is None: - connection.hdel(self.key, prop_name) - else: - connection.hset(self.key, prop_name, val) - - def _get_property(self, prop_name: str, pipeline: Optional[PipelineType] = None) -> Optional[str]: - connection = pipeline if pipeline is not None else self.connection - res = connection.hget(self.key, prop_name) - return as_str(res) - - def scheduler_pid(self) -> Optional[int]: - if len(self.queues) == 0: - logger.warning("No queues to get scheduler pid from") - return None - pid = self.connection.get(DjangoScheduler.get_locking_key(self.queues[0].name)) - return int(pid.decode()) if pid is not None else None - - -class DjangoQueue(Queue): - """A subclass of RQ's QUEUE that allows jobs to be stored temporarily to be enqueued later at the end of Django's - request/response cycle.""" - - REGISTRIES = dict( - finished="finished_job_registry", - failed="failed_job_registry", - scheduled="scheduled_job_registry", - started="started_job_registry", - deferred="deferred_job_registry", - canceled="canceled_job_registry", - ) - - def __init__(self, *args, **kwargs) -> None: - kwargs["job_class"] = JobExecution - super(DjangoQueue, self).__init__(*args, **kwargs) - - def get_registry(self, name: str) -> Union[None, BaseRegistry, "DjangoQueue"]: - name = name.lower() - if name == "queued": - return self - elif name in DjangoQueue.REGISTRIES: - return getattr(self, DjangoQueue.REGISTRIES[name]) - return None - - @property - def finished_job_registry(self) -> FinishedJobRegistry: - return FinishedJobRegistry(self.name, self.connection) - - @property - def started_job_registry(self) -> StartedJobRegistry: - return StartedJobRegistry( - self.name, - self.connection, - job_class=JobExecution, - ) - - @property - def deferred_job_registry(self) -> DeferredJobRegistry: - return DeferredJobRegistry( - self.name, - self.connection, - job_class=JobExecution, - ) - - @property - def failed_job_registry(self) -> FailedJobRegistry: - return FailedJobRegistry( - self.name, - self.connection, - job_class=JobExecution, - ) - - @property - def scheduled_job_registry(self) -> ScheduledJobRegistry: - return ScheduledJobRegistry( - self.name, - self.connection, - job_class=JobExecution, - ) - - @property - def canceled_job_registry(self) -> CanceledJobRegistry: - return CanceledJobRegistry( - self.name, - self.connection, - job_class=JobExecution, - ) - - def get_all_job_ids(self) -> List[str]: - res = list() - res.extend(self.get_job_ids()) - res.extend(self.finished_job_registry.get_job_ids()) - res.extend(self.started_job_registry.get_job_ids()) - res.extend(self.deferred_job_registry.get_job_ids()) - res.extend(self.failed_job_registry.get_job_ids()) - res.extend(self.scheduled_job_registry.get_job_ids()) - res.extend(self.canceled_job_registry.get_job_ids()) - return res - - def get_all_jobs(self) -> List[JobExecution]: - job_ids = self.get_all_job_ids() - return list(filter(lambda j: j is not None, [self.fetch_job(job_id) for job_id in job_ids])) - - def clean_registries(self) -> None: - self.started_job_registry.cleanup() - self.failed_job_registry.cleanup() - self.finished_job_registry.cleanup() - - def remove_job_id(self, job_id: str) -> None: - self.connection.lrem(self.key, 0, job_id) - - def last_job_id(self) -> Optional[str]: - return self.connection.lindex(self.key, 0) - - -class DjangoScheduler(RQScheduler): - def __init__(self, *args, **kwargs) -> None: - kwargs.setdefault("interval", settings.SCHEDULER_CONFIG.SCHEDULER_INTERVAL) - super(DjangoScheduler, self).__init__(*args, **kwargs) - - @staticmethod - def reschedule_all_jobs(): - for model_name in MODEL_NAMES: - model = apps.get_model(app_label="scheduler", model_name=model_name) - enabled_jobs = model.objects.filter(enabled=True) - for item in enabled_jobs: - logger.debug(f"Rescheduling {str(item)}") - item.save() - - def work(self) -> None: - django.setup() - super(DjangoScheduler, self).work() - - def enqueue_scheduled_jobs(self) -> None: - self.reschedule_all_jobs() - super(DjangoScheduler, self).enqueue_scheduled_jobs() diff --git a/scheduler/settings.py b/scheduler/settings.py index db770be..034b4b0 100644 --- a/scheduler/settings.py +++ b/scheduler/settings.py @@ -1,57 +1,39 @@ import logging -from dataclasses import dataclass -from enum import Enum -from typing import Callable +from typing import List, Dict from django.conf import settings from django.core.exceptions import ImproperlyConfigured -logger = logging.getLogger(__package__) +from scheduler.settings_types import SchedulerConfig, Broker, QueueConfiguration -QUEUES = dict() +logger = logging.getLogger("scheduler") +logging.basicConfig(level=logging.DEBUG) +_QUEUES: Dict[str, QueueConfiguration] = dict() +SCHEDULER_CONFIG: SchedulerConfig = SchedulerConfig() -class Broker(Enum): - REDIS = "redis" - FAKEREDIS = "fakeredis" - VALKEY = "valkey" - -@dataclass -class SchedulerConfig: - EXECUTIONS_IN_PAGE: int - DEFAULT_RESULT_TTL: int - DEFAULT_TIMEOUT: int - SCHEDULER_INTERVAL: int - BROKER: Broker - TOKEN_VALIDATION_METHOD: Callable[[str], bool] - - -def _token_validation(token: str) -> bool: - return False - - -SCHEDULER_CONFIG: SchedulerConfig = SchedulerConfig( - EXECUTIONS_IN_PAGE=20, - DEFAULT_RESULT_TTL=600, - DEFAULT_TIMEOUT=300, - SCHEDULER_INTERVAL=10, - BROKER=Broker.REDIS, - TOKEN_VALIDATION_METHOD=_token_validation, -) +class QueueNotFoundError(Exception): + pass def conf_settings(): - global QUEUES + global _QUEUES global SCHEDULER_CONFIG - QUEUES = getattr(settings, "SCHEDULER_QUEUES", None) - if QUEUES is None: + app_queues = getattr(settings, "SCHEDULER_QUEUES", None) + if app_queues is None: logger.warning("Configuration using RQ_QUEUES is deprecated. Use SCHEDULER_QUEUES instead") - QUEUES = getattr(settings, "RQ_QUEUES", None) - if QUEUES is None: + app_queues = getattr(settings, "RQ_QUEUES", None) + if app_queues is None: raise ImproperlyConfigured("You have to define SCHEDULER_QUEUES in settings.py") + for queue_name, queue_config in app_queues.items(): + if isinstance(queue_config, QueueConfiguration): + _QUEUES[queue_name] = queue_config + else: + _QUEUES[queue_name] = QueueConfiguration(**queue_config) + user_settings = getattr(settings, "SCHEDULER_CONFIG", {}) if "FAKEREDIS" in user_settings: logger.warning("Configuration using FAKEREDIS is deprecated. Use BROKER='fakeredis' instead") @@ -64,3 +46,13 @@ def conf_settings(): conf_settings() + + +def get_queue_names() -> List[str]: + return list(_QUEUES.keys()) + + +def get_queue_configuration(queue_name: str) -> QueueConfiguration: + if queue_name not in _QUEUES: + raise QueueNotFoundError(f"Queue {queue_name} not found, queues={_QUEUES.keys()}") + return _QUEUES[queue_name] diff --git a/scheduler/settings_types.py b/scheduler/settings_types.py new file mode 100644 index 0000000..a7b589f --- /dev/null +++ b/scheduler/settings_types.py @@ -0,0 +1,81 @@ +from dataclasses import dataclass +from enum import Enum +from typing import Callable, Dict, Optional, List, Tuple, Any, Self, Type + +from scheduler.timeouts import BaseDeathPenalty, UnixSignalDeathPenalty + + +class Broker(Enum): + REDIS = "redis" + FAKEREDIS = "fakeredis" + VALKEY = "valkey" + + +def _token_validation(token: str) -> bool: + return False + + +@dataclass(slots=True, kw_only=True) +class SchedulerConfig: + """Configuration for django-tasks-scheduler""" + + EXECUTIONS_IN_PAGE: int = 20 + SCHEDULER_INTERVAL: int = 10 + BROKER: Broker = Broker.REDIS + TOKEN_VALIDATION_METHOD: Callable[[str], bool] = _token_validation + CALLBACK_TIMEOUT: int = 60 # Callback timeout in seconds (success/failure/stopped) + # Default values, can be override per task + DEFAULT_SUCCESS_TTL: int = 10 * 60 # Time To Live (TTL) in seconds to keep successful job results + DEFAULT_FAILURE_TTL: int = 365 * 24 * 60 * 60 # Time To Live (TTL) in seconds to keep job failure information + DEFAULT_JOB_TTL: int = 10 * 60 # Time To Live (TTL) in seconds to keep job information + DEFAULT_JOB_TIMEOUT: int = 5 * 60 # timeout (seconds) for a job + # General configuration values + DEFAULT_WORKER_TTL: int = 10 * 60 # Time To Live (TTL) in seconds to keep worker information after last heartbeat + DEFAULT_MAINTENANCE_TASK_INTERVAL: int = 10 * 60 # The interval to run maintenance tasks in seconds. 10 minutes. + DEFAULT_JOB_MONITORING_INTERVAL: int = 30 # The interval to monitor jobs in seconds. + SCHEDULER_FALLBACK_PERIOD_SECS: int = 120 # Period (secs) to wait before requiring to reacquire locks + DEATH_PENALTY_CLASS: Type["BaseDeathPenalty"] = UnixSignalDeathPenalty + + +@dataclass(slots=True, frozen=True, kw_only=True) +class QueueConfiguration: + __CONNECTION_FIELDS__ = { + "URL", + "DB", + "UNIX_SOCKET_PATH", + "HOST", + "PORT", + "PASSWORD", + "SENTINELS", + "MASTER_NAME", + "CONNECTION_KWARGS", + } + DB: Optional[int] = None + # Redis connection parameters, either UNIX_SOCKET_PATH/URL/separate params (HOST, PORT, PASSWORD) should be provided + UNIX_SOCKET_PATH: Optional[str] = None + URL: Optional[str] = None + HOST: Optional[str] = None + PORT: Optional[int] = None + USERNAME: Optional[str] = None + PASSWORD: Optional[str] = None + + ASYNC: Optional[bool] = True + + SENTINELS: Optional[List[Tuple[str, int]]] = None + SENTINEL_KWARGS: Optional[Dict[str, str]] = None + MASTER_NAME: Optional[str] = None + CONNECTION_KWARGS: Optional[Dict[str, Any]] = None + + def __post_init__(self): + if not any((self.URL, self.UNIX_SOCKET_PATH, self.HOST, self.SENTINELS)): + raise ValueError(f"At least one of URL, UNIX_SOCKET_PATH, HOST must be provided: {self}") + if sum((self.URL is not None, self.UNIX_SOCKET_PATH is not None, self.HOST is not None)) > 1: + raise ValueError(f"Only one of URL, UNIX_SOCKET_PATH, HOST should be provided: {self}") + if self.HOST is not None and (self.PORT is None or self.DB is None): + raise ValueError(f"HOST requires PORT and DB: {self}") + + def same_connection_params(self, other: Self) -> bool: + for field in self.__CONNECTION_FIELDS__: + if getattr(self, field) != getattr(other, field): + return False + return True diff --git a/scheduler/templates/admin/scheduler/confirm_action.html b/scheduler/templates/admin/scheduler/confirm_action.html index c61b8bf..69dd45c 100644 --- a/scheduler/templates/admin/scheduler/confirm_action.html +++ b/scheduler/templates/admin/scheduler/confirm_action.html @@ -22,7 +22,7 @@