Resolve #122 -- Add Django-RQ support for async image processing (#124)

Co-authored-by: Johannes Maron <[email protected]>

Author: krtko1

.github/workflows/ci.yml

@@ -129,6 +129,7 @@ jobs:
         extras:
           - "celery"
           - "dramatiq"
+          - "django-rq"
           - "drf"
           - "cleanup"
     runs-on: ubuntu-latest
@@ -139,8 +140,8 @@ jobs:
         with:
           python-version: 3.x
       - name: Install redis
-        if: matrix.extras == 'dramatiq'
-        run: sudo apt-get install -y redis-server
+        if: matrix.extras == 'dramatiq' || matrix.extras == 'django-rq'
+        run: sudo apt install -y redis-server
       - run: python -m pip install .[test,${{ matrix.extras }}]
       - run: python -m pytest
       - uses: codecov/codecov-action@v3

README.md

@@ -9,7 +9,7 @@ Responsive cross-browser image library using modern codes like AVIF & WebP.
 * serve files with or without a CDN
 * placeholders for local development
 * migration support
-* async image processing for [Celery] or [Dramatiq]
+* async image processing for [Celery], [Dramatiq] or [Django RQ][django-rq]
 * [DRF] support
 
 [![PyPi Version](https://img.shields.io/pypi/v/django-pictures.svg)](https://pypi.python.org/pypi/django-pictures/)
@@ -287,3 +287,4 @@ which automatically deletes its file and corresponding `SimplePicture` files.
 [drf]: https://www.django-rest-framework.org/
 [celery]: https://docs.celeryproject.org/en/stable/
 [dramatiq]: https://dramatiq.io/
+[django-rq]: https://github.com/rq/django-rq

pictures/tasks.py

@@ -112,3 +112,30 @@ def process_picture(field_file: PictureFieldFile) -> None:  # noqa: F811
                 queue=conf.get_settings().QUEUE_NAME,
             )
         )
+
+
+try:
+    from django_rq import job
+except ImportError:
+    pass
+else:
+
+    @job(conf.get_settings().QUEUE_NAME)
+    def process_picture_with_django_rq(
+        app_name, model_name, field_name, file_name, storage_construct
+    ) -> None:
+        process_picture_async(
+            app_name, model_name, field_name, file_name, storage_construct
+        )
+
+    def process_picture(field_file: PictureFieldFile) -> None:  # noqa: F811
+        opts = field_file.instance._meta
+        transaction.on_commit(
+            lambda: process_picture_with_django_rq.delay(
+                app_name=opts.app_label,
+                model_name=opts.model_name,
+                field_name=field_file.field.name,
+                file_name=field_file.name,
+                storage_construct=field_file.storage.deconstruct(),
+            )
+        )

pyproject.toml

@@ -49,6 +49,9 @@ dramatiq = [
 celery = [
   "celery",
 ]
+django-rq = [
+  "django-rq",
+]
 drf = [
   "djangorestframework",
 ]

tests/conftest.py

@@ -32,7 +32,19 @@ def stub_worker():
     try:
         import dramatiq
     except ImportError:
-        yield Mock()
+        try:
+            from django_rq import get_worker
+        except ImportError:
+            yield Mock()
+        else:
+
+            class Meta:
+                @staticmethod
+                def join():
+                    get_worker("pictures").work(burst=True)
+
+            yield Meta
+
     else:
         broker = dramatiq.get_broker()
         broker.emit_after("process_boot")

tests/test_migrations.py

@@ -19,10 +19,14 @@
 except ImportError:
     celery = None
 
+try:
+    import django_rq
+except ImportError:
+    django_rq = None
 
 skip_dramatiq = pytest.mark.skipif(
-    not all(x is None for x in [dramatiq, celery]),
-    reason="dramatiq and celery are installed",
+    not all(x is None for x in [dramatiq, celery, django_rq]),
+    reason="dramatiq, celery and django-rq are installed",
 )
 
 

tests/testapp/settings.py

@@ -162,3 +162,21 @@
 
 CELERY_BROKER_URL = "redis:///2"
 CELERY_TASK_ALWAYS_EAGER = True
+
+# django-rq
+try:
+    import django_rq  # NoQA
+except ImportError:  # django_rq is not installed
+    pass
+else:
+    INSTALLED_APPS += ["django_rq"]
+
+RQ_QUEUES = {
+    "pictures": {
+        "URL": os.getenv("REDIS_URL", "redis:///3"),
+    }
+}
+
+RQ = {
+    "WORKER_CLASS": "rq.worker.SimpleWorker",
+}