Python 3.9 migration (#63)

Author: sherwinlu

.circleci/config.yml

@@ -3,24 +3,22 @@ workflows:
    version: 2
    eb-sqs-jobs:
      jobs:
-       - test-python-3-7
-       - test-python-2-7
+       - test-python-3-9
 jobs:
-  test-python-3-7: &test-template
+  test-python-3-9:
     docker:
-      - image: circleci/python:3.7.6
+      - image: cimg/python:3.9.16
     steps:
       - add_ssh_keys
       - checkout
       - run:
-          name: Install dependencies
+          name: Install pip packages
           command: |
-            sudo pip install -r development.txt
+            python3 -m venv venv
+            . venv/bin/activate
+            pip install -r development.txt
       - run:
           name: Run tests
           command: |
+            . venv/bin/activate
             python -m django test --settings=eb_sqs.test_settings
-  test-python-2-7:
-    <<: *test-template
-    docker:
-      - image: circleci/python:2.7.16

development.txt

@@ -1,5 +1,5 @@
-boto3==1.9.86
-Django==1.10.6
-mock==2.0.0
-moto==1.3.13
-requests==2.10.0
+boto3==1.26.99
+Django==4.1.7
+mock==5.0.1
+moto==4.1.6
+requests==2.28.2

eb_sqs/auto_tasks/exceptions.py

@@ -1,12 +1,13 @@
+from typing import Any
+
+
 class RetryableTaskException(Exception):
-    def __init__(self, inner, delay=None, count_retries=None, max_retries_func=None):
-        # type: (Exception, int, bool, Any) -> None
+    def __init__(self, inner: Exception, delay: int = None, count_retries: bool = None, max_retries_func: Any = None):
         self._inner = inner
 
         self.delay = delay
         self.count_retries = count_retries
         self.max_retries_func = max_retries_func
 
-    def __repr__(self):
-        # type: () -> str
+    def __repr__(self) -> str:
         return repr(self._inner)

eb_sqs/auto_tasks/service.py

@@ -1,5 +1,6 @@
 import importlib
 import logging
+from typing import Any
 
 from eb_sqs.auto_tasks.exceptions import RetryableTaskException
 from eb_sqs.decorators import task
@@ -24,7 +25,7 @@ def _auto_task_wrapper(module_name, class_name, func_name, *args, **kwargs):
         class_ = getattr(module, class_name)  # find class
 
         auto_task_executor_service = _AutoTaskExecutorService(func_name)
-        instance = class_(auto_task_service=auto_task_executor_service)  # instantiate class using _AutoTaskExecutorService
+        instance = class_(auto_task_service=auto_task_executor_service)
 
         executor_func_name = auto_task_executor_service.get_executor_func_name()
         if executor_func_name:
@@ -54,12 +55,12 @@ def _auto_task_wrapper(module_name, class_name, func_name, *args, **kwargs):
                 exc.max_retries_func()
             else:
                 # by default log an error
-                logger.error('Reached max retries in auto task {}.{}.{} with error: {}'.format(module_name, class_name, func_name, repr(exc)))
+                logger.error('Reached max retries in auto task {}.{}.{} with error: {}'.format(module_name, class_name,
+                                                                                               func_name, repr(exc)))
 
 
 class AutoTaskService(object):
-    def register_task(self, method, queue_name=None, max_retries=None):
-        # type: (Any, str, int) -> None
+    def register_task(self, method: Any, queue_name: str = None, max_retries: int = None):
         instance = method.__self__
         class_ = instance.__class__
         func_name = method.__name__
@@ -82,14 +83,12 @@ def _auto_task_wrapper_invoker(*args, **kwargs):
 
 
 class _AutoTaskExecutorService(AutoTaskService):
-    def __init__(self, func_name):
-        # type: (str) -> None
+    def __init__(self, func_name: str):
         self._func_name = func_name
 
         self._executor_func_name = None
 
-    def register_task(self, method, queue_name=None, max_retries=None):
-        # type: (Any, str, int) -> None
+    def register_task(self, method: Any, queue_name: str = None, max_retries: int = None):
         if self._func_name == method.__name__:
             # circuit breaker to allow actually executing the method once
             instance = method.__self__
@@ -99,6 +98,5 @@ def register_task(self, method, queue_name=None, max_retries=None):
 
         super(_AutoTaskExecutorService, self).register_task(method, queue_name, max_retries)
 
-    def get_executor_func_name(self):
-        # type: () -> str
+    def get_executor_func_name(self) -> str:
         return self._executor_func_name

eb_sqs/aws/sqs_queue_client.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import, unicode_literals
+from typing import Any
 
 import boto3
 from botocore.config import Config
@@ -10,15 +10,13 @@
 
 class SqsQueueClient(QueueClient):
     def __init__(self):
-        # type: () -> None
         self.sqs = boto3.resource('sqs',
                                   region_name=settings.AWS_REGION,
                                   config=Config(retries={'max_attempts': settings.AWS_MAX_RETRIES})
                                   )
         self.queue_cache = {}
 
-    def _get_queue(self, queue_name, use_cache=True):
-        # type: (unicode, bool) -> Any
+    def _get_queue(self, queue_name: str, use_cache: bool = True) -> Any:
         full_queue_name = '{}{}'.format(settings.QUEUE_PREFIX, queue_name)
 
         queue = self._get_sqs_queue(full_queue_name, use_cache)
@@ -27,8 +25,7 @@ def _get_queue(self, queue_name, use_cache=True):
 
         return queue
 
-    def _get_sqs_queue(self, queue_name, use_cache):
-        # type: (unicode, bool) -> Any
+    def _get_sqs_queue(self, queue_name: str, use_cache: bool) -> Any:
         if use_cache and self.queue_cache.get(queue_name):
             return self.queue_cache[queue_name]
 
@@ -43,8 +40,7 @@ def _get_sqs_queue(self, queue_name, use_cache):
             else:
                 raise ex
 
-    def _add_sqs_queue(self, queue_name):
-        # type: (unicode) -> Any
+    def _add_sqs_queue(self, queue_name: str) -> Any:
         if settings.AUTO_ADD_QUEUE:
             queue = self.sqs.create_queue(
                 QueueName=queue_name,
@@ -58,8 +54,7 @@ def _add_sqs_queue(self, queue_name):
         else:
             raise QueueDoesNotExistException(queue_name)
 
-    def add_message(self, queue_name, msg, delay):
-        # type: (unicode, unicode, int) -> None
+    def add_message(self, queue_name: str, msg: str, delay: int):
         try:
             queue = self._get_queue(queue_name)
             try:

eb_sqs/decorators.py

@@ -1,19 +1,16 @@
-from __future__ import absolute_import, unicode_literals
+from typing import Any
 
 from eb_sqs import settings
 from eb_sqs.worker.worker_factory import WorkerFactory
 from eb_sqs.worker.worker_task import WorkerTask
 
 
-def _get_kwarg_val(kwargs, key, default):
-    # type: (dict, str, Any) -> Any
+def _get_kwarg_val(kwargs: dict, key: str, default: Any) -> Any:
     return kwargs.pop(key, default) if kwargs else default
 
 
-def func_delay_decorator(func, queue_name, max_retries_count, use_pickle):
-    # type: (Any, str, int, bool) -> (tuple, dict)
-    def wrapper(*args, **kwargs):
-        # type: (tuple, dict) -> Any
+def func_delay_decorator(func: Any, queue_name: str, max_retries_count: int, use_pickle: bool) -> (tuple, dict):
+    def wrapper(*args: tuple, **kwargs: dict) -> Any:
         queue = _get_kwarg_val(kwargs, 'queue_name', queue_name if queue_name else settings.DEFAULT_QUEUE)
         max_retries = _get_kwarg_val(kwargs, 'max_retries', max_retries_count if max_retries_count else settings.DEFAULT_MAX_RETRIES)
         pickle = _get_kwarg_val(kwargs, 'use_pickle', use_pickle if use_pickle else settings.USE_PICKLE)
@@ -28,10 +25,8 @@ def wrapper(*args, **kwargs):
     return wrapper
 
 
-def func_retry_decorator(worker_task):
-    # type: (WorkerTask) -> (tuple, dict)
-    def wrapper(*args, **kwargs):
-        # type: (tuple, dict) -> Any
+def func_retry_decorator(worker_task: WorkerTask) -> (tuple, dict):
+    def wrapper(*args: tuple, **kwargs: dict) -> Any:
         execute_inline = _get_kwarg_val(kwargs, 'execute_inline', False) or settings.EXECUTE_INLINE
         delay = _get_kwarg_val(kwargs, 'delay', settings.DEFAULT_DELAY)
         count_retries = _get_kwarg_val(kwargs, 'count_retries', settings.DEFAULT_COUNT_RETRIES)
@@ -42,14 +37,12 @@ def wrapper(*args, **kwargs):
 
 
 class task(object):
-    def __init__(self, queue_name=None, max_retries=None, use_pickle=None):
-        # type: (str, int, bool) -> None
+    def __init__(self, queue_name: str = None, max_retries: int = None, use_pickle: bool = None):
         self.queue_name = queue_name
         self.max_retries = max_retries
         self.use_pickle = use_pickle
 
-    def __call__(self, *args, **kwargs):
-        # type: (tuple, dict) -> Any
+    def __call__(self, *args: tuple, **kwargs: dict) -> Any:
         func = args[0]
         func.retry_num = 0
         func.delay = func_delay_decorator(func, self.queue_name, self.max_retries, self.use_pickle)

eb_sqs/management/commands/process_queue.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import, unicode_literals
-
 from django.core.management import BaseCommand, CommandError
 
 from eb_sqs.worker.service import WorkerService

eb_sqs/settings.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import, unicode_literals
-
 from django.conf import settings
 
 AWS_REGION = getattr(settings, 'EB_AWS_REGION', 'us-east-1')  # type: str

eb_sqs/test_settings.py

@@ -34,8 +34,6 @@
     },
 ]
 
-ROOT_URLCONF = 'eb_sqs.urls'
-
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,

eb_sqs/tests/tests_decorators.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import, unicode_literals
-
 from unittest import TestCase
 
 from mock import Mock
@@ -11,19 +9,18 @@
 
 
 @task()
-def dummy_task(msg):
-    # type: (unicode) -> None
+def dummy_task(msg: str):
     if not msg:
         raise Exception('No message')
 
+
 @task(queue_name='CustomQueue')
 def dummy_task_custom_queue():
-    # type: (unicode) -> None
     pass
 
+
 @task()
-def dummy_retry_task(msg):
-    # type: (unicode) -> None
+def dummy_retry_task(msg: str):
     if dummy_retry_task.retry_num == 0:
         dummy_retry_task.retry()
     else: