Merge branch 'allow-runtime-settings-overrides'

Author: PeterJCLaw

django_lightweight_queue/app_settings.py

@@ -1,53 +1,103 @@
-from typing import Dict, Union, Mapping, TypeVar, Callable, Optional, Sequence
+"""
+Internal API facade over Django settings.
 
-from django.conf import settings
+This provides a typed interface over Django settings as well as handling default
+values for settings not set by users and supporting the addition of overrides in
+some management commands.
+"""
+
+from typing import Any, Dict, List, Union, Callable, Optional, Sequence
+
+from typing_extensions import Protocol
+
+from django.conf import settings as django_settings
 
 from . import constants
 from .types import Logger, QueueName
 
-T = TypeVar('T')
 
+class LongNameAdapter:
+    def __init__(self, target: Any) -> None:
+        self.target = target
+
+    def __getattr__(self, name: str) -> Any:
+        return getattr(self.target, f'{constants.SETTING_NAME_PREFIX}{name}')
+
+
+class Settings(Protocol):
+    WORKERS: Dict[QueueName, int]
+    BACKEND: str
+    LOGGER_FACTORY: Union[str, Callable[[str], Logger]]
+
+    # Allow per-queue overrides of the backend.
+    BACKEND_OVERRIDES: Dict[QueueName, str]
+    MIDDLEWARE: Sequence[str]
+
+    # Apps to ignore when looking for tasks. Apps must be specified as the dotted
+    # name used in `INSTALLED_APPS`. This is expected to be useful when you need to
+    # have a file called `tasks.py` within an app, but don't want
+    # django-lightweight-queue to import that file.
+    # Note: this _doesn't_ prevent tasks being registered from these apps.
+    IGNORE_APPS: Sequence[str]
+
+    REDIS_HOST: str
+    REDIS_PORT: int
+    REDIS_PASSWORD: Optional[str]
+    REDIS_DATABASE: int
+    REDIS_PREFIX: str
+
+    ENABLE_PROMETHEUS: bool
+    # Workers will export metrics on this port, and ports following it
+    PROMETHEUS_START_PORT: int
+
+    ATOMIC_JOBS: bool
+
+
+class LayeredSettings(Settings, Protocol):
+    def add_layer(self, layer: Settings) -> None:
+        ...
+
+
+class Defaults(Settings):
+    WORKERS: Dict[QueueName, int] = {}
+    BACKEND = 'django_lightweight_queue.backends.synchronous.SynchronousBackend'
+    LOGGER_FACTORY = 'logging.getLogger'
+
+    BACKEND_OVERRIDES: Dict[QueueName, str] = {}
+    MIDDLEWARE = ('django_lightweight_queue.middleware.logging.LoggingMiddleware',)
+
+    IGNORE_APPS: Sequence[str] = ()
+
+    REDIS_HOST = '127.0.0.1'
+    REDIS_PORT = 6379
+    REDIS_PASSWORD = None
+    REDIS_DATABASE = 0
+    REDIS_PREFIX = ""
 
-def setting(suffix: str, default: T) -> T:
-    attr_name = '{}{}'.format(constants.SETTING_NAME_PREFIX, suffix)
-    return getattr(settings, attr_name, default)
+    ENABLE_PROMETHEUS = False
 
+    PROMETHEUS_START_PORT = 9300
 
-WORKERS = setting('WORKERS', {})  # type: Dict[QueueName, int]
-BACKEND = setting(
-    'BACKEND',
-    'django_lightweight_queue.backends.synchronous.SynchronousBackend',
-)  # type: str
+    ATOMIC_JOBS = True
 
-LOGGER_FACTORY = setting(
-    'LOGGER_FACTORY',
-    'logging.getLogger',
-)  # type: Union[str, Callable[[str], Logger]]
 
-# Allow per-queue overrides of the backend.
-BACKEND_OVERRIDES = setting('BACKEND_OVERRIDES', {})  # type: Mapping[QueueName, str]
+class AppSettings:
+    def __init__(self, layers: List[Settings]) -> None:
+        self._layers = layers
 
-MIDDLEWARE = setting('MIDDLEWARE', (
-    'django_lightweight_queue.middleware.logging.LoggingMiddleware',
-    'django_lightweight_queue.middleware.transaction.TransactionMiddleware',
-))  # type: Sequence[str]
+    def add_layer(self, layer: Settings) -> None:
+        self._layers.append(layer)
 
-# Apps to ignore when looking for tasks. Apps must be specified as the dotted
-# name used in `INSTALLED_APPS`. This is expected to be useful when you need to
-# have a file called `tasks.py` within an app, but don't want
-# django-lightweight-queue to import that file.
-# Note: this _doesn't_ prevent tasks being registered from these apps.
-IGNORE_APPS = setting('IGNORE_APPS', ())  # type: Sequence[str]
+    def __getattr__(self, name: str) -> Any:
+        # reverse so that later layers override earlier ones
+        for layer in reversed(self._layers):
+            if hasattr(layer, name):
+                return getattr(layer, name)
 
-# Backend-specific settings
-REDIS_HOST = setting('REDIS_HOST', '127.0.0.1')  # type: str
-REDIS_PORT = setting('REDIS_PORT', 6379)  # type: int
-REDIS_PASSWORD = setting('REDIS_PASSWORD', None)  # type: Optional[str]
-REDIS_DATABASE = setting('REDIS_DATABASE', 0)  # type: int
-REDIS_PREFIX = setting('REDIS_PREFIX', '')  # type: str
+        raise AttributeError(f"Sorry, '{name}' is not a valid setting.")
 
-ENABLE_PROMETHEUS = setting('ENABLE_PROMETHEUS', False)  # type: bool
-# Workers will export metrics on this port, and ports following it
-PROMETHEUS_START_PORT = setting('PROMETHEUS_START_PORT', 9300)  # type: int
 
-ATOMIC_JOBS = setting('ATOMIC_JOBS', True)  # type: bool
+app_settings: LayeredSettings = AppSettings(layers=[
+    Defaults(),
+    LongNameAdapter(django_settings),
+])

django_lightweight_queue/backends/redis.py

@@ -3,11 +3,11 @@
 
 import redis
 
-from .. import app_settings
 from ..job import Job
 from .base import BackendWithClear, BackendWithPauseResume
 from ..types import QueueName, WorkerNumber
 from ..utils import block_for_time
+from ..app_settings import app_settings
 
 
 class RedisBackend(BackendWithPauseResume, BackendWithClear):

django_lightweight_queue/backends/reliable_redis.py

@@ -3,7 +3,6 @@
 
 import redis
 
-from .. import app_settings
 from ..job import Job
 from .base import (
     BackendWithClear,
@@ -12,6 +11,7 @@
 )
 from ..types import QueueName, WorkerNumber
 from ..utils import block_for_time, get_worker_numbers
+from ..app_settings import app_settings
 from ..progress_logger import ProgressLogger, NULL_PROGRESS_LOGGER
 
 # Work around https://github.com/python/mypy/issues/9914. Name needs to match

django_lightweight_queue/exposition.py

@@ -6,8 +6,8 @@
 
 from prometheus_client.exposition import MetricsHandler
 
-from . import app_settings
 from .types import QueueName, WorkerNumber
+from .app_settings import app_settings
 
 
 def get_config_response(

django_lightweight_queue/management/commands/queue_configuration.py

@@ -1,7 +1,7 @@
 from typing import Any
 
-from ... import app_settings
 from ...utils import get_backend, get_queue_counts
+from ...app_settings import app_settings
 from ...command_utils import CommandWithExtraSettings
 from ...cron_scheduler import get_cron_config
 

django_lightweight_queue/runner.py

@@ -4,10 +4,10 @@
 import subprocess
 from typing import Dict, Tuple, Callable, Optional
 
-from . import app_settings
 from .types import Logger, QueueName, WorkerNumber
 from .utils import get_backend, set_process_title
 from .exposition import metrics_http_server
+from .app_settings import app_settings
 from .machine_types import Machine
 from .cron_scheduler import (
     CronScheduler,

django_lightweight_queue/task.py

@@ -12,10 +12,10 @@
     Optional,
 )
 
-from . import app_settings
 from .job import Job
 from .types import QueueName
 from .utils import get_backend, contribute_implied_queue_name
+from .app_settings import app_settings
 
 TCallable = TypeVar('TCallable', bound=Callable[..., Any])
 

django_lightweight_queue/utils.py

@@ -21,8 +21,9 @@
 from django.core.exceptions import MiddlewareNotUsed
 from django.utils.module_loading import module_has_submodule
 
-from . import constants, app_settings
+from . import constants
 from .types import Logger, QueueName, WorkerNumber
+from .app_settings import Defaults, app_settings, LongNameAdapter
 
 if TYPE_CHECKING:
     from .backends.base import BaseBackend
@@ -47,18 +48,15 @@ def with_prefix(names: Iterable[str]) -> Set[str]:
             for name in names
         )
 
-    setting_names = get_setting_names(app_settings)
+    setting_names = get_setting_names(Defaults())
     extra_names = get_setting_names(extra_settings)
 
     unexpected_names = extra_names - with_prefix(setting_names)
     if unexpected_names:
         unexpected_str = "' ,'".join(unexpected_names)
         warnings.warn("Ignoring unexpected setting(s) '{}'.".format(unexpected_str))
 
-    override_names = extra_names - unexpected_names
-    for name in override_names:
-        short_name = name[len(constants.SETTING_NAME_PREFIX):]
-        setattr(app_settings, short_name, getattr(extra_settings, name))
+    app_settings.add_layer(LongNameAdapter(extra_settings))
 
 
 @lru_cache()

django_lightweight_queue/worker.py

@@ -12,9 +12,9 @@
 
 from django.db import connections, transaction
 
-from . import app_settings
 from .types import QueueName, WorkerNumber
 from .utils import get_logger, get_backend, set_process_title
+from .app_settings import app_settings
 from .backends.base import BaseBackend
 
 if app_settings.ENABLE_PROMETHEUS:

tests/test_extra_settings.py

@@ -1,6 +1,6 @@
-import importlib
 from typing import Optional
 from pathlib import Path
+from unittest import mock
 
 from django.test import SimpleTestCase
 
@@ -27,11 +27,19 @@ def length(self, queue: QueueName) -> int:
 class ExtraSettingsTests(SimpleTestCase):
     def setUp(self) -> None:
         get_backend.cache_clear()
+
+        self.settings: app_settings.Settings = app_settings.AppSettings([app_settings.Defaults()])
+        self._settings_patch = mock.patch(
+            'django_lightweight_queue.utils.app_settings',
+            new=self.settings,
+        )
+        self._settings_patch.start()
+
         super().setUp()
 
     def tearDown(self) -> None:
-        importlib.reload(app_settings)
         get_backend.cache_clear()
+        self._settings_patch.stop()
         super().tearDown()
 
     def test_updates_settings(self) -> None:
@@ -40,20 +48,20 @@ def test_updates_settings(self) -> None:
         backend = get_backend('test-queue')
         self.assertIsInstance(backend, TestBackend)
 
-        self.assertEqual('a very bad password', app_settings.REDIS_PASSWORD)
+        self.assertEqual('a very bad password', self.settings.REDIS_PASSWORD)
 
     def test_warns_about_unexpected_settings(self) -> None:
         with self.assertWarnsRegex(Warning, r'Ignoring unexpected setting.+\bNOT_REDIS_PASSWORD\b'):
             load_extra_settings(str(TESTS_DIR / '_demo_extra_settings_unexpected.py'))
 
-        self.assertEqual('expected', app_settings.REDIS_PASSWORD)
+        self.assertEqual('expected', self.settings.REDIS_PASSWORD)
 
     def test_updates_settings_with_falsey_values(self) -> None:
         load_extra_settings(str(TESTS_DIR / '_demo_extra_settings.py'))
         load_extra_settings(str(TESTS_DIR / '_demo_extra_settings_falsey.py'))
 
-        self.assertIsNone(app_settings.REDIS_PASSWORD)
-        self.assertFalse(app_settings.ATOMIC_JOBS)
+        self.assertIsNone(self.settings.REDIS_PASSWORD)
+        self.assertFalse(self.settings.ATOMIC_JOBS)
 
     def test_rejects_missing_file(self) -> None:
         with self.assertRaises(FileNotFoundError):