diff --git a/django_lightweight_queue/app_settings.py b/django_lightweight_queue/app_settings.py index d6e8e58..002cee4 100644 --- a/django_lightweight_queue/app_settings.py +++ b/django_lightweight_queue/app_settings.py @@ -1,53 +1,103 @@ -from typing import Dict, Union, Mapping, TypeVar, Callable, Optional, Sequence +""" +Internal API facade over Django settings. -from django.conf import settings +This provides a typed interface over Django settings as well as handling default +values for settings not set by users and supporting the addition of overrides in +some management commands. +""" + +from typing import Any, Dict, List, Union, Callable, Optional, Sequence + +from typing_extensions import Protocol + +from django.conf import settings as django_settings from . import constants from .types import Logger, QueueName -T = TypeVar('T') +class LongNameAdapter: + def __init__(self, target: Any) -> None: + self.target = target + + def __getattr__(self, name: str) -> Any: + return getattr(self.target, f'{constants.SETTING_NAME_PREFIX}{name}') + + +class Settings(Protocol): + WORKERS: Dict[QueueName, int] + BACKEND: str + LOGGER_FACTORY: Union[str, Callable[[str], Logger]] + + # Allow per-queue overrides of the backend. + BACKEND_OVERRIDES: Dict[QueueName, str] + MIDDLEWARE: Sequence[str] + + # Apps to ignore when looking for tasks. Apps must be specified as the dotted + # name used in `INSTALLED_APPS`. This is expected to be useful when you need to + # have a file called `tasks.py` within an app, but don't want + # django-lightweight-queue to import that file. + # Note: this _doesn't_ prevent tasks being registered from these apps. + IGNORE_APPS: Sequence[str] + + REDIS_HOST: str + REDIS_PORT: int + REDIS_PASSWORD: Optional[str] + REDIS_DATABASE: int + REDIS_PREFIX: str + + ENABLE_PROMETHEUS: bool + # Workers will export metrics on this port, and ports following it + PROMETHEUS_START_PORT: int + + ATOMIC_JOBS: bool + + +class LayeredSettings(Settings, Protocol): + def add_layer(self, layer: Settings) -> None: + ... + + +class Defaults(Settings): + WORKERS: Dict[QueueName, int] = {} + BACKEND = 'django_lightweight_queue.backends.synchronous.SynchronousBackend' + LOGGER_FACTORY = 'logging.getLogger' + + BACKEND_OVERRIDES: Dict[QueueName, str] = {} + MIDDLEWARE = ('django_lightweight_queue.middleware.logging.LoggingMiddleware',) + + IGNORE_APPS: Sequence[str] = () + + REDIS_HOST = '127.0.0.1' + REDIS_PORT = 6379 + REDIS_PASSWORD = None + REDIS_DATABASE = 0 + REDIS_PREFIX = "" -def setting(suffix: str, default: T) -> T: - attr_name = '{}{}'.format(constants.SETTING_NAME_PREFIX, suffix) - return getattr(settings, attr_name, default) + ENABLE_PROMETHEUS = False + PROMETHEUS_START_PORT = 9300 -WORKERS = setting('WORKERS', {}) # type: Dict[QueueName, int] -BACKEND = setting( - 'BACKEND', - 'django_lightweight_queue.backends.synchronous.SynchronousBackend', -) # type: str + ATOMIC_JOBS = True -LOGGER_FACTORY = setting( - 'LOGGER_FACTORY', - 'logging.getLogger', -) # type: Union[str, Callable[[str], Logger]] -# Allow per-queue overrides of the backend. -BACKEND_OVERRIDES = setting('BACKEND_OVERRIDES', {}) # type: Mapping[QueueName, str] +class AppSettings: + def __init__(self, layers: List[Settings]) -> None: + self._layers = layers -MIDDLEWARE = setting('MIDDLEWARE', ( - 'django_lightweight_queue.middleware.logging.LoggingMiddleware', - 'django_lightweight_queue.middleware.transaction.TransactionMiddleware', -)) # type: Sequence[str] + def add_layer(self, layer: Settings) -> None: + self._layers.append(layer) -# Apps to ignore when looking for tasks. Apps must be specified as the dotted -# name used in `INSTALLED_APPS`. This is expected to be useful when you need to -# have a file called `tasks.py` within an app, but don't want -# django-lightweight-queue to import that file. -# Note: this _doesn't_ prevent tasks being registered from these apps. -IGNORE_APPS = setting('IGNORE_APPS', ()) # type: Sequence[str] + def __getattr__(self, name: str) -> Any: + # reverse so that later layers override earlier ones + for layer in reversed(self._layers): + if hasattr(layer, name): + return getattr(layer, name) -# Backend-specific settings -REDIS_HOST = setting('REDIS_HOST', '127.0.0.1') # type: str -REDIS_PORT = setting('REDIS_PORT', 6379) # type: int -REDIS_PASSWORD = setting('REDIS_PASSWORD', None) # type: Optional[str] -REDIS_DATABASE = setting('REDIS_DATABASE', 0) # type: int -REDIS_PREFIX = setting('REDIS_PREFIX', '') # type: str + raise AttributeError(f"Sorry, '{name}' is not a valid setting.") -ENABLE_PROMETHEUS = setting('ENABLE_PROMETHEUS', False) # type: bool -# Workers will export metrics on this port, and ports following it -PROMETHEUS_START_PORT = setting('PROMETHEUS_START_PORT', 9300) # type: int -ATOMIC_JOBS = setting('ATOMIC_JOBS', True) # type: bool +app_settings: LayeredSettings = AppSettings(layers=[ + Defaults(), + LongNameAdapter(django_settings), +]) diff --git a/django_lightweight_queue/backends/redis.py b/django_lightweight_queue/backends/redis.py index b2e6609..41417ed 100644 --- a/django_lightweight_queue/backends/redis.py +++ b/django_lightweight_queue/backends/redis.py @@ -3,11 +3,11 @@ import redis -from .. import app_settings from ..job import Job from .base import BackendWithClear, BackendWithPauseResume from ..types import QueueName, WorkerNumber from ..utils import block_for_time +from ..app_settings import app_settings class RedisBackend(BackendWithPauseResume, BackendWithClear): diff --git a/django_lightweight_queue/backends/reliable_redis.py b/django_lightweight_queue/backends/reliable_redis.py index 70f04e1..8d75cbe 100644 --- a/django_lightweight_queue/backends/reliable_redis.py +++ b/django_lightweight_queue/backends/reliable_redis.py @@ -3,7 +3,6 @@ import redis -from .. import app_settings from ..job import Job from .base import ( BackendWithClear, @@ -12,6 +11,7 @@ ) from ..types import QueueName, WorkerNumber from ..utils import block_for_time, get_worker_numbers +from ..app_settings import app_settings from ..progress_logger import ProgressLogger, NULL_PROGRESS_LOGGER # Work around https://github.com/python/mypy/issues/9914. Name needs to match diff --git a/django_lightweight_queue/exposition.py b/django_lightweight_queue/exposition.py index d17f7ab..53fa53c 100644 --- a/django_lightweight_queue/exposition.py +++ b/django_lightweight_queue/exposition.py @@ -6,8 +6,8 @@ from prometheus_client.exposition import MetricsHandler -from . import app_settings from .types import QueueName, WorkerNumber +from .app_settings import app_settings def get_config_response( diff --git a/django_lightweight_queue/management/commands/queue_configuration.py b/django_lightweight_queue/management/commands/queue_configuration.py index 096bd57..b96a518 100644 --- a/django_lightweight_queue/management/commands/queue_configuration.py +++ b/django_lightweight_queue/management/commands/queue_configuration.py @@ -1,7 +1,7 @@ from typing import Any -from ... import app_settings from ...utils import get_backend, get_queue_counts +from ...app_settings import app_settings from ...command_utils import CommandWithExtraSettings from ...cron_scheduler import get_cron_config diff --git a/django_lightweight_queue/runner.py b/django_lightweight_queue/runner.py index 174d50f..1db541b 100644 --- a/django_lightweight_queue/runner.py +++ b/django_lightweight_queue/runner.py @@ -4,10 +4,10 @@ import subprocess from typing import Dict, Tuple, Callable, Optional -from . import app_settings from .types import Logger, QueueName, WorkerNumber from .utils import get_backend, set_process_title from .exposition import metrics_http_server +from .app_settings import app_settings from .machine_types import Machine from .cron_scheduler import ( CronScheduler, diff --git a/django_lightweight_queue/task.py b/django_lightweight_queue/task.py index 1e457c8..25f57ad 100644 --- a/django_lightweight_queue/task.py +++ b/django_lightweight_queue/task.py @@ -12,10 +12,10 @@ Optional, ) -from . import app_settings from .job import Job from .types import QueueName from .utils import get_backend, contribute_implied_queue_name +from .app_settings import app_settings TCallable = TypeVar('TCallable', bound=Callable[..., Any]) diff --git a/django_lightweight_queue/utils.py b/django_lightweight_queue/utils.py index ab14344..e3d45b7 100644 --- a/django_lightweight_queue/utils.py +++ b/django_lightweight_queue/utils.py @@ -21,8 +21,9 @@ from django.core.exceptions import MiddlewareNotUsed from django.utils.module_loading import module_has_submodule -from . import constants, app_settings +from . import constants from .types import Logger, QueueName, WorkerNumber +from .app_settings import Defaults, app_settings, LongNameAdapter if TYPE_CHECKING: from .backends.base import BaseBackend @@ -47,7 +48,7 @@ def with_prefix(names: Iterable[str]) -> Set[str]: for name in names ) - setting_names = get_setting_names(app_settings) + setting_names = get_setting_names(Defaults()) extra_names = get_setting_names(extra_settings) unexpected_names = extra_names - with_prefix(setting_names) @@ -55,10 +56,7 @@ def with_prefix(names: Iterable[str]) -> Set[str]: unexpected_str = "' ,'".join(unexpected_names) warnings.warn("Ignoring unexpected setting(s) '{}'.".format(unexpected_str)) - override_names = extra_names - unexpected_names - for name in override_names: - short_name = name[len(constants.SETTING_NAME_PREFIX):] - setattr(app_settings, short_name, getattr(extra_settings, name)) + app_settings.add_layer(LongNameAdapter(extra_settings)) @lru_cache() diff --git a/django_lightweight_queue/worker.py b/django_lightweight_queue/worker.py index 74b12fb..9465c96 100644 --- a/django_lightweight_queue/worker.py +++ b/django_lightweight_queue/worker.py @@ -12,9 +12,9 @@ from django.db import connections, transaction -from . import app_settings from .types import QueueName, WorkerNumber from .utils import get_logger, get_backend, set_process_title +from .app_settings import app_settings from .backends.base import BaseBackend if app_settings.ENABLE_PROMETHEUS: diff --git a/tests/test_extra_settings.py b/tests/test_extra_settings.py index 4a6b78c..98f8edb 100644 --- a/tests/test_extra_settings.py +++ b/tests/test_extra_settings.py @@ -1,6 +1,6 @@ -import importlib from typing import Optional from pathlib import Path +from unittest import mock from django.test import SimpleTestCase @@ -27,11 +27,19 @@ def length(self, queue: QueueName) -> int: class ExtraSettingsTests(SimpleTestCase): def setUp(self) -> None: get_backend.cache_clear() + + self.settings: app_settings.Settings = app_settings.AppSettings([app_settings.Defaults()]) + self._settings_patch = mock.patch( + 'django_lightweight_queue.utils.app_settings', + new=self.settings, + ) + self._settings_patch.start() + super().setUp() def tearDown(self) -> None: - importlib.reload(app_settings) get_backend.cache_clear() + self._settings_patch.stop() super().tearDown() def test_updates_settings(self) -> None: @@ -40,20 +48,20 @@ def test_updates_settings(self) -> None: backend = get_backend('test-queue') self.assertIsInstance(backend, TestBackend) - self.assertEqual('a very bad password', app_settings.REDIS_PASSWORD) + self.assertEqual('a very bad password', self.settings.REDIS_PASSWORD) def test_warns_about_unexpected_settings(self) -> None: with self.assertWarnsRegex(Warning, r'Ignoring unexpected setting.+\bNOT_REDIS_PASSWORD\b'): load_extra_settings(str(TESTS_DIR / '_demo_extra_settings_unexpected.py')) - self.assertEqual('expected', app_settings.REDIS_PASSWORD) + self.assertEqual('expected', self.settings.REDIS_PASSWORD) def test_updates_settings_with_falsey_values(self) -> None: load_extra_settings(str(TESTS_DIR / '_demo_extra_settings.py')) load_extra_settings(str(TESTS_DIR / '_demo_extra_settings_falsey.py')) - self.assertIsNone(app_settings.REDIS_PASSWORD) - self.assertFalse(app_settings.ATOMIC_JOBS) + self.assertIsNone(self.settings.REDIS_PASSWORD) + self.assertFalse(self.settings.ATOMIC_JOBS) def test_rejects_missing_file(self) -> None: with self.assertRaises(FileNotFoundError): diff --git a/tests/test_pause_resume.py b/tests/test_pause_resume.py index eb70d38..b33c265 100644 --- a/tests/test_pause_resume.py +++ b/tests/test_pause_resume.py @@ -1,11 +1,11 @@ import io import datetime -import unittest from unittest import mock import fakeredis import freezegun +from django.test import SimpleTestCase, override_settings from django.core.management import call_command, CommandError from django_lightweight_queue.types import QueueName @@ -18,7 +18,7 @@ ) -class PauseResumeTests(unittest.TestCase): +class PauseResumeTests(SimpleTestCase): longMessage = True def assertPaused(self, queue: QueueName, context: str) -> None: @@ -49,11 +49,8 @@ def setUp(self) -> None: super().setUp() - # Can't use override_settings due to the copying of the settings values into - # module values at startup. - @mock.patch( - 'django_lightweight_queue.app_settings.BACKEND', - new='django_lightweight_queue.backends.redis.RedisBackend', + @override_settings( + LIGHTWEIGHT_QUEUE_BACKEND='django_lightweight_queue.backends.redis.RedisBackend', ) def test_pause_resume(self) -> None: QUEUE = QueueName('test-pauseable-queue') diff --git a/tests/test_reliable_redis_backend.py b/tests/test_reliable_redis_backend.py index cae1f35..9ed6132 100644 --- a/tests/test_reliable_redis_backend.py +++ b/tests/test_reliable_redis_backend.py @@ -8,6 +8,8 @@ import fakeredis +from django.test import SimpleTestCase, override_settings + from django_lightweight_queue.job import Job from django_lightweight_queue.types import QueueName from django_lightweight_queue.backends.reliable_redis import ( @@ -18,7 +20,7 @@ from .mixins import RedisCleanupMixin -class ReliableRedisTests(RedisCleanupMixin, unittest.TestCase): +class ReliableRedisTests(RedisCleanupMixin, SimpleTestCase): longMessage = True prefix = settings.LIGHTWEIGHT_QUEUE_REDIS_PREFIX @@ -49,9 +51,8 @@ def mock_workers(self, workers: Mapping[str, int]) -> Iterator[None]: with unittest.mock.patch( 'django_lightweight_queue.utils._accepting_implied_queues', new=False, - ), unittest.mock.patch.dict( - 'django_lightweight_queue.app_settings.WORKERS', - workers, + ), override_settings( + LIGHTWEIGHT_QUEUE_WORKERS=workers, ): yield diff --git a/tests/test_task.py b/tests/test_task.py index 3edace1..d96f55d 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -5,6 +5,8 @@ import fakeredis +from django.test import SimpleTestCase, override_settings + from django_lightweight_queue import task from django_lightweight_queue.types import QueueName, WorkerNumber from django_lightweight_queue.utils import get_path, get_backend @@ -20,7 +22,8 @@ def dummy_task(num: int) -> None: pass -class TaskTests(unittest.TestCase): +@override_settings(LIGHTWEIGHT_QUEUE_BACKEND='test-backend') +class TaskTests(SimpleTestCase): longMessage = True prefix = settings.LIGHTWEIGHT_QUEUE_REDIS_PREFIX @@ -29,9 +32,8 @@ def mock_workers(self, workers: Mapping[str, int]) -> Iterator[None]: with unittest.mock.patch( 'django_lightweight_queue.utils._accepting_implied_queues', new=False, - ), unittest.mock.patch.dict( - 'django_lightweight_queue.app_settings.WORKERS', - workers, + ), override_settings( + LIGHTWEIGHT_QUEUE_WORKERS=workers, ): yield @@ -53,12 +55,6 @@ def mocked_get_path(path: str) -> Any: return lambda: self.backend return get_path(path) - backend_patch = mock.patch( - 'django_lightweight_queue.app_settings.BACKEND', - new='test-backend', - ) - backend_patch.start() - self.addCleanup(backend_patch.stop) get_path_patch = mock.patch( 'django_lightweight_queue.utils.get_path', side_effect=mocked_get_path,