diff --git a/django_lightweight_queue/app_settings.py b/django_lightweight_queue/app_settings.py index b4bd19c..42daead 100644 --- a/django_lightweight_queue/app_settings.py +++ b/django_lightweight_queue/app_settings.py @@ -1,52 +1,59 @@ -from typing import Dict, Union, Mapping, TypeVar, Callable, Optional, Sequence +from typing import Any, Dict, List, Union, Callable, Optional, Sequence -from django.conf import settings +from django.conf import settings as django_settings from . import constants from .types import Logger, QueueName -T = TypeVar('T') - -def setting(suffix: str, default: T) -> T: - attr_name = '{}{}'.format(constants.SETTING_NAME_PREFIX, suffix) - return getattr(settings, attr_name, default) - - -WORKERS = setting('WORKERS', {}) # type: Dict[QueueName, int] -BACKEND = setting( - 'BACKEND', - 'django_lightweight_queue.backends.synchronous.SynchronousBackend', -) # type: str - -LOGGER_FACTORY = setting( - 'LOGGER_FACTORY', - 'logging.getLogger', -) # type: Union[str, Callable[[str], Logger]] - -# Allow per-queue overrides of the backend. -BACKEND_OVERRIDES = setting('BACKEND_OVERRIDES', {}) # type: Mapping[QueueName, str] - -MIDDLEWARE = setting('MIDDLEWARE', ( - 'django_lightweight_queue.middleware.logging.LoggingMiddleware', - 'django_lightweight_queue.middleware.transaction.TransactionMiddleware', -)) # type: Sequence[str] - -# Apps to ignore when looking for tasks. Apps must be specified as the dotted -# name used in `INSTALLED_APPS`. This is expected to be useful when you need to -# have a file called `tasks.py` within an app, but don't want -# django-lightweight-queue to import that file. -# Note: this _doesn't_ prevent tasks being registered from these apps. -IGNORE_APPS = setting('IGNORE_APPS', ()) # type: Sequence[str] - -# Backend-specific settings -REDIS_HOST = setting('REDIS_HOST', '127.0.0.1') # type: str -REDIS_PORT = setting('REDIS_PORT', 6379) # type: int -REDIS_PASSWORD = setting('REDIS_PASSWORD', None) # type: Optional[str] -REDIS_PREFIX = setting('REDIS_PREFIX', '') # type: str - -ENABLE_PROMETHEUS = setting('ENABLE_PROMETHEUS', False) # type: bool -# Workers will export metrics on this port, and ports following it -PROMETHEUS_START_PORT = setting('PROMETHEUS_START_PORT', 9300) # type: int - -ATOMIC_JOBS = setting('ATOMIC_JOBS', True) # type: bool +class Settings: + _uses_short_names: bool = True # used later in checking for values + + +class Defaults(Settings): + WORKERS: Dict[QueueName, int] = {} + BACKEND: str = 'django_lightweight_queue.backends.synchronous.SynchronousBackend' + LOGGER_FACTORY: Union[str, Callable[[str], Logger]] = 'logging.getLogger' + # Allow per-queue overrides of the backend. + BACKEND_OVERRIDES: Dict[QueueName, str] = {} + MIDDLEWARE: Sequence[str] = ('django_lightweight_queue.middleware.logging.LoggingMiddleware',) + # Apps to ignore when looking for tasks. Apps must be specified as the dotted + # name used in `INSTALLED_APPS`. This is expected to be useful when you need to + # have a file called `tasks.py` within an app, but don't want + # django-lightweight-queue to import that file. + # Note: this _doesn't_ prevent tasks being registered from these apps. + IGNORE_APPS: Sequence[str] = () + REDIS_HOST: str = '127.0.0.1' + REDIS_PORT: int = 6379 + REDIS_PASSWORD: Optional[str] = None + REDIS_PREFIX: str = "" + ENABLE_PROMETHEUS: bool = False + # Workers will export metrics on this port, and ports following it + PROMETHEUS_START_PORT: int = 9300 + ATOMIC_JOBS: bool = True + + +class AppSettings: + def __init__(self, layers: List[Settings]) -> None: + self._layers = layers + + def add_layer(self, layer: Settings) -> None: + self._layers.append(layer) + + def __getattr__(self, name: str) -> Any: + # reverse so that later layers override earlier ones + for layer in reversed(self._layers): + # check to see if the layer is internal or external + use_short_names = getattr(layer, "_uses_short_names", False) + attr_name = ( + name + if use_short_names + else '{}{}'.format(constants.SETTING_NAME_PREFIX, name) + ) + if hasattr(layer, attr_name): + return getattr(layer, attr_name) + + raise AttributeError(f"Sorry, '{name}' is not a valid setting.") + + +app_settings = AppSettings(layers=[Defaults(), django_settings]) diff --git a/django_lightweight_queue/backends/redis.py b/django_lightweight_queue/backends/redis.py index 6d4b689..4a136f5 100644 --- a/django_lightweight_queue/backends/redis.py +++ b/django_lightweight_queue/backends/redis.py @@ -3,11 +3,11 @@ import redis -from .. import app_settings from ..job import Job from .base import BackendWithPauseResume from ..types import QueueName, WorkerNumber from ..utils import block_for_time +from ..app_settings import app_settings class RedisBackend(BackendWithPauseResume): diff --git a/django_lightweight_queue/backends/reliable_redis.py b/django_lightweight_queue/backends/reliable_redis.py index 8012763..9eca0ad 100644 --- a/django_lightweight_queue/backends/reliable_redis.py +++ b/django_lightweight_queue/backends/reliable_redis.py @@ -3,11 +3,11 @@ import redis -from .. import app_settings from ..job import Job from .base import BackendWithDeduplicate, BackendWithPauseResume from ..types import QueueName, WorkerNumber from ..utils import block_for_time, get_worker_numbers +from ..app_settings import app_settings from ..progress_logger import ProgressLogger, NULL_PROGRESS_LOGGER # Work around https://github.com/python/mypy/issues/9914. Name needs to match diff --git a/django_lightweight_queue/exposition.py b/django_lightweight_queue/exposition.py index d17f7ab..53fa53c 100644 --- a/django_lightweight_queue/exposition.py +++ b/django_lightweight_queue/exposition.py @@ -6,8 +6,8 @@ from prometheus_client.exposition import MetricsHandler -from . import app_settings from .types import QueueName, WorkerNumber +from .app_settings import app_settings def get_config_response( diff --git a/django_lightweight_queue/management/commands/queue_configuration.py b/django_lightweight_queue/management/commands/queue_configuration.py index feb6f4c..787ce98 100644 --- a/django_lightweight_queue/management/commands/queue_configuration.py +++ b/django_lightweight_queue/management/commands/queue_configuration.py @@ -2,8 +2,8 @@ from django.core.management.base import BaseCommand, CommandParser -from ... import app_settings from ...utils import get_backend, get_queue_counts, load_extra_config +from ...app_settings import app_settings from ...cron_scheduler import get_cron_config diff --git a/django_lightweight_queue/runner.py b/django_lightweight_queue/runner.py index f3811e4..11855b8 100644 --- a/django_lightweight_queue/runner.py +++ b/django_lightweight_queue/runner.py @@ -4,10 +4,10 @@ import subprocess from typing import Dict, Tuple, Callable, Optional -from . import app_settings from .types import Logger, QueueName, WorkerNumber from .utils import get_backend, set_process_title from .exposition import metrics_http_server +from .app_settings import app_settings from .machine_types import Machine from .cron_scheduler import ( CronScheduler, diff --git a/django_lightweight_queue/task.py b/django_lightweight_queue/task.py index 1e457c8..25f57ad 100644 --- a/django_lightweight_queue/task.py +++ b/django_lightweight_queue/task.py @@ -12,10 +12,10 @@ Optional, ) -from . import app_settings from .job import Job from .types import QueueName from .utils import get_backend, contribute_implied_queue_name +from .app_settings import app_settings TCallable = TypeVar('TCallable', bound=Callable[..., Any]) diff --git a/django_lightweight_queue/utils.py b/django_lightweight_queue/utils.py index 38a84f3..b3dae17 100644 --- a/django_lightweight_queue/utils.py +++ b/django_lightweight_queue/utils.py @@ -21,8 +21,9 @@ from django.core.exceptions import MiddlewareNotUsed from django.utils.module_loading import module_has_submodule -from . import constants, app_settings +from . import constants from .types import Logger, QueueName, WorkerNumber +from .app_settings import Settings, app_settings if TYPE_CHECKING: from .backends.base import BaseBackend @@ -33,6 +34,11 @@ def load_extra_config(file_path: str) -> None: + + class Overrides(Settings): + """Container for holding internally overridden settings.""" + pass + extra_settings = imp.load_source('extra_settings', file_path) def get_setting_names(module: object) -> Set[str]: @@ -53,9 +59,14 @@ def with_prefix(names: Iterable[str]) -> Set[str]: warnings.warn("Ignoring unexpected setting(s) '{}'.".format(unexpected_str)) override_names = extra_names - unexpected_names + + overrides = Overrides() + for name in override_names: short_name = name[len(constants.SETTING_NAME_PREFIX):] - setattr(app_settings, short_name, getattr(extra_settings, name)) + setattr(overrides, short_name, getattr(extra_settings, name)) + + app_settings.add_layer(overrides) @lru_cache() diff --git a/django_lightweight_queue/worker.py b/django_lightweight_queue/worker.py index da25dd2..565c31e 100644 --- a/django_lightweight_queue/worker.py +++ b/django_lightweight_queue/worker.py @@ -12,9 +12,9 @@ from django.db import connections, transaction -from . import app_settings from .types import QueueName, WorkerNumber from .utils import get_logger, get_backend, set_process_title +from .app_settings import app_settings from .backends.base import BaseBackend if app_settings.ENABLE_PROMETHEUS: diff --git a/tests/test_pause_resume.py b/tests/test_pause_resume.py index eb70d38..9a9efe1 100644 --- a/tests/test_pause_resume.py +++ b/tests/test_pause_resume.py @@ -52,7 +52,7 @@ def setUp(self) -> None: # Can't use override_settings due to the copying of the settings values into # module values at startup. @mock.patch( - 'django_lightweight_queue.app_settings.BACKEND', + 'django_lightweight_queue.app_settings.Defaults.BACKEND', new='django_lightweight_queue.backends.redis.RedisBackend', ) def test_pause_resume(self) -> None: diff --git a/tests/test_reliable_redis_backend.py b/tests/test_reliable_redis_backend.py index 6359ca3..6a88210 100644 --- a/tests/test_reliable_redis_backend.py +++ b/tests/test_reliable_redis_backend.py @@ -50,7 +50,7 @@ def mock_workers(self, workers: Mapping[str, int]) -> Iterator[None]: 'django_lightweight_queue.utils._accepting_implied_queues', new=False, ), unittest.mock.patch.dict( - 'django_lightweight_queue.app_settings.WORKERS', + 'django_lightweight_queue.app_settings.app_settings.WORKERS', workers, ): yield diff --git a/tests/test_task.py b/tests/test_task.py index 48f749b..779614b 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -30,7 +30,7 @@ def mock_workers(self, workers: Mapping[str, int]) -> Iterator[None]: 'django_lightweight_queue.utils._accepting_implied_queues', new=False, ), unittest.mock.patch.dict( - 'django_lightweight_queue.app_settings.WORKERS', + 'django_lightweight_queue.app_settings.Defaults.WORKERS', workers, ): yield @@ -54,7 +54,7 @@ def mocked_get_path(path: str) -> Any: return get_path(path) patch = mock.patch( - 'django_lightweight_queue.app_settings.BACKEND', + 'django_lightweight_queue.app_settings.Defaults.BACKEND', new='test-backend', ) patch.start()