diff --git a/django_lightweight_queue/app_settings.py b/django_lightweight_queue/app_settings.py index b4bd19c..ad33f34 100644 --- a/django_lightweight_queue/app_settings.py +++ b/django_lightweight_queue/app_settings.py @@ -1,6 +1,6 @@ -from typing import Dict, Union, Mapping, TypeVar, Callable, Optional, Sequence +from typing import Any, Dict, Union, TypeVar, Callable, Optional, Sequence -from django.conf import settings +from django.conf import settings as django_settings from . import constants from .types import Logger, QueueName @@ -8,45 +8,195 @@ T = TypeVar('T') -def setting(suffix: str, default: T) -> T: - attr_name = '{}{}'.format(constants.SETTING_NAME_PREFIX, suffix) - return getattr(settings, attr_name, default) +class Settings: + def _get(self, suffix: str, default: T) -> T: + attr_name = '{}{}'.format(constants.SETTING_NAME_PREFIX, suffix) + return getattr(django_settings, attr_name, default) + # adjustable values at runtime + _workers = None + _backend = None + _logger_factory = None + _backend_overrides = None + _middleware = None + _ignore_apps = None + _redis_host = None + _redis_port = None + _redis_password = None + _redis_prefix = None + _enable_prometheus = None + _prometheus_start_port = None + _atomic_jobs = None + _site_url = None -WORKERS = setting('WORKERS', {}) # type: Dict[QueueName, int] -BACKEND = setting( - 'BACKEND', - 'django_lightweight_queue.backends.synchronous.SynchronousBackend', -) # type: str + def get_empty_dict(self) -> Dict[Any, Any]: + """ + Declare dummy type to make mypy happy. -LOGGER_FACTORY = setting( - 'LOGGER_FACTORY', - 'logging.getLogger', -) # type: Union[str, Callable[[str], Logger]] + Mypy cannot handle types changing after instantiation, which is + exactly what happens to any setting that is a dict. This helper + method works around https://github.com/python/mypy/issues/6463 + and makes mypy happy -- at no point will we actually return + Dict[Any, Any] because between the time that mypy reads it and + the time that we actually need it, it will have been populated + with the value that we actually need it to be. + """ + return {} -# Allow per-queue overrides of the backend. -BACKEND_OVERRIDES = setting('BACKEND_OVERRIDES', {}) # type: Mapping[QueueName, str] + @property + def WORKERS(self) -> Dict[QueueName, int]: + if not self._workers: + self._workers = self._get('WORKERS', self.get_empty_dict()) + return self._workers -MIDDLEWARE = setting('MIDDLEWARE', ( - 'django_lightweight_queue.middleware.logging.LoggingMiddleware', - 'django_lightweight_queue.middleware.transaction.TransactionMiddleware', -)) # type: Sequence[str] + @WORKERS.setter + def WORKERS(self, value): + self._workers = value -# Apps to ignore when looking for tasks. Apps must be specified as the dotted -# name used in `INSTALLED_APPS`. This is expected to be useful when you need to -# have a file called `tasks.py` within an app, but don't want -# django-lightweight-queue to import that file. -# Note: this _doesn't_ prevent tasks being registered from these apps. -IGNORE_APPS = setting('IGNORE_APPS', ()) # type: Sequence[str] + @property + def BACKEND(self) -> str: + if not self._backend: + self._backend = self._get( + 'BACKEND', + 'django_lightweight_queue.backends.synchronous.SynchronousBackend', + ) + return self._backend -# Backend-specific settings -REDIS_HOST = setting('REDIS_HOST', '127.0.0.1') # type: str -REDIS_PORT = setting('REDIS_PORT', 6379) # type: int -REDIS_PASSWORD = setting('REDIS_PASSWORD', None) # type: Optional[str] -REDIS_PREFIX = setting('REDIS_PREFIX', '') # type: str + @BACKEND.setter + def BACKEND(self, value): + self._backend = value -ENABLE_PROMETHEUS = setting('ENABLE_PROMETHEUS', False) # type: bool -# Workers will export metrics on this port, and ports following it -PROMETHEUS_START_PORT = setting('PROMETHEUS_START_PORT', 9300) # type: int + @property + def LOGGER_FACTORY(self) -> Union[str, Callable[[str], Logger]]: + if not self._logger_factory: + self._logger_factory = self._get( + 'LOGGER_FACTORY', + 'logging.getLogger', + ) + return self._logger_factory -ATOMIC_JOBS = setting('ATOMIC_JOBS', True) # type: bool + @LOGGER_FACTORY.setter + def LOGGER_FACTORY(self, value): + self._logger_factory = value + + @property + def BACKEND_OVERRIDES(self) -> Dict[QueueName, str]: + # Allow per-queue overrides of the backend. + if not self._backend_overrides: + self._backend_overrides = self._get('BACKEND_OVERRIDES', self.get_empty_dict()) + return self._backend_overrides + + @BACKEND_OVERRIDES.setter + def BACKEND_OVERRIDES(self, value): + self._backend_overrides = value + + @property + def MIDDLEWARE(self) -> Sequence[str]: + if not self._middleware: + self._middleware = self._get('MIDDLEWARE', ( + 'django_lightweight_queue.middleware.logging.LoggingMiddleware', + )) + return self._middleware + + @MIDDLEWARE.setter + def MIDDLEWARE(self, value): + self._middleware = value + + @property + def IGNORE_APPS(self) -> Sequence[str]: + # Apps to ignore when looking for tasks. Apps must be specified as the dotted + # name used in `INSTALLED_APPS`. This is expected to be useful when you need to + # have a file called `tasks.py` within an app, but don't want + # django-lightweight-queue to import that file. + # Note: this _doesn't_ prevent tasks being registered from these apps. + if not self._ignore_apps: + self._ignore_apps = self._get('IGNORE_APPS', ()) + return self._ignore_apps + + @IGNORE_APPS.setter + def IGNORE_APPS(self, value): + self._ignore_apps = value + + @property + def REDIS_HOST(self) -> str: + if not self._redis_host: + self._redis_host = self._get('REDIS_HOST', '127.0.0.1') + return self._redis_host + + @REDIS_HOST.setter + def REDIS_HOST(self, value): + self._redis_host = value + + @property + def REDIS_PORT(self) -> int: + if not self._redis_port: + self._redis_port = self._get('REDIS_PORT', 6379) + return self._redis_port + + @REDIS_PORT.setter + def REDIS_PORT(self, value): + self._redis_port = value + + @property + def REDIS_PASSWORD(self) -> Optional[str]: + if not self._redis_password: + self._redis_password = self._get('REDIS_PASSWORD', None) + return self._redis_password + + @REDIS_PASSWORD.setter + def REDIS_PASSWORD(self, value): + self._redis_password = value + + @property + def REDIS_PREFIX(self) -> str: + if not self._redis_prefix: + self._redis_prefix = self._get('REDIS_PREFIX', '') + return self._redis_prefix + + @REDIS_PREFIX.setter + def REDIS_PREFIX(self, value): + self._redis_prefix = value + + @property + def ENABLE_PROMETHEUS(self) -> bool: + if not self._enable_prometheus: + self._enable_prometheus = self._get('ENABLE_PROMETHEUS', False) + return self._enable_prometheus + + @ENABLE_PROMETHEUS.setter + def ENABLE_PROMETHEUS(self, value): + self._enable_prometheus = value + + @property + def PROMETHEUS_START_PORT(self) -> int: + # Workers will export metrics on this port, and ports following it + if not self._prometheus_start_port: + self._prometheus_start_port = self._get('PROMETHEUS_START_PORT', 9300) + return self._prometheus_start_port + + @PROMETHEUS_START_PORT.setter + def PROMETHEUS_START_PORT(self, value): + self._prometheus_start_port = value + + @property + def ATOMIC_JOBS(self) -> bool: + if not self._atomic_jobs: + self._atomic_jobs = self._get('ATOMIC_JOBS', True) + return self._atomic_jobs + + @ATOMIC_JOBS.setter + def ATOMIC_JOBS(self, value): + self._atomic_jobs = value + + @property + def SITE_URL(self) -> str: + if not self._site_url: + self._site_url = self._get('SITE_URL', "http://localhost:8000") + return self._site_url + + @SITE_URL.setter + def SITE_URL(self, value): + self._site_url = value + + +app_settings = Settings() diff --git a/django_lightweight_queue/backends/debug_web.py b/django_lightweight_queue/backends/debug_web.py index b3f07de..8044805 100644 --- a/django_lightweight_queue/backends/debug_web.py +++ b/django_lightweight_queue/backends/debug_web.py @@ -1,11 +1,11 @@ import urllib.parse -from django.conf import settings from django.shortcuts import reverse from ..job import Job from .base import BaseBackend from ..types import QueueName, WorkerNumber +from ..app_settings import app_settings class DebugWebBackend(BaseBackend): @@ -20,9 +20,9 @@ class DebugWebBackend(BaseBackend): """ def enqueue(self, job: Job, queue: QueueName) -> None: - path = reverse('django-lightweight-queue:debug-run') + path = reverse('django_lightweight_queue:debug-run') query_string = urllib.parse.urlencode({'job': job.to_json()}) - url = "{}{}?{}".format(settings.SITE_URL, path, query_string) + url = "{}{}?{}".format(app_settings.SITE_URL, path, query_string) print(url) def dequeue(self, queue: QueueName, worker_num: WorkerNumber, timeout: float) -> None: diff --git a/django_lightweight_queue/backends/redis.py b/django_lightweight_queue/backends/redis.py index 6d4b689..4a136f5 100644 --- a/django_lightweight_queue/backends/redis.py +++ b/django_lightweight_queue/backends/redis.py @@ -3,11 +3,11 @@ import redis -from .. import app_settings from ..job import Job from .base import BackendWithPauseResume from ..types import QueueName, WorkerNumber from ..utils import block_for_time +from ..app_settings import app_settings class RedisBackend(BackendWithPauseResume): diff --git a/django_lightweight_queue/backends/reliable_redis.py b/django_lightweight_queue/backends/reliable_redis.py index 8012763..9eca0ad 100644 --- a/django_lightweight_queue/backends/reliable_redis.py +++ b/django_lightweight_queue/backends/reliable_redis.py @@ -3,11 +3,11 @@ import redis -from .. import app_settings from ..job import Job from .base import BackendWithDeduplicate, BackendWithPauseResume from ..types import QueueName, WorkerNumber from ..utils import block_for_time, get_worker_numbers +from ..app_settings import app_settings from ..progress_logger import ProgressLogger, NULL_PROGRESS_LOGGER # Work around https://github.com/python/mypy/issues/9914. Name needs to match diff --git a/django_lightweight_queue/exposition.py b/django_lightweight_queue/exposition.py index d17f7ab..53fa53c 100644 --- a/django_lightweight_queue/exposition.py +++ b/django_lightweight_queue/exposition.py @@ -6,8 +6,8 @@ from prometheus_client.exposition import MetricsHandler -from . import app_settings from .types import QueueName, WorkerNumber +from .app_settings import app_settings def get_config_response( diff --git a/django_lightweight_queue/management/commands/queue_configuration.py b/django_lightweight_queue/management/commands/queue_configuration.py index feb6f4c..787ce98 100644 --- a/django_lightweight_queue/management/commands/queue_configuration.py +++ b/django_lightweight_queue/management/commands/queue_configuration.py @@ -2,8 +2,8 @@ from django.core.management.base import BaseCommand, CommandParser -from ... import app_settings from ...utils import get_backend, get_queue_counts, load_extra_config +from ...app_settings import app_settings from ...cron_scheduler import get_cron_config diff --git a/django_lightweight_queue/runner.py b/django_lightweight_queue/runner.py index f3811e4..11855b8 100644 --- a/django_lightweight_queue/runner.py +++ b/django_lightweight_queue/runner.py @@ -4,10 +4,10 @@ import subprocess from typing import Dict, Tuple, Callable, Optional -from . import app_settings from .types import Logger, QueueName, WorkerNumber from .utils import get_backend, set_process_title from .exposition import metrics_http_server +from .app_settings import app_settings from .machine_types import Machine from .cron_scheduler import ( CronScheduler, diff --git a/django_lightweight_queue/task.py b/django_lightweight_queue/task.py index 1e457c8..25f57ad 100644 --- a/django_lightweight_queue/task.py +++ b/django_lightweight_queue/task.py @@ -12,10 +12,10 @@ Optional, ) -from . import app_settings from .job import Job from .types import QueueName from .utils import get_backend, contribute_implied_queue_name +from .app_settings import app_settings TCallable = TypeVar('TCallable', bound=Callable[..., Any]) diff --git a/django_lightweight_queue/utils.py b/django_lightweight_queue/utils.py index 38a84f3..cc51720 100644 --- a/django_lightweight_queue/utils.py +++ b/django_lightweight_queue/utils.py @@ -21,8 +21,9 @@ from django.core.exceptions import MiddlewareNotUsed from django.utils.module_loading import module_has_submodule -from . import constants, app_settings +from . import constants from .types import Logger, QueueName, WorkerNumber +from .app_settings import app_settings if TYPE_CHECKING: from .backends.base import BaseBackend diff --git a/django_lightweight_queue/worker.py b/django_lightweight_queue/worker.py index da25dd2..565c31e 100644 --- a/django_lightweight_queue/worker.py +++ b/django_lightweight_queue/worker.py @@ -12,9 +12,9 @@ from django.db import connections, transaction -from . import app_settings from .types import QueueName, WorkerNumber from .utils import get_logger, get_backend, set_process_title +from .app_settings import app_settings from .backends.base import BaseBackend if app_settings.ENABLE_PROMETHEUS: diff --git a/tests/test_pause_resume.py b/tests/test_pause_resume.py index eb70d38..842bb48 100644 --- a/tests/test_pause_resume.py +++ b/tests/test_pause_resume.py @@ -52,7 +52,7 @@ def setUp(self) -> None: # Can't use override_settings due to the copying of the settings values into # module values at startup. @mock.patch( - 'django_lightweight_queue.app_settings.BACKEND', + 'django_lightweight_queue.app_settings.Settings.BACKEND', new='django_lightweight_queue.backends.redis.RedisBackend', ) def test_pause_resume(self) -> None: diff --git a/tests/test_reliable_redis_backend.py b/tests/test_reliable_redis_backend.py index 6359ca3..6a88210 100644 --- a/tests/test_reliable_redis_backend.py +++ b/tests/test_reliable_redis_backend.py @@ -50,7 +50,7 @@ def mock_workers(self, workers: Mapping[str, int]) -> Iterator[None]: 'django_lightweight_queue.utils._accepting_implied_queues', new=False, ), unittest.mock.patch.dict( - 'django_lightweight_queue.app_settings.WORKERS', + 'django_lightweight_queue.app_settings.app_settings.WORKERS', workers, ): yield diff --git a/tests/test_task.py b/tests/test_task.py index 48f749b..0fe8a29 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -30,7 +30,7 @@ def mock_workers(self, workers: Mapping[str, int]) -> Iterator[None]: 'django_lightweight_queue.utils._accepting_implied_queues', new=False, ), unittest.mock.patch.dict( - 'django_lightweight_queue.app_settings.WORKERS', + 'django_lightweight_queue.app_settings.Settings.WORKERS', workers, ): yield @@ -54,7 +54,7 @@ def mocked_get_path(path: str) -> Any: return get_path(path) patch = mock.patch( - 'django_lightweight_queue.app_settings.BACKEND', + 'django_lightweight_queue.app_settings.Settings.BACKEND', new='test-backend', ) patch.start()