diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 206b02f..1be4735 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,14 +2,14 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.5.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml - id: check-added-large-files - repo: https://github.com/asottile/pyupgrade - rev: v2.31.1 + rev: v3.15.0 hooks: - id: pyupgrade args: [--py38-plus] diff --git a/bin/run_backend.py b/bin/run_backend.py index e2bf085..bf1dfd7 100755 --- a/bin/run_backend.py +++ b/bin/run_backend.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +from __future__ import annotations + import argparse import os import time @@ -24,14 +26,14 @@ def check_running(name: str) -> bool: return False -def launch_cache(storage_directory: Optional[Path]=None): +def launch_cache(storage_directory: Path | None=None) -> None: if not storage_directory: storage_directory = get_homedir() if not check_running('cache'): Popen(["./run_redis.sh"], cwd=(storage_directory / 'cache')) -def shutdown_cache(storage_directory: Optional[Path]=None): +def shutdown_cache(storage_directory: Path | None=None) -> None: if not storage_directory: storage_directory = get_homedir() r = Redis(unix_socket_path=get_socket_path('cache')) @@ -39,12 +41,12 @@ def shutdown_cache(storage_directory: Optional[Path]=None): print('Redis cache database shutdown.') -def launch_all(): +def launch_all() -> None: launch_cache() -def check_all(stop: bool=False): - backends: Dict[str, bool] = {'cache': False} +def check_all(stop: bool=False) -> None: + backends: dict[str, bool] = {'cache': False} while True: for db_name in backends.keys(): try: @@ -65,11 +67,11 @@ def check_all(stop: bool=False): time.sleep(1) -def stop_all(): +def stop_all() -> None: shutdown_cache() -def main(): +def main() -> None: parser = argparse.ArgumentParser(description='Manage backend DBs.') parser.add_argument("--start", action='store_true', default=False, help="Start all") parser.add_argument("--stop", action='store_true', default=False, help="Stop all") diff --git a/bin/shutdown.py b/bin/shutdown.py index b1b3de3..beb0421 100755 --- a/bin/shutdown.py +++ b/bin/shutdown.py @@ -5,7 +5,7 @@ from project.default import AbstractManager -def main(): +def main() -> None: AbstractManager.force_shutdown() time.sleep(5) while True: diff --git a/bin/start.py b/bin/start.py index f9f3885..87fdfdf 100755 --- a/bin/start.py +++ b/bin/start.py @@ -5,7 +5,7 @@ from project.default import get_homedir -def main(): +def main() -> None: # Just fail if the env isn't set. get_homedir() print('Start backend (redis)...') diff --git a/bin/start_website.py b/bin/start_website.py index 3e0e147..53e74ab 100755 --- a/bin/start_website.py +++ b/bin/start_website.py @@ -14,13 +14,13 @@ class Website(AbstractManager): - def __init__(self, loglevel: Optional[int]=None): + def __init__(self, loglevel: Optional[int]=None) -> None: super().__init__(loglevel) self.script_name = 'website' self.process = self._launch_website() self.set_running() - def _launch_website(self): + def _launch_website(self) -> Popen: # type: ignore[type-arg] website_dir = get_homedir() / 'website' ip = get_config('generic', 'website_listen_ip') port = get_config('generic', 'website_listen_port') @@ -32,7 +32,7 @@ def _launch_website(self): cwd=website_dir) -def main(): +def main() -> None: w = Website() w.run(sleep_in_sec=10) diff --git a/bin/stop.py b/bin/stop.py index 8e03a8d..8e5ee72 100755 --- a/bin/stop.py +++ b/bin/stop.py @@ -8,7 +8,7 @@ from project.default import get_homedir, get_socket_path -def main(): +def main() -> None: get_homedir() p = Popen(['shutdown']) p.wait() diff --git a/bin/update.py b/bin/update.py index a557359..660b307 100755 --- a/bin/update.py +++ b/bin/update.py @@ -15,14 +15,14 @@ logging.config.dictConfig(get_config('logging')) -def compute_hash_self(): +def compute_hash_self() -> bytes: m = hashlib.sha256() with (get_homedir() / 'bin' / 'update.py').open('rb') as f: m.update(f.read()) return m.digest() -def keep_going(ignore=False): +def keep_going(ignore: bool=False) -> None: if ignore: return keep_going = input('Continue? (y/N) ') @@ -31,7 +31,7 @@ def keep_going(ignore=False): sys.exit() -def run_command(command, expect_fail: bool=False, capture_output: bool=True): +def run_command(command: str, expect_fail: bool=False, capture_output: bool=True) -> None: args = shlex.split(command) homedir = get_homedir() process = subprocess.run(args, cwd=homedir, capture_output=capture_output) @@ -42,7 +42,7 @@ def run_command(command, expect_fail: bool=False, capture_output: bool=True): sys.exit() -def check_poetry_version(): +def check_poetry_version() -> None: args = shlex.split("poetry self -V") homedir = get_homedir() process = subprocess.run(args, cwd=homedir, capture_output=True) @@ -58,7 +58,7 @@ def check_poetry_version(): sys.exit() -def main(): +def main() -> None: parser = argparse.ArgumentParser(description='Pull latest release, update dependencies, update and validate the config files, update 3rd deps for the website.') parser.add_argument('--yes', default=False, action='store_true', help='Run all commands without asking.') args = parser.parse_args() diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..6e76e80 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,8 @@ +[mypy] +strict = True +warn_return_any = False +show_error_context = True +pretty = True + +[mypy-docs.source.*] +ignore_errors = True diff --git a/project/default/__init__.py b/project/default/__init__.py index b4099e6..f752ebf 100644 --- a/project/default/__init__.py +++ b/project/default/__init__.py @@ -16,3 +16,16 @@ from .helpers import get_homedir, load_configs, get_config, safe_create_dir, get_socket_path, try_make_file # noqa os.chdir(get_homedir()) + +__all__ = [ + 'AbstractManager', + 'MissingEnv', + 'CreateDirectoryException', + 'ConfigError', + 'get_homedir', + 'load_configs', + 'get_config', + 'safe_create_dir', + 'get_socket_path', + 'try_make_file', +] diff --git a/project/default/abstractmanager.py b/project/default/abstractmanager.py index 1b96a5c..3d3fb8b 100644 --- a/project/default/abstractmanager.py +++ b/project/default/abstractmanager.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +from __future__ import annotations + import asyncio import logging import os @@ -20,18 +22,18 @@ class AbstractManager(ABC): script_name: str - def __init__(self, loglevel: Optional[int]=None): + def __init__(self, loglevel: int | None=None): self.loglevel: int = loglevel if loglevel is not None else get_config('generic', 'loglevel') or logging.INFO self.logger = logging.getLogger(f'{self.__class__.__name__}') self.logger.setLevel(self.loglevel) self.logger.info(f'Initializing {self.__class__.__name__}') - self.process: Optional[Popen] = None + self.process: Popen | None = None # type: ignore[type-arg] self.__redis = Redis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True) self.force_stop = False @staticmethod - def is_running() -> List[Tuple[str, float]]: + def is_running() -> list[tuple[str, float]]: try: r = Redis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True) for script_name, score in r.zrangebyscore('running', '-inf', '+inf', withscores=True): @@ -52,7 +54,7 @@ def is_running() -> List[Tuple[str, float]]: return [] @staticmethod - def clear_running(): + def clear_running() -> None: try: r = Redis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True) r.delete('running') @@ -60,14 +62,14 @@ def clear_running(): print('Unable to connect to redis, the system is down.') @staticmethod - def force_shutdown(): + def force_shutdown() -> None: try: r = Redis(unix_socket_path=get_socket_path('cache'), db=1, decode_responses=True) r.set('shutdown', 1) except RedisConnectionError: print('Unable to connect to redis, the system is down.') - def set_running(self, number: Optional[int]=None) -> None: + def set_running(self, number: int | None=None) -> None: if number == 0: self.__redis.zrem('running', self.script_name) else: @@ -111,7 +113,7 @@ def shutdown_requested(self) -> bool: def _to_run_forever(self) -> None: raise NotImplementedError('This method must be implemented by the child') - def _kill_process(self): + def _kill_process(self) -> None: if self.process is None: return kill_order = [signal.SIGWINCH, signal.SIGTERM, signal.SIGINT, signal.SIGKILL] @@ -167,7 +169,7 @@ def run(self, sleep_in_sec: int) -> None: def _wait_to_finish(self) -> None: self.logger.info('Not implemented, nothing to wait for.') - async def stop(self): + async def stop(self) -> None: self.force_stop = True async def _to_run_forever_async(self) -> None: @@ -176,7 +178,7 @@ async def _to_run_forever_async(self) -> None: async def _wait_to_finish_async(self) -> None: self.logger.info('Not implemented, nothing to wait for.') - async def stop_async(self): + async def stop_async(self) -> None: """Method to pass the signal handler: loop.add_signal_handler(signal.SIGTERM, lambda: loop.create_task(p.stop())) """ diff --git a/project/default/helpers.py b/project/default/helpers.py index db8192e..7e4481f 100644 --- a/project/default/helpers.py +++ b/project/default/helpers.py @@ -1,4 +1,7 @@ #!/usr/bin/env python3 + +from __future__ import annotations + import json import logging import os @@ -9,7 +12,7 @@ from . import env_global_name from .exceptions import ConfigError, CreateDirectoryException, MissingEnv -configs: Dict[str, Dict[str, Any]] = {} +configs: dict[str, dict[str, Any]] = {} logger = logging.getLogger('Helpers') @@ -34,7 +37,7 @@ def get_homedir() -> Path: @lru_cache(64) -def load_configs(path_to_config_files: Optional[Union[str, Path]]=None): +def load_configs(path_to_config_files: str | Path | None=None) -> None: global configs if configs: return @@ -57,7 +60,7 @@ def load_configs(path_to_config_files: Optional[Union[str, Path]]=None): @lru_cache(64) -def get_config(config_type: str, entry: Optional[str]=None, quiet: bool=False) -> Any: +def get_config(config_type: str, entry: str | None=None, quiet: bool=False) -> Any: """Get an entry from the given config_type file. Automatic fallback to the sample file""" global configs if not configs: @@ -96,7 +99,7 @@ def get_socket_path(name: str) -> str: return str(get_homedir() / mapping[name]) -def try_make_file(filename: Path): +def try_make_file(filename: Path) -> bool: try: filename.touch(exist_ok=False) return True diff --git a/project/projectname.py b/project/projectname.py index e510e47..bfe6ea2 100644 --- a/project/projectname.py +++ b/project/projectname.py @@ -18,8 +18,8 @@ def __init__(self) -> None: path=get_socket_path('cache'), decode_responses=True) @property - def redis(self): + def redis(self) -> Redis: # type: ignore[type-arg] return Redis(connection_pool=self.redis_pool) - def check_redis_up(self): + def check_redis_up(self) -> bool: return self.redis.ping() diff --git a/pyproject.toml b/pyproject.toml index adc9bd2..974aa7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,18 +28,3 @@ types-redis = "^4.6.0.20240106" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" - -[tool.mypy] -python_version = "3.10" -check_untyped_defs = true -ignore_errors = false -ignore_missing_imports = false -strict_optional = true -no_implicit_optional = true -warn_unused_ignores = true -warn_redundant_casts = true -warn_unused_configs = true -warn_unreachable = true - -show_error_context = true -pretty = true diff --git a/tools/validate_config_files.py b/tools/validate_config_files.py index f785211..f6c9f71 100755 --- a/tools/validate_config_files.py +++ b/tools/validate_config_files.py @@ -7,7 +7,7 @@ from project.default import get_homedir -def validate_generic_config_file(): +def validate_generic_config_file() -> bool: sample_config = get_homedir() / 'config' / 'generic.json.sample' with sample_config.open() as f: generic_config_sample = json.load(f) @@ -53,7 +53,7 @@ def validate_generic_config_file(): return True -def update_user_configs(): +def update_user_configs() -> bool: for file_name in ['generic']: with (get_homedir() / 'config' / f'{file_name}.json').open() as f: try: diff --git a/website/web/__init__.py b/website/web/__init__.py index a73267a..1ef3468 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -24,7 +24,7 @@ @api.route('/redis_up') @api.doc(description='Check if redis is up and running') -class RedisUp(Resource): +class RedisUp(Resource): # type: ignore[misc] - def get(self): + def get(self) -> bool: return project.check_redis_up() diff --git a/website/web/helpers.py b/website/web/helpers.py index acadfd1..72721fb 100644 --- a/website/web/helpers.py +++ b/website/web/helpers.py @@ -1,13 +1,17 @@ #!/usr/bin/env python3 +from __future__ import annotations + import os from functools import lru_cache from pathlib import Path +from flask import Request + from project.default import get_homedir -def src_request_ip(request) -> str: +def src_request_ip(request: Request) -> str | None: # NOTE: X-Real-IP is the IP passed by the reverse proxy in the headers. real_ip = request.headers.get('X-Real-IP') if not real_ip: