Merge pull request #148 from SmoFlaDru/dev-benno

Add celery + Redis for cronjobs in docker. Docker improvements. Fix remaining viz queries.

Author: Bensge

.dockerignore

@@ -2,4 +2,5 @@ venv
 .venv
 .git
 .idea
-node_modules
+node_modules
+docker-compose.override.yml

Dockerfile

@@ -7,7 +7,9 @@ RUN npm run package
 
 FROM python:3.12-slim-bookworm
 
-WORKDIR /app
+RUN groupadd -g 1234 spybot && useradd -m -u 1234 -g spybot spybot
+
+WORKDIR /home/spybot
 
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
@@ -17,8 +19,10 @@ EXPOSE 8000
 RUN pip install uv
 COPY pyproject.toml pyproject.toml
 RUN uv sync
-COPY . .
+COPY --chown=spybot:spybot . .
 
-COPY --from=frontend-build frontend/output frontend/output
+COPY --chown=spybot:spybot --from=frontend-build frontend/output frontend/output
 
+USER spybot
+RUN mkdir spybot_static
 CMD ["sh", "run.sh"]

Spybot2/__init__.py

@@ -0,0 +1,3 @@
+from .celery import app as celery_app
+
+__all__ = ("celery_app",)

Spybot2/celery.py

@@ -0,0 +1,9 @@
+import os
+
+from celery import Celery
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "Spybot2.settings")
+
+app = Celery("Spybot2")
+app.config_from_object("django.conf:settings", namespace="CELERY")
+app.autodiscover_tasks()

Spybot2/settings.py

@@ -14,6 +14,7 @@
 import environ
 import os
 import sentry_sdk
+from celery.schedules import crontab
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -50,7 +51,7 @@
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = env.bool('DEBUG', False)
 
-ALLOWED_HOSTS = [SERVER_IP, TS_IP, 'localhost', '127.0.0.1', 'spybot.localhost.direct', '192.168.59.100']
+ALLOWED_HOSTS = [SERVER_IP, TS_IP, 'localhost', '127.0.0.1', 'spybot.localhost.direct', '192.168.59.100', '192.168.59.100:20001']
 
 CSRF_TRUSTED_ORIGINS = [f"https://{SERVER_IP}"]
 
@@ -64,7 +65,6 @@
 
 INSTALLED_APPS = [
     'spybot',
-    'django_crontab',
     'django.contrib.admin',
     'django.contrib.auth',
     'django.contrib.contenttypes',
@@ -117,10 +117,6 @@
         'CONN_MAX_AGE': 3600,
         'CONN_HEALTH_CHECKS': True,
     },
-    #'default': {
-    #    'ENGINE': 'django.db.backends.sqlite3',
-    #    'NAME': BASE_DIR / 'db.sqlite3',
-    #}
 }
 
 
@@ -172,7 +168,7 @@
 # https://docs.djangoproject.com/en/4.1/howto/static-files/
 
 STATIC_URL = 'static/'
-STATIC_ROOT = '../spybot_static/'
+STATIC_ROOT = 'spybot_static/'
 
 STATICFILES_DIRS = [
     BASE_DIR / "frontend/output/",
@@ -183,13 +179,26 @@
 
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 
-# django-crontab config
+# Celery
+CELERY_BROKER_URL = "redis://redis:6379"
+CELERY_RESULT_BACKEND = "redis://redis:6379"
+
+# Celery beat
+CELERY_BEAT_SCHEDULE = {
+    'record_hourly_activity': {
+        'task': 'spybot.tasks.record_hourly_activity',
+        'schedule': crontab(minute=59), # every hour
+        'args': (),
+        'options': {},
+    },
+    'end_of_week_awards': {
+        'task': 'spybot.tasks.end_of_week_awards',
+        'schedule': crontab(minute=59, hour=23, day_of_week='sunday'), # every Sunday evening
+        'args': (),
+        'options': {},
+    },
+}
 
-CRONJOBS = [
-    ('59 23 * * SUN', 'spybot.recorder.cron.cron.end_of_week_awards'),
-    ('59 * * * *', 'spybot.recorder.cron.cron.record_hourly_activity')
-]
-CRONTAB_COMMAND_PREFIX = env.str('CRONTAB_COMMAND_PREFIX', '')
 
 
 # Testing

docker-compose-local.yml

@@ -0,0 +1,17 @@
+services:
+  spybot:
+    build:
+      dockerfile: Dockerfile
+  caddy:
+    ports:
+      - "80:80"
+#      - "443:443"
+#      - "443:443/udp"
+    volumes:
+      - $PWD/infrastructure/Caddyfile:/etc/caddy/Caddyfile
+      - caddy_data:/data
+      - caddy_config:/config
+      - static_files:/spybot_static
+  db:
+    ports:
+      - "5432:5432"

docker-compose.override.yml

@@ -3,7 +3,7 @@ services:
     image: 'ghcr.io/smofladru/spybot2:sha-${COMMIT_SHA}'
     restart: unless-stopped
     volumes:
-      - static_files:/spybot_static
+      - static_files:/home/spybot/spybot_static
       - $PWD/.env:/app/.env
     expose:
       - 8000
@@ -28,6 +28,8 @@ services:
       - spybot_pg_data:/var/lib/postgresql/data
     secrets:
       - db_password
+  redis:
+    image: redis:7.4.2-bookworm
 volumes:
   caddy_data:
   caddy_config:

infrastructure/docker-compose.yml

@@ -13,7 +13,6 @@ dependencies = [
     "Django ~=5.1",
     "ts3 ~=2.0.0b3",
     "django-environ ~=0.11.2",
-    "django-crontab ~=0.7.1",
     "num2words ~=0.5.12",
     "requests ~=2.31",
     "fido2 ~=1.1",
@@ -22,6 +21,8 @@ dependencies = [
     "sentry-sdk>=2.13.0",
     "gunicorn>=23.0.0",
     "psycopg2-binary>=2.9.10",
+    "celery>=5.4.0",
+    "redis>=5.2.1",
 ]
 
 

pyproject.toml

@@ -1,15 +1,21 @@
 #!/bin/bash
 
-ls -a
-# activate venv
-#.venv/bin/activate
-
 # copy static files to directory for http server
 .venv/bin/python manage.py collectstatic --noinput
 
 # run DB migrations if necessary
 .venv/bin/python manage.py migrate
 
+# run celery worker process
+.venv/bin/python -m celery -A Spybot2 worker -l info &
+CELERY_WORKER_JOB=$!
+trap 'kill CELERY_WORKER_JOB' EXIT HUP TERM INT
+
+# run celery beat process
+.venv/bin/python -m celery -A Spybot2 beat -l info &
+CELERY_BEAT_JOB=$!
+trap 'kill CELERY_BEAT_JOB' EXIT HUP TERM INT
+
 # start recorder in background and terminate on script exit
 .venv/bin/python manage.py recorder &
 RECORDER_JOB=$!

run.sh

@@ -0,0 +1,8 @@
+temp_file=$(mktemp)
+echo $temp_file
+ssh elcheapo -C "docker exec spybot-db-1 pg_dump -U postgres spybot" > $temp_file
+docker cp $temp_file spybot2-db-1:/import_dump.sql
+docker exec --user postgres spybot2-db-1 dropdb -f --if-exists spybot
+docker exec --user postgres spybot2-db-1 createdb spybot
+docker exec spybot2-db-1 psql -U postgres spybot -f /import_dump.sql
+rm "${temp_file}"

scripts/copy_to_local_db.sh

@@ -0,0 +1,10 @@
+from celery import shared_task
+
+
+@shared_task()
+def end_of_week_awards():
+    print("shared week awards job")
+
+@shared_task()
+def record_hourly_activity():
+    print("record hourly activity job")