added logging

.filebeat/filebeat.yml

@@ -0,0 +1,8 @@
+filebeat.inputs:
+- type: log
+  enabled: true
+  paths:
+    - /usr/share/filebeat/logs/*.log
+
+output.logstash:
+  hosts: ["logstash:5044"]

.gitignore

@@ -3,6 +3,7 @@
 /.vscode/
 /src/media/
 /src/staticfiles/
+*log
 *.mo
 *.pyc
 htmlcov/

.logstash/Dockerfile

@@ -0,0 +1,3 @@
+FROM docker.elastic.co/logstash/logstash:8.13.0
+
+COPY pipeline/logstash.conf /usr/share/logstash/pipeline

.logstash/pipeline/logstash.conf

@@ -0,0 +1,22 @@
+
+input {
+  beats {
+    port => 5044
+    ssl => false
+  }
+}
+
+filter{
+  json {
+    source => "message"
+    target => "jsoncontent"
+  }
+}
+
+output {
+  elasticsearch {
+    hosts => "elasticsearch:9200"
+    manage_template => false
+    index => "elk-%{+YYYY.MM.dd}"
+  }
+}

docker-compose.yaml

@@ -49,7 +49,7 @@ services:
     restart: always
     ports:
       - "6379:6379"
-  
+
   # Production service - "service", "image", and "container_name" should all contain the same
   # reference, based on the name of the service.
   sdnginx:
@@ -109,9 +109,53 @@ services:
       - celery
       - celery-beat
 
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2
+    container_name: signal_documentation-elasticsearch
+    environment:
+      - discovery.type=single-node
+    ports:
+      - "9200:9200"
+      - "9300:9300"
+
+  kinaba:
+    image: docker.elastic.co/kibana/kibana:7.10.2
+    container_name: signal_documentation-kibana
+    ports:
+      - "5601:5601"
+    links:
+      - elasticsearch
+    depends_on:
+      - elasticsearch
+
+  logstash:
+    image: logstash:latest
+    restart: always
+    build:
+      context: .logstash
+      dockerfile: Dockerfile
+    depends_on:
+      - elasticsearch
+    volumes:
+      - logs_volume:/logs:ro
+
+  filebeat:
+    image: docker.elastic.co/beats/filebeat:7.10.2
+    container_name: signal_documentation-filebeat
+    volumes:
+      - ./filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml
+      - ./src/logs:/usr/share/filebeat/logs
+    environment:
+      LASTICSEARCH_URL: http://elasticsearch:9200
+    links:
+      - elasticsearch
+    depends_on:
+      - elasticsearch
+
 volumes:
   mysql:
   sdwebapp:
   static:
   celery:
   celery-beat:
+  logs_volume:

src/signal_documentation/settings.py

@@ -10,6 +10,7 @@
 https://docs.djangoproject.com/en/4.2/ref/settings/
 """
 import os
+from distutils.util import strtobool
 from pathlib import Path
 from typing import Any
 
@@ -43,7 +44,7 @@
 
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG: bool = True
+DEBUG = bool(strtobool(os.getenv('DEBUG', 'False')))
 
 
 # SECURITY WARNING: keep the secret key used in production secret!
@@ -52,7 +53,7 @@
 else:
     SECRET_KEY: str | None = os.environ.get('SECRET_KEY')   # type: ignore
     if not SECRET_KEY:
         raise RuntimeError('Could not find a SECRET_KEY in environment')
 
 ALLOWED_HOSTS: list[str] = os.environ.get('ALLOWED_HOSTS').split(',') if os.environ.get('ALLOWED_HOSTS') else []  # type: ignore
 
@@ -158,7 +159,7 @@
 }
 
 
-PAGE_SIZE = os.environ.get('PAGE_SIZE', 10)
+PAGE_SIZE: int = int(os.environ.get('PAGE_SIZE', 10))
 
 
 CRISPY_ALLOWED_TEMPLATE_PACKS = "bootstrap5"
@@ -189,6 +190,43 @@
     ),
 }
 
+# Logging
+# https://docs.djangoproject.com/en/4.2/topics/logging/
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'simple': {
+            'format': '[%(asctime)s] %(levelname)s | %(name)s | %(message)s',
+            'datefmt': '%Y-%m-%d %H:%M:%S',
+        },
+    },
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+            'formatter': 'simple',
+        },
+        'file': {
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': os.path.join(BASE_DIR, 'logs', 'signal_documentation.log'),
+            'formatter': 'simple',
+            'maxBytes': 1024*1024*15,  # 15MB
+            'backupCount': 10,
+        }
+    },
+    'loggers': {
+        'django': {
+            'handlers': ['file', 'console'],
+            'level': 'INFO',
+        },
+    },
+}
+
+if DEBUG:
+    for logger in LOGGING['loggers']:
+        LOGGING['loggers'][logger]['handlers'] = ['console']
+
 
 # DRF Spectacular settings
 # https://drf-spectacular.readthedocs.io/en/latest/settings.html

src/signals/tools.py

@@ -1,7 +1,10 @@
+import logging
 from datetime import datetime
 
 from signals.models import Signal
 
+logger = logging.getLogger('default')
+
 
 class SignalLastUpdatedParser:
 
@@ -33,9 +36,12 @@ def set_data(self) -> None:
                 try:
                     signal = Signal.objects.get(name=signal_data['signal_basename'], source__name=signal_data['source'])
                 except Signal.DoesNotExist:
-                    # TODO: Log this
+                    logger.warning(
+                        f"Signal {signal_data['signal_basename']} not found in db. Update failed."
+                    )
                     continue
                 signal.last_updated = self.format_date(str(signal_data['max_issue']))
                 signal.from_date = self.format_date(str(signal_data['min_time']))
                 signal.to_date = self.format_date(str(signal_data['max_time']))
                 signal.save()
+                logger.info(f"Signal {signal_data['signal_basename']} successfully updated.")