Merge pull request #205 from Harvard-University-iCommons/patricktonne…

…/TLT-4192/dockerize

Dockerfile

@@ -0,0 +1,14 @@
+# syntax=docker/dockerfile:experimental
+
+FROM 482956169056.dkr.ecr.us-east-1.amazonaws.com/uw/python-postgres-build:v0.5 as build
+COPY lti_emailer/requirements/*.txt /code/
+RUN --mount=type=ssh,id=build_ssh_key ./python_venv/bin/pip3 install gunicorn && ./python_venv/bin/pip3 install -r aws.txt
+COPY . /code/
+RUN chmod a+x /code/docker-entrypoint.sh
+
+FROM 482956169056.dkr.ecr.us-east-1.amazonaws.com/uw/python-postgres-base:v0.5
+COPY --from=build /code /code/
+ENV PYTHONUNBUFFERED 1
+WORKDIR /code
+ENTRYPOINT ["/code/docker-entrypoint.sh"]
+EXPOSE 8000

docker-entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+./python_venv/bin/python3 manage.py migrate                  # Apply database migrations
+./python_venv/bin/python3 manage.py collectstatic --noinput  # Collect static files
+
+# Start Gunicorn processes
+echo Starting Gunicorn.
+exec ./python_venv/bin/gunicorn -c lti_emailer/settings/gunicorn.conf.py lti_emailer.wsgi:application

lti_emailer/requirements/base.txt

@@ -6,12 +6,16 @@ flanker==0.9.0
 hiredis==1.0.0
 psycopg2==2.8.3
 redis==3.3.8
-requests==2.22.0
+requests==2.26.0
 cx_oracle==7.2.2
 django-storages==1.7.2
+splunk_handler==3.0.0
+python-json-logger==2.0.2
+django-allow-cidr==0.3.1
+django-watchman==1.2.0
 
-git+ssh://[email protected]/penzance/[email protected]#egg=canvas-python-sdk==1.0
-git+ssh://[email protected]/Harvard-University-iCommons/django-icommons-common.git@v2.0.2#egg=django-icommons-common==2.0.2
-git+ssh://[email protected]/Harvard-University-iCommons/django-icommons-ui.git@v2.0#egg=django-icommons-ui==2.0
+git+ssh://[email protected]/penzance/canvas_python_sdk.git@v1.2.0#egg=canvas-python-sdk==1.2.0
+git+ssh://[email protected]/Harvard-University-iCommons/django-icommons-common.git@v2.6#egg=django-icommons-common==2.6
+git+ssh://[email protected]/Harvard-University-iCommons/django-icommons-ui.git@v2.1#egg=django-icommons-ui==2.1
 git+ssh://[email protected]/Harvard-University-iCommons/[email protected]#egg=django-auth-lti==2.0.1
-git+https://github.com/Harvard-University-iCommons/django-ssm-parameter-store.git@v0.3#egg=django-ssm-parameter-store==0.3
+git+https://github.com/Harvard-University-iCommons/django-ssm-parameter-store.git@v0.6#egg=django-ssm-parameter-store==0.6

lti_emailer/settings/aws.py

@@ -3,6 +3,7 @@
 
 # tlt hostnames
 ALLOWED_HOSTS = ['.tlt.harvard.edu']
+ALLOWED_CIDR_NETS = [SECURE_SETTINGS.get('vpc_cidr_block')]
 
 # AWS Email Settings
 EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'

lti_emailer/settings/base.py

@@ -10,6 +10,7 @@
 
 SECURE_SETTINGS = load_secure_settings()
 
+
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 # SECURITY WARNING: keep the secret key used in production secret!
@@ -18,6 +19,8 @@
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = SECURE_SETTINGS.get('enable_debug', False)
 
+ENV = SECURE_SETTINGS.get('env_name')
+
 # Application definition
 
 INSTALLED_APPS = [
@@ -33,7 +36,8 @@
     'djng',
     'lti_emailer',
     'mailing_list',
-    'mailgun'
+    'mailgun',
+    'watchman'
 ]
 
 MIDDLEWARE = [
@@ -47,6 +51,7 @@
     'django.contrib.messages.middleware.MessageMiddleware',
     # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'django.middleware.security.SecurityMiddleware',
+    'allow_cidr.middleware.AllowCIDRMiddleware'
 ]
 
 AUTHENTICATION_BACKENDS = (
@@ -168,12 +173,27 @@
 
 # Logging
 
-_DEFAULT_LOG_LEVEL = SECURE_SETTINGS.get('log_level', 'DEBUG')
-_LOG_ROOT = SECURE_SETTINGS.get('log_root', '')  # Default to current directory
-
 # Turn off default Django logging
 # https://docs.djangoproject.com/en/2.2/topics/logging/#disabling-logging-configuration
-LOGGING_CONFIG = None
+# LOGGING_CONFIG = None
+
+_DEFAULT_LOG_LEVEL = SECURE_SETTINGS.get('log_level', logging.DEBUG)
+
+JSON_LOG_FORMAT = '%(asctime)s %(created)f %(exc_info)s %(filename)s %(funcName)s %(levelname)s %(levelno)s %(name)s %(lineno)d %(module)s %(message)s %(pathname)s %(process)s'
+
+
+class ContextFilter(logging.Filter):
+
+    def __init__(self, **kwargs):
+        self.extra = kwargs
+
+    def filter(self, record):
+
+        for k in self.extra:
+            setattr(record, k, self.extra[k])
+
+        return True
+
 
 LOGGING = {
     'version': 1,
@@ -185,84 +205,102 @@
         },
         'simple': {
             'format': '%(levelname)s\t%(name)s:%(lineno)s\t%(message)s',
-        }
+        },
+        'json': {
+            '()': 'pythonjsonlogger.jsonlogger.JsonFormatter',
+            'format': JSON_LOG_FORMAT,
+        },
     },
-    # Borrowing some default filters for app loggers
     'filters': {
         'require_debug_true': {
             '()': 'django.utils.log.RequireDebugTrue',
         },
-    },
-    # This is the default logger for any apps or libraries that use the logger
-    # package, but are not represented in the `loggers` dict below.  A level
-    # must be set and handlers defined.  Setting this logger is equivalent to
-    # setting and empty string logger in the loggers dict below, but the separation
-    # here is a bit more explicit.  See link for more details:
-    # https://docs.python.org/2.7/library/logging.config.html#dictionary-schema-details
-    'root': {
-        'level': logging.WARNING,
-        'handlers': ['console', 'app_logfile'],
+        'context': {
+            '()': 'lti_emailer.settings.base.ContextFilter',
+            'env': ENV,
+            'project': 'lti_emailer',
+            'department': 'uw',
+        }
     },
     'handlers': {
-        # Log to a text file that can be rotated by logrotate
-        'app_logfile': {
+        # By default, log to Splunk
+        'default': {
+            'class': 'splunk_handler.SplunkHandler',
+            'formatter': 'json',
+            'sourcetype': 'json',
+            'source': 'django-lti_emailer',
+            'host': 'http-inputs-harvard.splunkcloud.com',
+            'port': '443',
+            'index': 'soc-isites',
+            'token': SECURE_SETTINGS['splunk_token'],
             'level': _DEFAULT_LOG_LEVEL,
-            'class': 'logging.handlers.WatchedFileHandler',
-            'filename': os.path.join(_LOG_ROOT, 'django-lti_emailer.log'),
-            'formatter': 'verbose',
+            'filters': ['context']
+        },
+        'gunicorn': {
+            'class': 'splunk_handler.SplunkHandler',
+            'formatter': 'json',
+            'sourcetype': 'json',
+            'source': 'gunicorn-lti_emailer',
+            'host': 'http-inputs-harvard.splunkcloud.com',
+            'port': '443',
+            'index': 'soc-isites',
+            'token': SECURE_SETTINGS['splunk_token'],
+            'level': _DEFAULT_LOG_LEVEL,
+            'filters': ['context'],
         },
         'console': {
-            'level': logging.DEBUG,
+            'level': _DEFAULT_LOG_LEVEL,
             'class': 'logging.StreamHandler',
-            'formatter': 'simple',
+            'formatter': 'verbose',
             'filters': ['require_debug_true'],
-        },
+        }
+    },
+    'root': {
+        'level': logging.WARNING,
+        'handlers': ['default'],
     },
     'loggers': {
         # TODO: remove this catch-all handler in favor of app-specific handlers
         '': {
-            'handlers': ['console', 'app_logfile'],
+            'handlers': ['default'],
             'level': _DEFAULT_LOG_LEVEL,
         },
         'django.request': {
-            'handlers': ['console', 'app_logfile'],
+            'handlers': ['default'],
             'level': 'ERROR',
             'propagate': False,
         },
         'django': {
-            'handlers': ['console'],
+            'handlers': ['default'],
             'propagate': False,
         },
         'py.warnings': {
-            'handlers': ['console'],
+            'handlers': ['default'],
             'propagate': False,
         },
         'lti_emailer': {
             'level': _DEFAULT_LOG_LEVEL,
-            'handlers': ['console', 'app_logfile'],
+            'handlers': ['default'],
             'propagate': False,
         },
         'mailgun': {
             'level': _DEFAULT_LOG_LEVEL,
-            'handlers': ['console', 'app_logfile'],
+            'handlers': ['default'],
             'propagate': False,
         },
         'mailing_list': {
             'level': _DEFAULT_LOG_LEVEL,
-            'handlers': ['console', 'app_logfile'],
+            'handlers': ['default'],
             'propagate': False,
         },
         'icommons_common': {
-           'handlers': ['console', 'app_logfile'],
+           'handlers': ['default'],
            'level': _DEFAULT_LOG_LEVEL,
            'propagate': False,
         },
     }
 }
 
-# Currently deployed environment
-ENV_NAME = SECURE_SETTINGS.get('env_name', 'local')
-
 # Other app specific settings
 
 LTI_OAUTH_CREDENTIALS = SECURE_SETTINGS.get('lti_oauth_credentials', None)
@@ -313,3 +351,15 @@
 CACHE_KEY_MESSAGE_HANDLED_TIMEOUT = 60 * 60 * 8  # 8 hours
 
 NO_REPLY_ADDRESS = SECURE_SETTINGS.get('no_reply_address', '[email protected]')
+
+WATCHMAN_TOKENS = SECURE_SETTINGS['watchman_token']
+WATCHMAN_TOKEN_NAME = SECURE_SETTINGS['watchman_token_name']
+WATCHMAN_CHECKS = (
+    'watchman.checks.databases',
+    'watchman.checks.caches',
+)
+
+try:
+    from build_info import BUILD_INFO
+except ImportError:
+    BUILD_INFO = {}

lti_emailer/settings/gunicorn.conf.py

@@ -0,0 +1,9 @@
+# gunicorn configuration
+
+bind = "0.0.0.0:8000"
+workers = 3
+
+# These log settings assume that gunicorn log config will be included in the django base.py logging configuration
+accesslog = "-"
+errorlog = "-"
+access_log_format = '{"request": "%(r)s", "http_status_code": "%(s)s", "http_request_url": "%(U)s", "http_query_string": "%(q)s", "http_verb": "%(m)s", "http_version": "%(H)s", "http_referer": "%(f)s", "x_forwarded_for": "%({x-forwarded-for}i)s", "remote_address": "%(h)s", "request_usec": "%(D)s", "request_sec": "%(L)s"}'

lti_emailer/urls.py

@@ -1,5 +1,7 @@
 from django.urls import path, re_path, include
 from django.conf import settings
+import watchman.views
+
 from lti_emailer import views
 from icommons_ui import views as icommons_ui_views
 
@@ -11,6 +13,8 @@
     re_path(r'^mailing_list/', include(('mailing_list.urls', 'mailing_list'), namespace='mailing_list')),
     re_path(r'^mailgun/', include(('mailgun.urls', 'mailgun'), namespace='mailgun')),
     path('not_authorized/', icommons_ui_views.not_authorized, name="not_authorized"),
+    path('w/', include('watchman.urls')),
+    re_path(r'^status/?$', watchman.views.bare_status),
 ]
 
 if settings.DEBUG:

mailing_list/templates/mailing_list/admin_index.html

@@ -224,7 +224,11 @@ <h4 class="panel-title">
     </div>
   </main>
 </div>
-
 </body>
+
+{# Place build information into an HTML comment to avoid cluttering the page for users #}
+{% if build_info %}
+    <!-- BUILD INFO: {{ build_info.image_tag }} | {{ build_info.image_hash_tag }} | {{ build_info.build_timestamp }} -->
+{% endif %}
 {% endblock body %}
 

mailing_list/views.py

@@ -27,6 +27,7 @@
 def admin_index(request):
     logged_in_user_id = request.LTI['lis_person_sourcedid']
     canvas_course_id = request.LTI.get('custom_canvas_course_id')
+    build_info = settings.BUILD_INFO
 
     course = get_course(canvas_course_id)
 
@@ -39,7 +40,7 @@ def admin_index(request):
 
     logger.info("Rendering mailing_list admin_index view for user %s",
                 logged_in_user_id)
-    return render(request, 'mailing_list/admin_index.html', {'course_name': course_name,})
+    return render(request, 'mailing_list/admin_index.html', {'course_name': course_name, 'build_info': build_info})
 
 
 @login_required