Merge pull request #209 from Harvard-University-iCommons/feature/remo…

…ve-member-counts

Remove member counts; move ML address to CC field

.gitignore

@@ -22,3 +22,4 @@ parameters.*
 # files generated via `runserver_plus` CLI command
 *.crt
 *.key
+.python-version

lti_emailer/canvas_api_client.py

@@ -99,8 +99,8 @@ def get_section(canvas_course_id, section_id):
     return None
 
 
-def get_sections(canvas_course_id):
-    return canvas_api_helper_sections.get_sections(canvas_course_id)
+def get_sections(canvas_course_id, fetch_enrollments=True):
+    return canvas_api_helper_sections.get_sections(canvas_course_id, fetch_enrollments=fetch_enrollments)
 
 
 def get_teaching_staff_enrollments(canvas_course_id):

lti_emailer/requirements/base.txt

@@ -1,4 +1,4 @@
-Django==3.2.13
+Django==3.2.14
 django-cached-authentication-middleware==0.2.2
 django-redis-cache==3.0.1
 flanker==0.9.11

lti_emailer/settings/base.py

@@ -298,7 +298,17 @@ def filter(self, record):
            'level': _DEFAULT_LOG_LEVEL,
            'propagate': False,
         },
-    }
+        'canvas_sdk': {
+           'handlers': ['default'],
+           'level': _DEFAULT_LOG_LEVEL,
+           'propagate': False,
+        },
+        'django_auth_lti': {
+           'handlers': ['default'],
+           'level': 'ERROR',
+           'propagate': False,
+        },
+    },
 }
 
 # Other app specific settings

lti_emailer/settings/local.py

@@ -14,6 +14,12 @@
     'INTERCEPT_REDIRECTS': False,
 }
 
+LOGGING['handlers']['default'] = {
+    'level': DEBUG,
+    'class': 'logging.StreamHandler',
+    'formatter': 'verbose',
+    'filters': ['require_debug_true'],
+}
 dictConfig(LOGGING)
 
 # REST API info needed for selenium_common

lti_emailer/views.py

@@ -30,12 +30,12 @@ def tool_config(request):
     url = "https://{}{}".format(request.get_host(), reverse('lti_launch'))
     url = _url(url)
 
-    title = 'LTI Emailer'
+    title = 'Course Emailer'
     lti_tool_config = ToolConfig(
         title=title,
         launch_url=url,
         secure_launch_url=url,
-        description="This LTI tool allows email functionality for this course site."
+        description="This tool provides an email list for the course and each section associated iwth this course site."
     )
 
     # this is how to tell Canvas that this tool provides a course navigation link:

mailgun/decorators.py

@@ -1,29 +1,39 @@
-import logging
 import hashlib
 import hmac
+import json
+import logging
 import time
-
 from functools import wraps
 
 from django.conf import settings
 from django.shortcuts import redirect
 from django.urls import reverse_lazy
 
-
 logger = logging.getLogger(__name__)
 
 
 def authenticate(redirect_url=reverse_lazy('mailgun:auth_error')):
     def decorator(view_func):
         @wraps(view_func)
         def _wrapped_view(request, *args, **kwargs):
-            try:
-                timestamp = request.POST['timestamp']
-                token = request.POST['token']
-                signature = request.POST['signature']
-            except KeyError as e:
-                logger.error("Received mailgun callback request with missing auth param %s", e)
-                return redirect(redirect_url)
+            logger.debug(f'authenticating webhook request content type {request.content_type}')
+            if request.content_type == 'application/json':
+                payload = json.loads(request.body)
+                try:
+                    timestamp = payload['signature']['timestamp']
+                    token = payload['signature']['token']
+                    signature = payload['signature']['signature']
+                except KeyError:
+                    logger.error(f'no signature found in request: {payload}')
+                    return redirect(redirect_url)
+            else:
+                try:
+                    timestamp = request.POST['timestamp']
+                    token = request.POST['token']
+                    signature = request.POST['signature']
+                except KeyError as e:
+                    logger.error("Received mailgun callback request with missing auth param %s", e)
+                    return redirect(redirect_url)
 
             time_diff = time.time() - float(timestamp)
             if time_diff >= settings.MAILGUN_CALLBACK_TIMEOUT:

mailgun/listserv_client.py

@@ -33,6 +33,9 @@ def send_mail(self, list_address, from_address, to_address, subject='',
                   message_id=None):
         api_url = "%s%s/messages" % (settings.LISTSERV_API_URL,
                                      settings.LISTSERV_DOMAIN)
+
+        logger.debug(f'send_mail called with list_address={list_address} from_address={from_address} to_address={to_address} subject={subject} original_to_address={original_to_address} original_cc_address={original_cc_address} message_id={message_id}')
+
         payload = {
             'from': list_address,
             'h:List-Id': '<{}>'.format(list_address),
@@ -59,19 +62,24 @@ def send_mail(self, list_address, from_address, to_address, subject='',
         #
         # these are prefixed with h: so that mailgun doesn't think we want it
         # to send copies to these addresses as well.
+        cc_list = []
+
         if original_to_address:
-            payload['h:To'] = '%recipient.original_to_address%'
-            recip_var_dict = {'original_to_address': ','.join(original_to_address)}
-        else:
-            recip_var_dict = {}
+            cc_list.extend(original_to_address)
+
         if original_cc_address:
-            payload['h:Cc'] = original_cc_address
+            cc_list.extend(original_cc_address)
+
+        if cc_list:
+            payload['h:cc'] = ','.join(cc_list)
 
         # we accept a single address or a list of addresses in to_address.
         # if it's a list, add in recipient_variables to make sure mailgun
         # doesn't include the whole list in the to: field, per
         #   https://documentation.mailgun.com/user_manual.html#batch-sending
         if not isinstance(to_address, str):
+            # the recipient variables aren't actually used, so we just send a fake dict
+            recip_var_dict = {'k': 'v'}
             recipient_variables = {e: recip_var_dict for e in to_address}
             payload['recipient-variables'] = json.dumps(recipient_variables)
 

mailgun/views.py

@@ -26,5 +26,10 @@ def log_post_data(request):
     :param request:
     :return HttpResponse:
     """
-    logger.info('[MAILGUN EVENT] %s', json.dumps(request.POST, separators=(',', ': '), sort_keys=True))
+    if request.content_type == 'application/json':
+        payload = json.loads(request.body)
+        logger.info(json.dumps(payload['event-data']))
+    else:
+        logger.info(json.dumps(request.POST, separators=(',', ': '), sort_keys=True))
+
     return HttpResponse("Successfully logged post data", status=200)

mailing_list/models.py

@@ -1,11 +1,11 @@
 import logging
-
 import re
+from timeit import default_timer as timer
+
 from django.conf import settings
 from django.core.cache import cache
 from django.db import models
 from flanker.addresslib import address as addresslib_address
-
 from lti_emailer import canvas_api_client
 from mailgun.listserv_client import MailgunClient as ListservClient
 
@@ -83,7 +83,7 @@ def get_or_create_or_delete_mailing_lists_for_canvas_course_id(self, canvas_cour
         :return: List of mailing list dictionaries for the given canvas_course_id
         """
         sis_course_id = canvas_api_client.get_course(canvas_course_id)['sis_course_id']
-        canvas_sections = canvas_api_client.get_sections(canvas_course_id)
+        canvas_sections = canvas_api_client.get_sections(canvas_course_id, fetch_enrollments=False)
         mailing_lists_by_section_id = self._get_mailing_lists_by_section_id(canvas_course_id)
 
         overrides = kwargs.get('defaults', {})
@@ -115,7 +115,6 @@ def get_or_create_or_delete_mailing_lists_for_canvas_course_id(self, canvas_cour
                 'name': 'Course Mailing List',
                 'address': course_list.address,
                 'access_level': course_list.access_level,
-                'members_count': len(course_list.members),
                 'is_course_list': True,
                 'cs_class_type': None,
                 'is_primary': False,
@@ -154,7 +153,6 @@ def get_or_create_or_delete_mailing_lists_for_canvas_course_id(self, canvas_cour
                 'name': s['name'],
                 'address': mailing_list.address,
                 'access_level': mailing_list.access_level,
-                'members_count': len(mailing_list.members),
                 'is_course_list': False,
                 'cs_class_type': cs_class_type,
                 'is_primary': s['sis_section_id'] == sis_course_id,
@@ -246,9 +244,12 @@ def address(self):
 
     @property
     def members(self):
+        start_time = timer()
         mailing_list_emails = self._get_enrolled_email_set()
         if not getattr(settings, 'IGNORE_WHITELIST', False):
             mailing_list_emails = mailing_list_emails.intersection(self._get_whitelist_email_set())
+        end_time = timer()
+        logger.debug(f'called ml.members - took {end_time-start_time}')
         return [{'address': e} for e in mailing_list_emails]
 
     @property

mailing_list/static/mailing_list/js/app.js

@@ -86,14 +86,12 @@
 
     var listsPromise = $http.get(URL_LISTS).success(function(data){
       var length = data.length;
-      var course_list_member_count = 0;
       var list;
 
       for (var i = 0; i < length; i++) {
         list = data[i];
         if(list.is_course_list) {
           ml.courseList.push(list);
-          course_list_member_count = list.members_count;
         } else if(!list.sis_section_id || list.cs_class_type == 'N'){
           ml.nonEnrollmentSectionLists.push(list);
         } else if(list.sis_section_id || list.cs_class_type == 'E'){

mailing_list/templates/mailing_list/_mailing_list_details.html

@@ -20,8 +20,7 @@
         data-original-title="Go to list of names, emails, and roles"
         data-placement="right"
         data-toggle="tooltip"
-        href="{{ ml.listMembersUrl(list) }}"><ng-pluralize count="list.members_count"
-                    when="{'0': 'No members', 'one': '1 member', 'other': '{} members'}"></ng-pluralize></a>)
+        href="{{ ml.listMembersUrl(list) }}">view members</a>)
   </p>
 </div>